From 5c7e38a21a4e9ceeb46a3bdcd3e51755430632da Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Mon, 31 Mar 2025 11:51:17 +0800 Subject: [PATCH] first add version 3.8.0 --- .clang-format | 175 + .clang-tidy | 141 + .dockerignore | 38 + .github/CODEOWNERS | 6 + .github/ISSUE_TEMPLATE/bug-report.md | 39 + .github/ISSUE_TEMPLATE/enhancement.md | 16 + .github/ISSUE_TEMPLATE/feature_request.md | 20 + .github/ISSUE_TEMPLATE/question.md | 18 + .github/Security.md | 30 + .github/actions/tagname-action/README.md | 25 + .github/actions/tagname-action/action.yml | 24 + .../actions/upload-assets-action/README.md | 22 + .../actions/upload-assets-action/action.yml | 40 + .../actions/upload-to-oss-action/README.md | 42 + .../actions/upload-to-oss-action/action.yml | 50 + .github/pull_request_template.md | 42 + .github/workflows/auto_cherry_pick.yml | 38 + .github/workflows/check_label.yml | 22 + .github/workflows/codeql-analysis.yml | 69 + .github/workflows/nightly.yml | 401 + .github/workflows/pull_request.yml | 239 + .github/workflows/rc.yml | 259 + .github/workflows/release.yml | 77 + .gitignore | 71 + .licenserc.yaml | 46 + .linters/cpp/checkKeyword.py | 172 + .linters/cpp/cpplint.py | 6605 +++++++++++++++++ .linters/cpp/hooks/pre-commit.sh | 89 + .linters/java/nebula_java_style_checks.xml | 236 + CMakeLists.txt | 121 + CONTRIBUTING.md | 1 + Coding_Style_Guide.md | 10 + LICENSE | 201 + README-CN.md | 122 + README.md | 135 +- cmake/CPackage.cmake | 172 + cmake/ClangTidy.cmake | 18 + cmake/FetchModule.cmake | 75 + cmake/FindBreakpad.cmake | 33 + cmake/FindBzip2.cmake | 34 + cmake/FindDoubleConversion.cmake | 30 + cmake/FindFatal.cmake | 27 + cmake/FindFbthrift.cmake | 33 + cmake/FindFizz.cmake | 30 + cmake/FindFolly.cmake | 30 + cmake/FindGflags.cmake | 30 + cmake/FindGlog.cmake | 30 + cmake/FindGoogletest.cmake | 30 + cmake/FindJemalloc.cmake | 29 + cmake/FindLibevent.cmake | 30 + cmake/FindLibunwind.cmake | 24 + cmake/FindMoldLinker.cmake | 18 + cmake/FindPCHSupport.cmake | 95 + cmake/FindProxygen.cmake | 31 + cmake/FindRocksdb.cmake | 30 + cmake/FindSnappy.cmake | 30 + cmake/FindSodium.cmake | 30 + cmake/FindWangle.cmake | 30 + cmake/FindZstd.cmake | 32 + cmake/ThriftGenerate.cmake | 138 + cmake/nebula/ABIConfig.cmake | 8 + cmake/nebula/CompilerLauncher.cmake | 29 + cmake/nebula/GeneralCMakeConfig.cmake | 25 + cmake/nebula/GeneralCMakeOptions.cmake | 31 + cmake/nebula/GeneralCompilerConfig.cmake | 93 + cmake/nebula/GitHooksConfig.cmake | 19 + cmake/nebula/GitInfoConfig.cmake | 12 + cmake/nebula/IncludeWhatYouUse.cmake | 10 + cmake/nebula/InstallThirdParty.cmake | 28 + cmake/nebula/LinkerConfig.cmake | 61 + cmake/nebula/MakeBisonRelocatable.cmake | 17 + cmake/nebula/NebulaCMakeMacros.cmake | 166 + cmake/nebula/NebulaCustomTargets.cmake | 30 + cmake/nebula/PlatformCheck.cmake | 3 + cmake/nebula/SanitizerConfig.cmake | 63 + cmake/nebula/ThirdPartyConfig.cmake | 172 + cmake/nebula/TimeTrace.cmake | 6 + codecov.yml | 13 + conf/CMakeLists.txt | 68 + conf/nebula-graphd.conf.default | 134 + conf/nebula-graphd.conf.production | 127 + conf/nebula-metad.conf.default | 53 + conf/nebula-metad.conf.production | 56 + conf/nebula-standalone.conf.default | 163 + conf/nebula-storaged-listener.conf.default | 58 + conf/nebula-storaged-listener.conf.production | 58 + conf/nebula-storaged.conf.default | 143 + conf/nebula-storaged.conf.production | 144 + conf/tuned/README.md | 8 + conf/tuned/nebula/tuned.conf | 32 + docker/Dockerfile | 66 + docker/Dockerfile.graphd | 24 + docker/Dockerfile.metad | 24 + docker/Dockerfile.storaged | 24 + docker/Dockerfile.tools | 21 + docker/README.md | 8 + docs/rfcs/0000-template.md | 64 + package/package.sh | 268 + package/postinst | 12 + package/rpm_postinst | 11 + resources/CMakeLists.txt | 17 + resources/README.md | 3 + resources/date_time_zonespec.csv | 520 ++ resources/gflags.json | 26 + scripts/CMakeLists.txt | 113 + scripts/README.md | 29 + scripts/meta-transfer-tools.sh | 91 + scripts/nebula-graphd.service | 6 + scripts/nebula-metad.service | 6 + scripts/nebula-standalone.service | 6 + scripts/nebula-storaged-listener.service | 6 + scripts/nebula-storaged.service | 6 + scripts/nebula.service | 301 + scripts/services.sh | 68 + scripts/utils.sh | 162 + src/CMakeLists.txt | 19 + src/clients/CMakeLists.txt | 6 + src/clients/graph/README.md | 8 + src/clients/meta/CMakeLists.txt | 16 + src/clients/meta/FileBasedClusterIdMan.cpp | 59 + src/clients/meta/FileBasedClusterIdMan.h | 28 + src/clients/meta/MetaClient.cpp | 3898 ++++++++++ src/clients/meta/MetaClient.h | 869 +++ src/clients/meta/stats/CMakeLists.txt | 9 + src/clients/meta/stats/MetaClientStats.cpp | 19 + src/clients/meta/stats/MetaClientStats.h | 18 + src/clients/meta/test/CMakeLists.txt | 14 + .../meta/test/FileBasedClusterIdManTest.cpp | 33 + src/clients/storage/CMakeLists.txt | 23 + src/clients/storage/GeneralStorageClient.h | 54 + src/clients/storage/InternalStorageClient.cpp | 183 + src/clients/storage/InternalStorageClient.h | 64 + src/clients/storage/StorageClient.cpp | 927 +++ src/clients/storage/StorageClient.h | 200 + src/clients/storage/StorageClientBase-inl.h | 323 + src/clients/storage/StorageClientBase.cpp | 15 + src/clients/storage/StorageClientBase.h | 214 + src/clients/storage/stats/CMakeLists.txt | 9 + .../storage/stats/StorageClientStats.cpp | 20 + .../storage/stats/StorageClientStats.h | 19 + src/codec/CMakeLists.txt | 12 + src/codec/Common.h | 111 + src/codec/NebulaCodecImpl.cpp | 123 + src/codec/RowReaderV2.cpp | 220 + src/codec/RowReaderV2.h | 69 + src/codec/RowReaderWrapper.cpp | 176 + src/codec/RowReaderWrapper.h | 258 + src/codec/RowWriterV2.cpp | 958 +++ src/codec/RowWriterV2.h | 269 + src/codec/include/NebulaCodec.h | 29 + src/codec/test/CMakeLists.txt | 78 + src/codec/test/RowReaderV2Test.cpp | 330 + src/codec/test/RowWriterBenchmark.cpp | 110 + src/codec/test/RowWriterV1-inl.h | 49 + src/codec/test/RowWriterV1.cpp | 233 + src/codec/test/RowWriterV1.h | 90 + src/codec/test/RowWriterV2Test.cpp | 818 ++ src/common/CMakeLists.txt | 29 + src/common/algorithm/CMakeLists.txt | 10 + src/common/algorithm/ReservoirSampling.cpp | 10 + src/common/algorithm/ReservoirSampling.h | 62 + src/common/algorithm/test/CMakeLists.txt | 10 + .../algorithm/test/ReservoirSamplingTest.cpp | 44 + src/common/base/Arena.cpp | 44 + src/common/base/Arena.h | 90 + src/common/base/Base.cpp | 55 + src/common/base/Base.h | 167 + src/common/base/CMakeLists.txt | 20 + src/common/base/CheckPointer.h | 35 + src/common/base/CollectNSucceeded-inl.h | 71 + src/common/base/CollectNSucceeded.h | 74 + src/common/base/CommonMacro.h | 19 + src/common/base/ConcurrentLRUCache.h | 295 + src/common/base/Cord.cpp | 210 + src/common/base/Cord.h | 74 + src/common/base/EitherOr.h | 477 ++ src/common/base/ErrorOr.h | 72 + src/common/base/GdbDebugScript.cpp | 19 + src/common/base/ICord.h | 250 + src/common/base/Logging.h | 199 + src/common/base/MurmurHash2.h | 125 + src/common/base/ObjectPool.h | 86 + src/common/base/SanitizerOptions.cpp | 69 + src/common/base/SignalHandler.cpp | 145 + src/common/base/SignalHandler.h | 114 + src/common/base/Status.cpp | 111 + src/common/base/Status.h | 256 + src/common/base/StatusOr.h | 358 + src/common/base/test/ArenaBenchmark.cpp | 88 + src/common/base/test/ArenaTest.cpp | 44 + src/common/base/test/CMakeLists.txt | 133 + .../base/test/ConcurrentLRUCacheTest.cpp | 167 + src/common/base/test/CordTest.cpp | 30 + src/common/base/test/CordTestT.cpp | 190 + src/common/base/test/EitherOrTest.cpp | 369 + src/common/base/test/ErrorOrTest.cpp | 67 + src/common/base/test/HashBenchmark.cpp | 143 + src/common/base/test/ICordBenchmark.cpp | 143 + src/common/base/test/ICordTest.cpp | 34 + src/common/base/test/LoggingBenchmark.cpp | 144 + src/common/base/test/MurmurHash2Test.cpp | 89 + src/common/base/test/ObjectPoolTest.cpp | 36 + .../base/test/RangeVsTransformBenchmark.cpp | 124 + src/common/base/test/SignalHandlerTest.cpp | 39 + src/common/base/test/StatusOrTest.cpp | 437 ++ src/common/base/test/StatusTest.cpp | 120 + src/common/charset/CMakeLists.txt | 10 + src/common/charset/Charset.cpp | 57 + src/common/charset/Charset.h | 89 + src/common/charset/test/CMakeLists.txt | 16 + src/common/charset/test/CharsetTest.cpp | 96 + src/common/conf/CMakeLists.txt | 10 + src/common/conf/Configuration.cpp | 281 + src/common/conf/Configuration.h | 71 + src/common/conf/test/CMakeLists.txt | 10 + src/common/conf/test/ConfigurationTest.cpp | 234 + src/common/context/ExpressionContext.h | 94 + src/common/cpp/helpers.h | 25 + src/common/datatypes/CMakeLists.txt | 20 + src/common/datatypes/CommonCpp2Ops.h | 60 + src/common/datatypes/DataSet.h | 226 + src/common/datatypes/DataSetOps-inl.h | 198 + src/common/datatypes/Date.cpp | 363 + src/common/datatypes/Date.h | 331 + src/common/datatypes/DateOps-inl.h | 723 ++ src/common/datatypes/Duration.cpp | 15 + src/common/datatypes/Duration.h | 154 + src/common/datatypes/DurationOps-inl.h | 203 + src/common/datatypes/Edge.cpp | 179 + src/common/datatypes/Edge.h | 107 + src/common/datatypes/EdgeOps-inl.h | 294 + src/common/datatypes/Geography.cpp | 395 + src/common/datatypes/Geography.h | 245 + src/common/datatypes/GeographyOps-inl.h | 684 ++ src/common/datatypes/HostAddr.cpp | 47 + src/common/datatypes/HostAddr.h | 94 + src/common/datatypes/HostAddrOps-inl.h | 167 + src/common/datatypes/KeyValue.h | 58 + src/common/datatypes/KeyValueOps-inl.h | 159 + src/common/datatypes/List.cpp | 44 + src/common/datatypes/List.h | 151 + src/common/datatypes/ListOps-inl.h | 140 + src/common/datatypes/Map.cpp | 103 + src/common/datatypes/Map.h | 95 + src/common/datatypes/MapOps-inl.h | 141 + src/common/datatypes/Path.cpp | 100 + src/common/datatypes/Path.h | 250 + src/common/datatypes/PathOps-inl.h | 425 ++ src/common/datatypes/Set.cpp | 69 + src/common/datatypes/Set.h | 80 + src/common/datatypes/SetOps-inl.h | 137 + src/common/datatypes/Value.cpp | 3042 ++++++++ src/common/datatypes/Value.h | 609 ++ src/common/datatypes/ValueOps-inl.h | 782 ++ src/common/datatypes/Vertex.cpp | 158 + src/common/datatypes/Vertex.h | 139 + src/common/datatypes/VertexOps-inl.h | 314 + src/common/datatypes/test/CMakeLists.txt | 155 + src/common/datatypes/test/DataSetTest.cpp | 48 + src/common/datatypes/test/DateTest.cpp | 146 + src/common/datatypes/test/EdgeBenchmark.cpp | 188 + src/common/datatypes/test/EdgeTest.cpp | 146 + src/common/datatypes/test/GeographyTest.cpp | 73 + src/common/datatypes/test/HostAddrTest.cpp | 43 + src/common/datatypes/test/PathTest.cpp | 258 + src/common/datatypes/test/ValueBenchmark.cpp | 129 + src/common/datatypes/test/ValueTest.cpp | 1888 +++++ src/common/datatypes/test/ValueToJsonTest.cpp | 318 + src/common/expression/AggregateExpression.cpp | 75 + src/common/expression/AggregateExpression.h | 109 + .../expression/ArithmeticExpression.cpp | 70 + src/common/expression/ArithmeticExpression.h | 76 + src/common/expression/AttributeExpression.cpp | 110 + src/common/expression/AttributeExpression.h | 48 + src/common/expression/BinaryExpression.cpp | 41 + src/common/expression/BinaryExpression.h | 60 + src/common/expression/CMakeLists.txt | 38 + src/common/expression/CaseExpression.cpp | 167 + src/common/expression/CaseExpression.h | 152 + src/common/expression/ColumnExpression.cpp | 44 + src/common/expression/ColumnExpression.h | 54 + src/common/expression/ConstantExpression.cpp | 44 + src/common/expression/ConstantExpression.h | 61 + src/common/expression/ContainerExpression.cpp | 236 + src/common/expression/ContainerExpression.h | 294 + src/common/expression/EdgeExpression.cpp | 21 + src/common/expression/EdgeExpression.h | 60 + src/common/expression/ExprVisitor.h | 99 + src/common/expression/ExprVisitorImpl.cpp | 212 + src/common/expression/ExprVisitorImpl.h | 74 + src/common/expression/Expression.cpp | 740 ++ src/common/expression/Expression.h | 231 + .../expression/FunctionCallExpression.cpp | 102 + .../expression/FunctionCallExpression.h | 143 + .../expression/LabelAttributeExpression.cpp | 33 + .../expression/LabelAttributeExpression.h | 93 + src/common/expression/LabelExpression.cpp | 46 + src/common/expression/LabelExpression.h | 54 + .../ListComprehensionExpression.cpp | 166 + .../expression/ListComprehensionExpression.h | 130 + src/common/expression/LogicalExpression.cpp | 176 + src/common/expression/LogicalExpression.h | 128 + .../expression/MatchPathPatternExpression.cpp | 50 + .../expression/MatchPathPatternExpression.h | 83 + src/common/expression/PathBuildExpression.cpp | 200 + src/common/expression/PathBuildExpression.h | 75 + src/common/expression/PredicateExpression.cpp | 289 + src/common/expression/PredicateExpression.h | 131 + src/common/expression/PropertyExpression.cpp | 247 + src/common/expression/PropertyExpression.h | 413 ++ src/common/expression/ReduceExpression.cpp | 131 + src/common/expression/ReduceExpression.h | 130 + .../expression/RelationalExpression.cpp | 272 + src/common/expression/RelationalExpression.h | 136 + src/common/expression/SubscriptExpression.cpp | 259 + src/common/expression/SubscriptExpression.h | 133 + .../expression/TextSearchExpression.cpp | 50 + src/common/expression/TextSearchExpression.h | 106 + .../expression/TypeCastingExpression.cpp | 137 + src/common/expression/TypeCastingExpression.h | 70 + src/common/expression/UUIDExpression.cpp | 39 + src/common/expression/UUIDExpression.h | 45 + src/common/expression/UnaryExpression.cpp | 136 + src/common/expression/UnaryExpression.h | 95 + src/common/expression/VariableExpression.cpp | 73 + src/common/expression/VariableExpression.h | 124 + src/common/expression/VertexExpression.cpp | 42 + src/common/expression/VertexExpression.h | 61 + .../test/AggregateExpressionBenchmark.cpp | 43 + .../test/AggregateExpressionTest.cpp | 352 + .../test/ArithmeticExpressionTest.cpp | 121 + .../test/AttributeExpressionTest.cpp | 188 + src/common/expression/test/CMakeLists.txt | 166 + .../expression/test/CaseExpressionTest.cpp | 236 + .../expression/test/ColumnExpressionTest.cpp | 77 + .../test/ConstantExpressionTest.cpp | 105 + .../test/ContainerExpressionTest.cpp | 104 + .../expression/test/EncodeDecodeTest.cpp | 525 ++ .../expression/test/ExpressionBenchmark.cpp | 221 + .../expression/test/ExpressionContextMock.cpp | 66 + .../expression/test/ExpressionContextMock.h | 159 + src/common/expression/test/ExpressionTest.cpp | 270 + .../test/FunctionCallExpressionBenchmark.cpp | 42 + .../test/FunctionCallExpressionTest.cpp | 132 + .../expression/test/LabelExpressionTest.cpp | 22 + .../test/ListComprehensionExpressionTest.cpp | 123 + .../expression/test/LogicalExpressionTest.cpp | 567 ++ .../test/PathBuildExpressionTest.cpp | 128 + .../test/PredicateExpressionTest.cpp | 148 + .../test/PropertyExpressionTest.cpp | 129 + .../expression/test/ReduceExpressionTest.cpp | 58 + .../test/RelationalExpressionTest.cpp | 1407 ++++ .../test/SubscriptExpressionTest.cpp | 458 ++ src/common/expression/test/TestBase.cpp | 32 + src/common/expression/test/TestBase.h | 181 + .../test/TypeCastingExpressionTest.cpp | 160 + .../expression/test/UnaryExpressionTest.cpp | 179 + .../test/VersionedVariableExpressionTest.cpp | 47 + src/common/fs/CMakeLists.txt | 12 + src/common/fs/FileUtils.cpp | 524 ++ src/common/fs/FileUtils.h | 263 + src/common/fs/TempDir.cpp | 38 + src/common/fs/TempDir.h | 45 + src/common/fs/TempFile.cpp | 37 + src/common/fs/TempFile.h | 42 + src/common/fs/test/CMakeLists.txt | 24 + src/common/fs/test/FileUtilsTest.cpp | 361 + src/common/fs/test/TempDirTest.cpp | 76 + src/common/fs/test/TempFileTest.cpp | 61 + src/common/function/AggFunctionManager.cpp | 318 + src/common/function/AggFunctionManager.h | 163 + src/common/function/CMakeLists.txt | 18 + src/common/function/FunctionManager.cpp | 3085 ++++++++ src/common/function/FunctionManager.h | 116 + src/common/function/FunctionUdfManager.cpp | 204 + src/common/function/FunctionUdfManager.h | 40 + src/common/function/GraphFunction.h | 38 + .../function/test/AggFunctionManagerTest.cpp | 138 + src/common/function/test/CMakeLists.txt | 56 + .../function/test/FunctionManagerTest.cpp | 2083 ++++++ .../test/TwiceTimezoneConversionTest.cpp | 62 + src/common/geo/CMakeLists.txt | 11 + src/common/geo/GeoFunction.cpp | 553 ++ src/common/geo/GeoFunction.h | 77 + src/common/geo/GeoIndex.cpp | 198 + src/common/geo/GeoIndex.h | 105 + src/common/geo/GeoUtils.h | 100 + src/common/geo/io/CMakeLists.txt | 39 + src/common/geo/io/wkb/ByteOrder.h | 80 + .../geo/io/wkb/ByteOrderDataIOStream.cpp | 51 + src/common/geo/io/wkb/ByteOrderDataIOStream.h | 77 + src/common/geo/io/wkb/CMakeLists.txt | 5 + src/common/geo/io/wkb/WKBReader.cpp | 128 + src/common/geo/io/wkb/WKBReader.h | 49 + src/common/geo/io/wkb/WKBWriter.cpp | 80 + src/common/geo/io/wkb/WKBWriter.h | 47 + src/common/geo/io/wkb/test/CMakeLists.txt | 13 + src/common/geo/io/wkb/test/WKBTest.cpp | 180 + src/common/geo/io/wkt/CMakeLists.txt | 5 + src/common/geo/io/wkt/WKTReader.cpp | 8 + src/common/geo/io/wkt/WKTReader.h | 85 + src/common/geo/io/wkt/WKTScanner.h | 79 + src/common/geo/io/wkt/WKTWriter.cpp | 89 + src/common/geo/io/wkt/WKTWriter.h | 35 + src/common/geo/io/wkt/test/CMakeLists.txt | 13 + src/common/geo/io/wkt/test/WKTTest.cpp | 110 + src/common/geo/io/wkt/wkt_parser.yy | 186 + src/common/geo/io/wkt/wkt_scanner.lex | 63 + src/common/geo/test/CMakeLists.txt | 46 + src/common/geo/test/GeoFunctionTest.cpp | 1187 +++ src/common/geo/test/GeoIndexTest.cpp | 318 + src/common/graph/AuthResponseOps-inl.h | 293 + src/common/graph/CMakeLists.txt | 10 + src/common/graph/ExecutionResponseOps-inl.h | 348 + src/common/graph/GraphCpp2Ops.h | 32 + src/common/graph/PairOps-inl.h | 167 + src/common/graph/PlanDescriptionOps-inl.h | 258 + src/common/graph/PlanNodeBranchInfoOps-inl.h | 179 + src/common/graph/PlanNodeDescriptionOps-inl.h | 363 + src/common/graph/ProfilingStatsOps-inl.h | 265 + src/common/graph/Response.cpp | 40 + src/common/graph/Response.h | 625 ++ src/common/graph/tests/CMakeLists.txt | 24 + .../graph/tests/ResponseEncodeDecodeTest.cpp | 167 + src/common/hdfs/CMakeLists.txt | 8 + src/common/hdfs/HdfsCommandHelper.cpp | 72 + src/common/hdfs/HdfsCommandHelper.h | 29 + src/common/hdfs/HdfsHelper.h | 32 + src/common/http/CMakeLists.txt | 7 + src/common/http/HttpClient.cpp | 178 + src/common/http/HttpClient.h | 99 + src/common/http/test/CMakeLists.txt | 19 + src/common/http/test/HttpClientTest.cpp | 54 + src/common/id/CMakeLists.txt | 20 + src/common/id/SegmentId.cpp | 70 + src/common/id/SegmentId.h | 52 + src/common/id/Snowflake.cpp | 66 + src/common/id/Snowflake.h | 60 + src/common/id/UUID.cpp | 20 + src/common/id/UUID.h | 28 + src/common/id/test/CMakeLists.txt | 109 + src/common/id/test/MockMetaClient.h | 22 + src/common/id/test/SegmentIdBenchmark.cpp | 86 + src/common/id/test/SegmentIdTest.cpp | 85 + src/common/id/test/SnowflakeBenchmark.cpp | 81 + src/common/id/test/SnowflakeTest.cpp | 64 + src/common/log/CMakeLists.txt | 8 + src/common/log/LogMonitor.cpp | 69 + src/common/log/LogMonitor.h | 45 + src/common/memory/CMakeLists.txt | 12 + src/common/memory/Memory.h | 182 + src/common/memory/MemoryTracker.cpp | 49 + src/common/memory/MemoryTracker.h | 225 + src/common/memory/MemoryUtils.cpp | 246 + src/common/memory/MemoryUtils.h | 49 + src/common/memory/NewDelete.cpp | 102 + src/common/memory/test/CMakeLists.txt | 21 + src/common/memory/test/MemoryUtilsTest.cpp | 50 + .../memory/test/SysInfoReadBenchmark.cpp | 60 + src/common/meta/CMakeLists.txt | 14 + src/common/meta/ClusterIdManBase.h | 44 + src/common/meta/Common.h | 69 + src/common/meta/GflagsManager.cpp | 217 + src/common/meta/GflagsManager.h | 42 + src/common/meta/IndexManager.cpp | 12 + src/common/meta/IndexManager.h | 43 + src/common/meta/NebulaSchemaProvider.cpp | 202 + src/common/meta/NebulaSchemaProvider.h | 205 + src/common/meta/SchemaManager.cpp | 39 + src/common/meta/SchemaManager.h | 123 + src/common/meta/SchemaProviderIf.cpp | 11 + src/common/meta/SchemaProviderIf.h | 134 + src/common/meta/ServerBasedIndexManager.cpp | 75 + src/common/meta/ServerBasedIndexManager.h | 45 + src/common/meta/ServerBasedSchemaManager.cpp | 178 + src/common/meta/ServerBasedSchemaManager.h | 87 + src/common/network/CMakeLists.txt | 10 + src/common/network/NetworkUtils.cpp | 338 + src/common/network/NetworkUtils.h | 61 + src/common/network/test/CMakeLists.txt | 34 + .../network/test/NetworkUtilsBenchmark.cpp | 55 + src/common/network/test/NetworkUtilsTest.cpp | 134 + src/common/plugin/CMakeLists.txt | 5 + src/common/plugin/fulltext/CMakeLists.txt | 11 + .../fulltext/elasticsearch/ESAdapter.cpp | 273 + .../plugin/fulltext/elasticsearch/ESAdapter.h | 105 + .../fulltext/elasticsearch/ESClient.cpp | 147 + .../plugin/fulltext/elasticsearch/ESClient.h | 64 + .../plugin/fulltext/test/CMakeLists.txt | 20 + .../fulltext/test/ESQueryResultExample.h | 57 + .../fulltext/test/ElasticsearchTest.cpp | 346 + src/common/process/CMakeLists.txt | 10 + src/common/process/ProcessUtils.cpp | 164 + src/common/process/ProcessUtils.h | 62 + src/common/process/test/CMakeLists.txt | 17 + src/common/process/test/ProcessTest.cpp | 110 + src/common/session/SessionManager.h | 125 + src/common/ssl/CMakeLists.txt | 9 + src/common/ssl/SSLConfig.cpp | 39 + src/common/ssl/SSLConfig.h | 27 + src/common/stats/CMakeLists.txt | 11 + src/common/stats/StatsManager-inl.h | 35 + src/common/stats/StatsManager.cpp | 554 ++ src/common/stats/StatsManager.h | 220 + src/common/stats/test/CMakeLists.txt | 67 + .../stats/test/StatsManagerBenchmark.cpp | 86 + .../stats/test/StatsManagerCrossLevelTest.cpp | 53 + .../stats/test/StatsManagerRateTest.cpp | 45 + src/common/stats/test/StatsManagerTest.cpp | 175 + src/common/test/ServerContext.h | 63 + src/common/thread/CMakeLists.txt | 12 + src/common/thread/GenericThreadPool.cpp | 59 + src/common/thread/GenericThreadPool.h | 178 + src/common/thread/GenericWorker.cpp | 185 + src/common/thread/GenericWorker.h | 262 + src/common/thread/NamedThread.cpp | 41 + src/common/thread/NamedThread.h | 73 + src/common/thread/test/CMakeLists.txt | 18 + .../thread/test/GenericThreadPoolTest.cpp | 139 + src/common/thread/test/GenericWorkerTest.cpp | 139 + src/common/thread/test/ThreadTest.cpp | 28 + src/common/thrift/CMakeLists.txt | 8 + src/common/thrift/ThriftClientManager-inl.h | 101 + src/common/thrift/ThriftClientManager.cpp | 10 + src/common/thrift/ThriftClientManager.h | 48 + src/common/thrift/ThriftCpp2OpsHelper.h | 26 + src/common/thrift/ThriftLocalClientManager.h | 41 + src/common/thrift/ThriftTypes.h | 50 + src/common/time/CMakeLists.txt | 21 + src/common/time/Constants.h | 21 + src/common/time/Duration.cpp | 124 + src/common/time/Duration.h | 48 + src/common/time/ScopedTimer.h | 58 + src/common/time/TimeConversion.cpp | 95 + src/common/time/TimeConversion.h | 103 + src/common/time/TimeUtils.cpp | 231 + src/common/time/TimeUtils.h | 231 + src/common/time/TimezoneInfo.cpp | 50 + src/common/time/TimezoneInfo.h | 88 + src/common/time/WallClock.cpp | 61 + src/common/time/WallClock.h | 39 + src/common/time/detail/TscHelper.cpp | 101 + src/common/time/detail/TscHelper.h | 47 + src/common/time/parser/CMakeLists.txt | 34 + src/common/time/parser/DatetimeReader.cpp | 61 + src/common/time/parser/DatetimeReader.h | 64 + src/common/time/parser/DatetimeScanner.h | 78 + src/common/time/parser/Result.h | 52 + src/common/time/parser/datetime_parser.yy | 318 + src/common/time/parser/datetime_scanner.lex | 91 + src/common/time/parser/test/CMakeLists.txt | 20 + .../time/parser/test/DateTimeParserTest.cpp | 390 + src/common/time/test/CMakeLists.txt | 79 + src/common/time/test/DurationBenchmark.cpp | 50 + src/common/time/test/DurationTest.cpp | 51 + src/common/time/test/ScopedTimerTest.cpp | 47 + src/common/time/test/TimeUtilsTest.cpp | 187 + src/common/time/test/TimezoneInfoTest.cpp | 48 + src/common/time/test/WallClockBenchmark.cpp | 91 + src/common/time/test/WallClockTest.cpp | 43 + src/common/utils/CMakeLists.txt | 25 + src/common/utils/DefaultValueContext.h | 95 + src/common/utils/IndexKeyUtils.cpp | 266 + src/common/utils/IndexKeyUtils.h | 563 ++ src/common/utils/LogIterator.h | 32 + src/common/utils/LogStrListIterator.cpp | 49 + src/common/utils/LogStrListIterator.h | 37 + src/common/utils/MemoryLockCore.h | 87 + src/common/utils/MemoryLockWrapper.h | 88 + src/common/utils/MetaKeyUtils.cpp | 1550 ++++ src/common/utils/MetaKeyUtils.h | 493 ++ src/common/utils/NebulaKeyUtils.cpp | 326 + src/common/utils/NebulaKeyUtils.h | 319 + src/common/utils/OperationKeyUtils.cpp | 70 + src/common/utils/OperationKeyUtils.h | 39 + src/common/utils/Types.h | 69 + src/common/utils/Utils.h | 53 + src/common/utils/test/CMakeLists.txt | 151 + src/common/utils/test/IndexKeyUtilsTest.cpp | 608 ++ src/common/utils/test/MetaKeyUtilsTest.cpp | 240 + src/common/utils/test/NebulaKeyUtilsTest.cpp | 199 + .../utils/test/OperationKeyUtilsTest.cpp | 33 + src/console/CMakeLists.txt | 39 + src/console/compile-console.sh | 10 + src/daemons/CMakeLists.txt | 277 + src/daemons/GraphDaemon.cpp | 186 + src/daemons/MetaDaemon.cpp | 250 + src/daemons/MetaDaemonInit.cpp | 238 + src/daemons/MetaDaemonInit.h | 25 + src/daemons/SetupBreakpad.cpp | 29 + src/daemons/SetupLogging.cpp | 63 + src/daemons/SetupLogging.h | 18 + src/daemons/StandAloneDaemon.cpp | 438 ++ src/daemons/StorageDaemon.cpp | 209 + src/graph/CMakeLists.txt | 16 + src/graph/context/CMakeLists.txt | 20 + src/graph/context/ExecutionContext.cpp | 123 + src/graph/context/ExecutionContext.h | 88 + src/graph/context/Iterator.h | 16 + src/graph/context/QueryContext.cpp | 46 + src/graph/context/QueryContext.h | 169 + src/graph/context/QueryExpressionContext.cpp | 120 + src/graph/context/QueryExpressionContext.h | 92 + src/graph/context/Result.cpp | 40 + src/graph/context/Result.h | 164 + src/graph/context/Symbols.cpp | 129 + src/graph/context/Symbols.h | 92 + src/graph/context/ValidateContext.h | 108 + src/graph/context/ast/AstContext.h | 24 + src/graph/context/ast/CypherAstContext.h | 269 + src/graph/context/ast/QueryAstContext.h | 188 + src/graph/context/iterator/DefaultIter.h | 98 + .../iterator/GetNbrsRespDataSetIter.cpp | 200 + .../context/iterator/GetNbrsRespDataSetIter.h | 63 + .../context/iterator/GetNeighborsIter.cpp | 605 ++ src/graph/context/iterator/GetNeighborsIter.h | 193 + src/graph/context/iterator/Iterator.cpp | 52 + src/graph/context/iterator/Iterator.h | 173 + src/graph/context/iterator/PropIter.cpp | 252 + src/graph/context/iterator/PropIter.h | 78 + src/graph/context/iterator/SequentialIter.cpp | 167 + src/graph/context/iterator/SequentialIter.h | 118 + src/graph/context/test/CMakeLists.txt | 108 + .../context/test/ExecutionContextTest.cpp | 63 + .../context/test/ExpressionContextTest.cpp | 30 + .../context/test/InputVarPropBenchmark.cpp | 49 + src/graph/context/test/IteratorBenchmark.cpp | 187 + src/graph/context/test/IteratorTest.cpp | 848 +++ src/graph/executor/CMakeLists.txt | 97 + src/graph/executor/ExecutionError.h | 30 + src/graph/executor/Executor.cpp | 772 ++ src/graph/executor/Executor.h | 203 + src/graph/executor/StorageAccessExecutor.cpp | 232 + src/graph/executor/StorageAccessExecutor.h | 191 + src/graph/executor/admin/AddHostsExecutor.cpp | 30 + src/graph/executor/admin/AddHostsExecutor.h | 27 + .../executor/admin/ChangePasswordExecutor.cpp | 38 + .../executor/admin/ChangePasswordExecutor.h | 27 + src/graph/executor/admin/CharsetExecutor.cpp | 49 + src/graph/executor/admin/CharsetExecutor.h | 31 + src/graph/executor/admin/ConfigExecutor.cpp | 120 + src/graph/executor/admin/ConfigExecutor.h | 58 + .../executor/admin/CreateUserExecutor.cpp | 38 + src/graph/executor/admin/CreateUserExecutor.h | 27 + .../executor/admin/DescribeUserExecutor.cpp | 56 + .../executor/admin/DescribeUserExecutor.h | 27 + .../executor/admin/DropHostsExecutor.cpp | 29 + src/graph/executor/admin/DropHostsExecutor.h | 24 + src/graph/executor/admin/DropUserExecutor.cpp | 34 + src/graph/executor/admin/DropUserExecutor.h | 27 + .../executor/admin/GrantRoleExecutor.cpp | 50 + src/graph/executor/admin/GrantRoleExecutor.h | 28 + .../executor/admin/KillQueryExecutor.cpp | 177 + src/graph/executor/admin/KillQueryExecutor.h | 35 + .../executor/admin/ListRolesExecutor.cpp | 59 + src/graph/executor/admin/ListRolesExecutor.h | 27 + .../executor/admin/ListUserRolesExecutor.cpp | 41 + .../executor/admin/ListUserRolesExecutor.h | 27 + .../executor/admin/ListUsersExecutor.cpp | 36 + src/graph/executor/admin/ListUsersExecutor.h | 27 + src/graph/executor/admin/ListenerExecutor.cpp | 87 + src/graph/executor/admin/ListenerExecutor.h | 40 + src/graph/executor/admin/PartExecutor.cpp | 57 + src/graph/executor/admin/PartExecutor.h | 24 + .../executor/admin/RevokeRoleExecutor.cpp | 52 + src/graph/executor/admin/RevokeRoleExecutor.h | 27 + src/graph/executor/admin/SessionExecutor.cpp | 154 + src/graph/executor/admin/SessionExecutor.h | 57 + .../executor/admin/ShowHostsExecutor.cpp | 159 + src/graph/executor/admin/ShowHostsExecutor.h | 29 + .../executor/admin/ShowMetaLeaderExecutor.cpp | 30 + .../executor/admin/ShowMetaLeaderExecutor.h | 23 + .../executor/admin/ShowQueriesExecutor.cpp | 91 + .../executor/admin/ShowQueriesExecutor.h | 29 + .../admin/ShowServiceClientsExecutor.cpp | 46 + .../admin/ShowServiceClientsExecutor.h | 27 + .../executor/admin/ShowStatsExecutor.cpp | 80 + src/graph/executor/admin/ShowStatsExecutor.h | 24 + .../executor/admin/SignInServiceExecutor.cpp | 35 + .../executor/admin/SignInServiceExecutor.h | 27 + .../executor/admin/SignOutServiceExecutor.cpp | 33 + .../executor/admin/SignOutServiceExecutor.h | 27 + src/graph/executor/admin/SnapshotExecutor.cpp | 70 + src/graph/executor/admin/SnapshotExecutor.h | 49 + src/graph/executor/admin/SpaceExecutor.cpp | 310 + src/graph/executor/admin/SpaceExecutor.h | 81 + .../executor/admin/SubmitJobExecutor.cpp | 220 + src/graph/executor/admin/SubmitJobExecutor.h | 31 + .../executor/admin/SwitchSpaceExecutor.cpp | 51 + .../executor/admin/SwitchSpaceExecutor.h | 24 + .../executor/admin/UpdateUserExecutor.cpp | 36 + src/graph/executor/admin/UpdateUserExecutor.h | 27 + src/graph/executor/admin/ZoneExecutor.cpp | 145 + src/graph/executor/admin/ZoneExecutor.h | 72 + src/graph/executor/algo/AllPathsExecutor.cpp | 764 ++ src/graph/executor/algo/AllPathsExecutor.h | 139 + .../executor/algo/BFSShortestPathExecutor.cpp | 283 + .../executor/algo/BFSShortestPathExecutor.h | 78 + src/graph/executor/algo/BatchShortestPath.cpp | 594 ++ src/graph/executor/algo/BatchShortestPath.h | 75 + .../algo/CartesianProductExecutor.cpp | 71 + .../executor/algo/CartesianProductExecutor.h | 35 + .../algo/MultiShortestPathExecutor.cpp | 404 + .../executor/algo/MultiShortestPathExecutor.h | 95 + src/graph/executor/algo/ShortestPathBase.cpp | 229 + src/graph/executor/algo/ShortestPathBase.h | 100 + .../executor/algo/ShortestPathExecutor.cpp | 69 + .../executor/algo/ShortestPathExecutor.h | 33 + .../executor/algo/SingleShortestPath.cpp | 401 + src/graph/executor/algo/SingleShortestPath.h | 61 + src/graph/executor/algo/SubgraphExecutor.cpp | 199 + src/graph/executor/algo/SubgraphExecutor.h | 78 + src/graph/executor/logic/ArgumentExecutor.cpp | 68 + src/graph/executor/logic/ArgumentExecutor.h | 26 + src/graph/executor/logic/LoopExecutor.cpp | 31 + src/graph/executor/logic/LoopExecutor.h | 43 + .../executor/logic/PassThroughExecutor.cpp | 27 + .../executor/logic/PassThroughExecutor.h | 24 + src/graph/executor/logic/SelectExecutor.cpp | 28 + src/graph/executor/logic/SelectExecutor.h | 51 + src/graph/executor/logic/StartExecutor.cpp | 17 + src/graph/executor/logic/StartExecutor.h | 23 + src/graph/executor/maintain/EdgeExecutor.cpp | 163 + src/graph/executor/maintain/EdgeExecutor.h | 69 + .../executor/maintain/EdgeIndexExecutor.cpp | 201 + .../executor/maintain/EdgeIndexExecutor.h | 64 + .../executor/maintain/FTIndexExecutor.cpp | 101 + src/graph/executor/maintain/FTIndexExecutor.h | 41 + src/graph/executor/maintain/TagExecutor.cpp | 158 + src/graph/executor/maintain/TagExecutor.h | 66 + .../executor/maintain/TagIndexExecutor.cpp | 201 + .../executor/maintain/TagIndexExecutor.h | 64 + src/graph/executor/mutate/DeleteExecutor.cpp | 222 + src/graph/executor/mutate/DeleteExecutor.h | 50 + src/graph/executor/mutate/InsertExecutor.cpp | 77 + src/graph/executor/mutate/InsertExecutor.h | 37 + src/graph/executor/mutate/UpdateExecutor.cpp | 140 + src/graph/executor/mutate/UpdateExecutor.h | 46 + .../executor/query/AggregateExecutor.cpp | 102 + src/graph/executor/query/AggregateExecutor.h | 25 + .../executor/query/AppendVerticesExecutor.cpp | 317 + .../executor/query/AppendVerticesExecutor.h | 51 + src/graph/executor/query/AssignExecutor.cpp | 32 + src/graph/executor/query/AssignExecutor.h | 23 + .../executor/query/DataCollectExecutor.cpp | 219 + .../executor/query/DataCollectExecutor.h | 53 + src/graph/executor/query/DedupExecutor.cpp | 42 + src/graph/executor/query/DedupExecutor.h | 24 + .../executor/query/ExpandAllExecutor.cpp | 523 ++ src/graph/executor/query/ExpandAllExecutor.h | 116 + src/graph/executor/query/ExpandExecutor.cpp | 388 + src/graph/executor/query/ExpandExecutor.h | 84 + src/graph/executor/query/FilterExecutor.cpp | 155 + src/graph/executor/query/FilterExecutor.h | 30 + .../query/FulltextIndexScanExecutor.cpp | 107 + .../query/FulltextIndexScanExecutor.h | 32 + src/graph/executor/query/GetEdgesExecutor.cpp | 112 + src/graph/executor/query/GetEdgesExecutor.h | 30 + .../executor/query/GetNeighborsExecutor.cpp | 102 + .../executor/query/GetNeighborsExecutor.h | 37 + src/graph/executor/query/GetPropExecutor.h | 52 + .../executor/query/GetVerticesExecutor.cpp | 72 + .../executor/query/GetVerticesExecutor.h | 30 + .../executor/query/IndexScanExecutor.cpp | 126 + src/graph/executor/query/IndexScanExecutor.h | 37 + .../executor/query/InnerJoinExecutor.cpp | 298 + src/graph/executor/query/InnerJoinExecutor.h | 71 + .../executor/query/IntersectExecutor.cpp | 50 + src/graph/executor/query/IntersectExecutor.h | 24 + src/graph/executor/query/JoinExecutor.cpp | 125 + src/graph/executor/query/JoinExecutor.h | 48 + src/graph/executor/query/LeftJoinExecutor.cpp | 260 + src/graph/executor/query/LeftJoinExecutor.h | 67 + src/graph/executor/query/LimitExecutor.cpp | 50 + src/graph/executor/query/LimitExecutor.h | 22 + src/graph/executor/query/MinusExecutor.cpp | 47 + src/graph/executor/query/MinusExecutor.h | 24 + .../executor/query/PatternApplyExecutor.cpp | 154 + .../executor/query/PatternApplyExecutor.h | 52 + src/graph/executor/query/ProjectExecutor.cpp | 78 + src/graph/executor/query/ProjectExecutor.h | 26 + .../executor/query/RollUpApplyExecutor.cpp | 180 + .../executor/query/RollUpApplyExecutor.h | 59 + src/graph/executor/query/SampleExecutor.cpp | 33 + src/graph/executor/query/SampleExecutor.h | 23 + .../executor/query/ScanEdgesExecutor.cpp | 47 + src/graph/executor/query/ScanEdgesExecutor.h | 21 + .../executor/query/ScanVerticesExecutor.cpp | 48 + .../executor/query/ScanVerticesExecutor.h | 26 + src/graph/executor/query/SetExecutor.cpp | 75 + src/graph/executor/query/SetExecutor.h | 29 + src/graph/executor/query/SortExecutor.cpp | 51 + src/graph/executor/query/SortExecutor.h | 22 + src/graph/executor/query/TopNExecutor.cpp | 91 + src/graph/executor/query/TopNExecutor.h | 31 + src/graph/executor/query/TraverseExecutor.cpp | 676 ++ src/graph/executor/query/TraverseExecutor.h | 111 + .../query/UnionAllVersionVarExecutor.cpp | 45 + .../query/UnionAllVersionVarExecutor.h | 24 + src/graph/executor/query/UnionExecutor.cpp | 38 + src/graph/executor/query/UnionExecutor.h | 25 + src/graph/executor/query/UnwindExecutor.cpp | 54 + src/graph/executor/query/UnwindExecutor.h | 27 + src/graph/executor/query/ValueExecutor.cpp | 21 + src/graph/executor/query/ValueExecutor.h | 22 + src/graph/executor/test/AggregateTest.cpp | 2181 ++++++ src/graph/executor/test/AssignTest.cpp | 183 + src/graph/executor/test/CMakeLists.txt | 113 + .../executor/test/CartesianProductTest.cpp | 166 + src/graph/executor/test/DataCollectTest.cpp | 268 + src/graph/executor/test/DedupTest.cpp | 79 + src/graph/executor/test/FilterTest.cpp | 90 + src/graph/executor/test/FindPathTest.cpp | 665 ++ src/graph/executor/test/GetNeighborsTest.cpp | 82 + src/graph/executor/test/JobTest.cpp | 69 + src/graph/executor/test/JoinTest.cpp | 401 + src/graph/executor/test/LimitTest.cpp | 76 + .../executor/test/LogicExecutorsTest.cpp | 103 + src/graph/executor/test/ProjectTest.cpp | 132 + src/graph/executor/test/QueryTestBase.h | 161 + src/graph/executor/test/SampleTest.cpp | 92 + src/graph/executor/test/SetExecutorTest.cpp | 464 ++ src/graph/executor/test/ShowQueriesTest.cpp | 94 + src/graph/executor/test/SortTest.cpp | 134 + src/graph/executor/test/StorageServerStub.cpp | 139 + src/graph/executor/test/TestMain.cpp | 23 + src/graph/executor/test/TopNTest.cpp | 116 + src/graph/executor/test/UnwindTest.cpp | 69 + src/graph/gc/CMakeLists.txt | 8 + src/graph/gc/GC.cpp | 40 + src/graph/gc/GC.h | 36 + src/graph/optimizer/CMakeLists.txt | 72 + src/graph/optimizer/OptContext.cpp | 33 + src/graph/optimizer/OptContext.h | 70 + src/graph/optimizer/OptGroup.cpp | 386 + src/graph/optimizer/OptGroup.h | 154 + src/graph/optimizer/OptRule.cpp | 255 + src/graph/optimizer/OptRule.h | 154 + src/graph/optimizer/Optimizer.cpp | 272 + src/graph/optimizer/Optimizer.h | 67 + .../optimizer/rule/CollapseProjectRule.cpp | 136 + .../optimizer/rule/CollapseProjectRule.h | 64 + .../optimizer/rule/CombineFilterRule.cpp | 72 + src/graph/optimizer/rule/CombineFilterRule.h | 57 + .../optimizer/rule/EdgeIndexFullScanRule.cpp | 42 + .../optimizer/rule/EdgeIndexFullScanRule.h | 30 + .../rule/EliminateAppendVerticesRule.cpp | 84 + .../rule/EliminateAppendVerticesRule.h | 30 + .../optimizer/rule/EliminateFilterRule.cpp | 76 + .../optimizer/rule/EliminateFilterRule.h | 56 + .../rule/EliminateRowCollectRule.cpp | 73 + .../optimizer/rule/EliminateRowCollectRule.h | 54 + .../rule/EmbedEdgeAllPredIntoTraverseRule.cpp | 224 + .../rule/EmbedEdgeAllPredIntoTraverseRule.h | 39 + .../rule/GeoPredicateEdgeIndexScanRule.cpp | 31 + .../rule/GeoPredicateEdgeIndexScanRule.h | 28 + .../rule/GeoPredicateIndexScanBaseRule.cpp | 172 + .../rule/GeoPredicateIndexScanBaseRule.h | 22 + .../rule/GeoPredicateTagIndexScanRule.cpp | 31 + .../rule/GeoPredicateTagIndexScanRule.h | 28 + ...tEdgesTransformAppendVerticesLimitRule.cpp | 157 + ...GetEdgesTransformAppendVerticesLimitRule.h | 92 + .../optimizer/rule/GetEdgesTransformRule.cpp | 135 + .../optimizer/rule/GetEdgesTransformRule.h | 82 + .../optimizer/rule/GetEdgesTransformUtils.h | 74 + .../optimizer/rule/IndexFullScanBaseRule.cpp | 85 + .../optimizer/rule/IndexFullScanBaseRule.h | 32 + src/graph/optimizer/rule/IndexScanRule.cpp | 101 + src/graph/optimizer/rule/IndexScanRule.h | 41 + .../rule/MergeGetNbrsAndDedupRule.cpp | 63 + .../optimizer/rule/MergeGetNbrsAndDedupRule.h | 55 + .../rule/MergeGetNbrsAndProjectRule.cpp | 85 + .../rule/MergeGetNbrsAndProjectRule.h | 59 + .../rule/MergeGetVerticesAndDedupRule.cpp | 62 + .../rule/MergeGetVerticesAndDedupRule.h | 55 + .../rule/MergeGetVerticesAndProjectRule.cpp | 84 + .../rule/MergeGetVerticesAndProjectRule.h | 59 + .../MergeLimitAndFulltextIndexScanRule.cpp | 92 + .../rule/MergeLimitAndFulltextIndexScanRule.h | 68 + .../OptimizeEdgeIndexScanByFilterRule.cpp | 156 + .../rule/OptimizeEdgeIndexScanByFilterRule.h | 35 + .../rule/OptimizeTagIndexScanByFilterRule.cpp | 190 + .../rule/OptimizeTagIndexScanByFilterRule.h | 35 + .../optimizer/rule/PushEFilterDownRule.cpp | 183 + .../optimizer/rule/PushEFilterDownRule.h | 41 + .../rule/PushFilterDownAggregateRule.cpp | 122 + .../rule/PushFilterDownAggregateRule.h | 60 + .../rule/PushFilterDownAllPathsRule.cpp | 104 + .../rule/PushFilterDownAllPathsRule.h | 58 + .../rule/PushFilterDownCrossJoinRule.cpp | 139 + .../rule/PushFilterDownCrossJoinRule.h | 37 + .../rule/PushFilterDownExpandAllRule.cpp | 144 + .../rule/PushFilterDownExpandAllRule.h | 63 + .../rule/PushFilterDownGetNbrsRule.cpp | 110 + .../rule/PushFilterDownGetNbrsRule.h | 63 + .../rule/PushFilterDownHashInnerJoinRule.cpp | 136 + .../rule/PushFilterDownHashInnerJoinRule.h | 37 + .../rule/PushFilterDownHashLeftJoinRule.cpp | 132 + .../rule/PushFilterDownHashLeftJoinRule.h | 37 + .../rule/PushFilterDownInnerJoinRule.cpp | 116 + .../rule/PushFilterDownInnerJoinRule.h | 66 + .../optimizer/rule/PushFilterDownNodeRule.cpp | 125 + .../optimizer/rule/PushFilterDownNodeRule.h | 35 + .../rule/PushFilterDownProjectRule.cpp | 170 + .../rule/PushFilterDownProjectRule.h | 70 + .../rule/PushFilterDownScanVerticesRule.cpp | 102 + .../rule/PushFilterDownScanVerticesRule.h | 62 + .../rule/PushFilterDownTraverseRule.cpp | 143 + .../rule/PushFilterDownTraverseRule.h | 43 + .../PushFilterThroughAppendVerticesRule.cpp | 123 + .../PushFilterThroughAppendVerticesRule.h | 46 + .../rule/PushLimitDownAllPathsRule.cpp | 74 + .../rule/PushLimitDownAllPathsRule.h | 56 + .../rule/PushLimitDownExpandAllRule.cpp | 86 + .../rule/PushLimitDownExpandAllRule.h | 62 + .../PushLimitDownFulltextIndexScanRule.cpp | 80 + .../rule/PushLimitDownFulltextIndexScanRule.h | 62 + .../PushLimitDownFulltextIndexScanRule2.cpp | 130 + .../PushLimitDownFulltextIndexScanRule2.h | 59 + .../rule/PushLimitDownGetEdgesRule.cpp | 79 + .../rule/PushLimitDownGetEdgesRule.h | 62 + .../rule/PushLimitDownGetNeighborsRule.cpp | 78 + .../rule/PushLimitDownGetNeighborsRule.h | 61 + .../rule/PushLimitDownGetVerticesRule.cpp | 79 + .../rule/PushLimitDownGetVerticesRule.h | 62 + .../rule/PushLimitDownIndexScanRule.cpp | 86 + .../rule/PushLimitDownIndexScanRule.h | 65 + .../rule/PushLimitDownProjectRule.cpp | 74 + .../optimizer/rule/PushLimitDownProjectRule.h | 61 + .../PushLimitDownScanAppendVerticesRule.cpp | 125 + .../PushLimitDownScanAppendVerticesRule.h | 72 + ...shLimitDownScanEdgesAppendVerticesRule.cpp | 117 + ...PushLimitDownScanEdgesAppendVerticesRule.h | 71 + .../rule/PushLimitDownScanEdgesRule.cpp | 77 + .../rule/PushLimitDownScanEdgesRule.h | 58 + .../PushStepLimitDownGetNeighborsRule.cpp | 79 + .../rule/PushStepLimitDownGetNeighborsRule.h | 65 + .../PushStepSampleDownGetNeighborsRule.cpp | 78 + .../rule/PushStepSampleDownGetNeighborsRule.h | 62 + .../rule/PushTopNDownIndexScanRule.cpp | 127 + .../rule/PushTopNDownIndexScanRule.h | 72 + .../rule/PushVFilterDownScanVerticesRule.cpp | 142 + .../rule/PushVFilterDownScanVerticesRule.h | 62 + .../RemoveAppendVerticesBelowJoinRule.cpp | 207 + .../rule/RemoveAppendVerticesBelowJoinRule.h | 46 + .../optimizer/rule/RemoveNoopProjectRule.cpp | 146 + .../optimizer/rule/RemoveNoopProjectRule.h | 54 + .../optimizer/rule/TagIndexFullScanRule.cpp | 38 + .../optimizer/rule/TagIndexFullScanRule.h | 30 + src/graph/optimizer/rule/TopNRule.cpp | 68 + src/graph/optimizer/rule/TopNRule.h | 56 + .../rule/UnionAllEdgeIndexScanRule.cpp | 31 + .../rule/UnionAllEdgeIndexScanRule.h | 29 + .../rule/UnionAllIndexScanBaseRule.cpp | 203 + .../rule/UnionAllIndexScanBaseRule.h | 23 + .../rule/UnionAllTagIndexScanRule.cpp | 31 + .../optimizer/rule/UnionAllTagIndexScanRule.h | 29 + src/graph/optimizer/test/CMakeLists.txt | 79 + .../optimizer/test/IndexScanRuleTest.cpp | 472 ++ src/graph/planner/CMakeLists.txt | 50 + src/graph/planner/Planner.cpp | 37 + src/graph/planner/Planner.h | 62 + src/graph/planner/PlannersRegister.cpp | 122 + src/graph/planner/PlannersRegister.h | 28 + src/graph/planner/SequentialPlanner.cpp | 60 + src/graph/planner/SequentialPlanner.h | 36 + src/graph/planner/match/ArgumentFinder.cpp | 44 + src/graph/planner/match/ArgumentFinder.h | 39 + src/graph/planner/match/CypherClausePlanner.h | 23 + src/graph/planner/match/EdgeIndexSeek.h | 18 + src/graph/planner/match/LabelIndexSeek.cpp | 256 + src/graph/planner/match/LabelIndexSeek.h | 54 + .../planner/match/MatchClausePlanner.cpp | 81 + src/graph/planner/match/MatchClausePlanner.h | 29 + src/graph/planner/match/MatchPathPlanner.cpp | 347 + src/graph/planner/match/MatchPathPlanner.h | 52 + src/graph/planner/match/MatchPlanner.cpp | 177 + src/graph/planner/match/MatchPlanner.h | 33 + src/graph/planner/match/MatchSolver.cpp | 307 + src/graph/planner/match/MatchSolver.h | 62 + .../planner/match/OrderByClausePlanner.cpp | 31 + .../planner/match/OrderByClausePlanner.h | 24 + src/graph/planner/match/PaginationPlanner.cpp | 31 + src/graph/planner/match/PaginationPlanner.h | 24 + src/graph/planner/match/PropIndexSeek.cpp | 181 + src/graph/planner/match/PropIndexSeek.h | 38 + .../planner/match/ReturnClausePlanner.cpp | 59 + src/graph/planner/match/ReturnClausePlanner.h | 24 + src/graph/planner/match/ScanSeek.cpp | 104 + src/graph/planner/match/ScanSeek.h | 38 + src/graph/planner/match/SegmentsConnector.cpp | 152 + src/graph/planner/match/SegmentsConnector.h | 59 + .../planner/match/ShortestPathPlanner.cpp | 127 + src/graph/planner/match/ShortestPathPlanner.h | 26 + src/graph/planner/match/StartVidFinder.cpp | 39 + src/graph/planner/match/StartVidFinder.h | 87 + .../planner/match/UnwindClausePlanner.cpp | 54 + src/graph/planner/match/UnwindClausePlanner.h | 25 + .../planner/match/VariablePropIndexSeek.cpp | 146 + .../planner/match/VariablePropIndexSeek.h | 44 + .../planner/match/VariableVertexIdSeek.cpp | 127 + .../planner/match/VariableVertexIdSeek.h | 43 + src/graph/planner/match/VertexIdSeek.cpp | 107 + src/graph/planner/match/VertexIdSeek.h | 43 + .../planner/match/WhereClausePlanner.cpp | 55 + src/graph/planner/match/WhereClausePlanner.h | 28 + src/graph/planner/match/WithClausePlanner.cpp | 69 + src/graph/planner/match/WithClausePlanner.h | 25 + .../planner/match/YieldClausePlanner.cpp | 93 + src/graph/planner/match/YieldClausePlanner.h | 32 + src/graph/planner/ngql/FetchEdgesPlanner.cpp | 67 + src/graph/planner/ngql/FetchEdgesPlanner.h | 44 + .../planner/ngql/FetchVerticesPlanner.cpp | 68 + src/graph/planner/ngql/FetchVerticesPlanner.h | 42 + src/graph/planner/ngql/GoPlanner.cpp | 327 + src/graph/planner/ngql/GoPlanner.h | 64 + src/graph/planner/ngql/LookupPlanner.cpp | 137 + src/graph/planner/ngql/LookupPlanner.h | 32 + src/graph/planner/ngql/MaintainPlanner.cpp | 65 + src/graph/planner/ngql/MaintainPlanner.h | 63 + src/graph/planner/ngql/PathPlanner.cpp | 508 ++ src/graph/planner/ngql/PathPlanner.h | 84 + src/graph/planner/ngql/SubgraphPlanner.cpp | 145 + src/graph/planner/ngql/SubgraphPlanner.h | 48 + src/graph/planner/plan/Admin.cpp | 198 + src/graph/planner/plan/Admin.h | 1389 ++++ src/graph/planner/plan/Algo.cpp | 193 + src/graph/planner/plan/Algo.h | 495 ++ src/graph/planner/plan/ExecutionPlan.cpp | 97 + src/graph/planner/plan/ExecutionPlan.h | 81 + src/graph/planner/plan/Logic.cpp | 98 + src/graph/planner/plan/Logic.h | 186 + src/graph/planner/plan/Maintain.cpp | 89 + src/graph/planner/plan/Maintain.h | 735 ++ src/graph/planner/plan/Mutate.cpp | 91 + src/graph/planner/plan/Mutate.h | 478 ++ src/graph/planner/plan/PlanNode.cpp | 489 ++ src/graph/planner/plan/PlanNode.h | 472 ++ src/graph/planner/plan/PlanNodeVisitor.h | 36 + src/graph/planner/plan/Query.cpp | 1091 +++ src/graph/planner/plan/Query.h | 2081 ++++++ src/graph/planner/plan/Scan.h | 458 ++ src/graph/planner/test/CMakeLists.txt | 72 + src/graph/planner/test/ExecutionPlanTest.cpp | 129 + .../AsyncMsgNotifyBasedScheduler.cpp | 332 + .../scheduler/AsyncMsgNotifyBasedScheduler.h | 94 + src/graph/scheduler/CMakeLists.txt | 10 + src/graph/scheduler/Scheduler.cpp | 73 + src/graph/scheduler/Scheduler.h | 35 + src/graph/service/Authenticator.h | 25 + src/graph/service/CMakeLists.txt | 30 + src/graph/service/CloudAuthenticator.cpp | 59 + src/graph/service/CloudAuthenticator.h | 29 + src/graph/service/GraphFlags.cpp | 120 + src/graph/service/GraphFlags.h | 77 + src/graph/service/GraphServer.cpp | 122 + src/graph/service/GraphServer.h | 46 + src/graph/service/GraphService.cpp | 275 + src/graph/service/GraphService.h | 65 + src/graph/service/PasswordAuthenticator.cpp | 20 + src/graph/service/PasswordAuthenticator.h | 29 + src/graph/service/PermissionCheck.cpp | 250 + src/graph/service/PermissionCheck.h | 29 + src/graph/service/PermissionManager.cpp | 219 + src/graph/service/PermissionManager.h | 42 + src/graph/service/QueryEngine.cpp | 83 + src/graph/service/QueryEngine.h | 62 + src/graph/service/QueryInstance.cpp | 266 + src/graph/service/QueryInstance.h | 70 + src/graph/service/RequestContext.h | 115 + src/graph/service/test/CMakeLists.txt | 10 + .../service/test/StandAloneTestGraphFlags.cpp | 13 + src/graph/session/CMakeLists.txt | 10 + src/graph/session/ClientSession.cpp | 108 + src/graph/session/ClientSession.h | 189 + src/graph/session/GraphSessionManager.cpp | 365 + src/graph/session/GraphSessionManager.h | 120 + src/graph/stats/CMakeLists.txt | 9 + src/graph/stats/GraphStats.cpp | 84 + src/graph/stats/GraphStats.h | 53 + src/graph/util/AnonColGenerator.h | 37 + src/graph/util/AnonVarGenerator.h | 50 + src/graph/util/AstUtils.h | 38 + src/graph/util/CMakeLists.txt | 27 + src/graph/util/Constants.cpp | 20 + src/graph/util/Constants.h | 23 + src/graph/util/ExpressionUtils.cpp | 1743 +++++ src/graph/util/ExpressionUtils.h | 280 + src/graph/util/FTIndexUtils.cpp | 45 + src/graph/util/FTIndexUtils.h | 41 + src/graph/util/IdGenerator.cpp | 13 + src/graph/util/IdGenerator.h | 47 + src/graph/util/IndexUtil.cpp | 127 + src/graph/util/IndexUtil.h | 39 + src/graph/util/OptimizerUtils.cpp | 947 +++ src/graph/util/OptimizerUtils.h | 186 + src/graph/util/ParserUtil.cpp | 187 + src/graph/util/ParserUtil.h | 45 + src/graph/util/PlannerUtil.cpp | 80 + src/graph/util/PlannerUtil.h | 36 + src/graph/util/SchemaUtil.cpp | 427 ++ src/graph/util/SchemaUtil.h | 96 + src/graph/util/ToJson.cpp | 323 + src/graph/util/ToJson.h | 107 + src/graph/util/Utils.cpp | 35 + src/graph/util/Utils.h | 42 + src/graph/util/ValidateUtil.cpp | 85 + src/graph/util/ValidateUtil.h | 35 + src/graph/util/ZoneUtil.cpp | 9 + src/graph/util/ZoneUtil.h | 22 + src/graph/util/test/CMakeLists.txt | 84 + src/graph/util/test/ExpressionUtilsTest.cpp | 831 +++ src/graph/util/test/IdGeneratorTest.cpp | 45 + src/graph/validator/ACLValidator.cpp | 201 + src/graph/validator/ACLValidator.h | 136 + src/graph/validator/AdminJobValidator.cpp | 72 + src/graph/validator/AdminJobValidator.h | 66 + src/graph/validator/AdminValidator.cpp | 716 ++ src/graph/validator/AdminValidator.h | 434 ++ src/graph/validator/AssignmentValidator.cpp | 41 + src/graph/validator/AssignmentValidator.h | 35 + src/graph/validator/CMakeLists.txt | 33 + src/graph/validator/ExplainValidator.cpp | 79 + src/graph/validator/ExplainValidator.h | 33 + src/graph/validator/FetchEdgesValidator.cpp | 197 + src/graph/validator/FetchEdgesValidator.h | 48 + .../validator/FetchVerticesValidator.cpp | 109 + src/graph/validator/FetchVerticesValidator.h | 42 + src/graph/validator/FindPathValidator.cpp | 97 + src/graph/validator/FindPathValidator.h | 35 + src/graph/validator/GetSubgraphValidator.cpp | 226 + src/graph/validator/GetSubgraphValidator.h | 42 + src/graph/validator/GoValidator.cpp | 323 + src/graph/validator/GoValidator.h | 58 + src/graph/validator/GroupByValidator.cpp | 208 + src/graph/validator/GroupByValidator.h | 45 + src/graph/validator/LimitValidator.cpp | 42 + src/graph/validator/LimitValidator.h | 30 + src/graph/validator/LookupValidator.cpp | 626 ++ src/graph/validator/LookupValidator.h | 67 + src/graph/validator/MaintainValidator.cpp | 677 ++ src/graph/validator/MaintainValidator.h | 447 ++ src/graph/validator/MatchValidator.cpp | 1353 ++++ src/graph/validator/MatchValidator.h | 118 + src/graph/validator/MutateValidator.cpp | 930 +++ src/graph/validator/MutateValidator.h | 206 + src/graph/validator/OrderByValidator.cpp | 79 + src/graph/validator/OrderByValidator.h | 30 + src/graph/validator/PipeValidator.cpp | 48 + src/graph/validator/PipeValidator.h | 52 + src/graph/validator/ReportError.h | 36 + src/graph/validator/SequentialValidator.cpp | 59 + src/graph/validator/SequentialValidator.h | 50 + src/graph/validator/SetValidator.cpp | 105 + src/graph/validator/SetValidator.h | 53 + src/graph/validator/UnwindValidator.cpp | 50 + src/graph/validator/UnwindValidator.h | 36 + src/graph/validator/UseValidator.cpp | 54 + src/graph/validator/UseValidator.h | 36 + src/graph/validator/Validator.cpp | 511 ++ src/graph/validator/Validator.h | 211 + src/graph/validator/YieldValidator.cpp | 363 + src/graph/validator/YieldValidator.h | 62 + src/graph/validator/test/ACLValidatorTest.cpp | 180 + .../validator/test/AdminValidatorTest.cpp | 60 + src/graph/validator/test/CMakeLists.txt | 100 + .../validator/test/ExplainValidatorTest.cpp | 149 + src/graph/validator/test/FetchEdgesTest.cpp | 337 + .../validator/test/FetchVerticesTest.cpp | 696 ++ .../validator/test/FindPathValidatorTest.cpp | 362 + .../test/GetSubgraphValidatorTest.cpp | 232 + .../validator/test/GroupByValidatorTest.cpp | 399 + .../validator/test/LookupValidatorTest.cpp | 174 + .../validator/test/MaintainValidatorTest.cpp | 37 + .../validator/test/MatchValidatorTest.cpp | 553 ++ src/graph/validator/test/MockIndexManager.cpp | 63 + src/graph/validator/test/MockIndexManager.h | 91 + .../validator/test/MockSchemaManager.cpp | 176 + src/graph/validator/test/MockSchemaManager.h | 138 + .../validator/test/MockSchemaManagerTest.cpp | 53 + .../validator/test/MutateValidatorTest.cpp | 237 + .../validator/test/QueryValidatorTest.cpp | 1125 +++ src/graph/validator/test/SymbolsTest.cpp | 141 + .../validator/test/ValidatorTestBase.cpp | 247 + src/graph/validator/test/ValidatorTestBase.h | 137 + .../validator/test/YieldValidatorTest.cpp | 654 ++ src/graph/visitor/CMakeLists.txt | 27 + src/graph/visitor/DeduceAliasTypeVisitor.cpp | 102 + src/graph/visitor/DeduceAliasTypeVisitor.h | 100 + src/graph/visitor/DeducePropsVisitor.cpp | 294 + src/graph/visitor/DeducePropsVisitor.h | 146 + src/graph/visitor/DeduceTypeVisitor.cpp | 825 ++ src/graph/visitor/DeduceTypeVisitor.h | 116 + src/graph/visitor/EvaluableExprVisitor.cpp | 105 + src/graph/visitor/EvaluableExprVisitor.h | 74 + .../visitor/ExtractFilterExprVisitor.cpp | 506 ++ src/graph/visitor/ExtractFilterExprVisitor.h | 121 + .../visitor/ExtractGroupSuiteVisitor.cpp | 200 + src/graph/visitor/ExtractGroupSuiteVisitor.h | 95 + src/graph/visitor/ExtractPropExprVisitor.cpp | 226 + src/graph/visitor/ExtractPropExprVisitor.h | 84 + src/graph/visitor/FindVisitor.cpp | 271 + src/graph/visitor/FindVisitor.h | 99 + src/graph/visitor/FoldConstantExprVisitor.cpp | 522 ++ src/graph/visitor/FoldConstantExprVisitor.h | 101 + src/graph/visitor/PropertyTrackerVisitor.cpp | 426 ++ src/graph/visitor/PropertyTrackerVisitor.h | 91 + src/graph/visitor/PrunePropertiesVisitor.cpp | 605 ++ src/graph/visitor/PrunePropertiesVisitor.h | 94 + src/graph/visitor/RewriteVisitor.cpp | 581 ++ src/graph/visitor/RewriteVisitor.h | 106 + .../ValidatePatternExpressionVisitor.cpp | 97 + .../ValidatePatternExpressionVisitor.h | 61 + src/graph/visitor/VidExtractVisitor.cpp | 516 ++ src/graph/visitor/VidExtractVisitor.h | 122 + src/graph/visitor/test/CMakeLists.txt | 85 + .../test/DeduceAliasTypeVisitorTest.cpp | 132 + .../visitor/test/DeduceTypeVisitorTest.cpp | 198 + .../test/ExtractFilterExprVisitorTest.cpp | 473 ++ .../test/ExtractGroupSuiteVisitorTest.cpp | 403 + .../visitor/test/FilterTransformTest.cpp | 116 + .../test/FoldConstantExprVisitorTest.cpp | 276 + .../test/RewriteRelExprVisitorTest.cpp | 256 + .../test/RewriteUnaryNotExprVisitorTest.cpp | 213 + src/graph/visitor/test/TestMain.cpp | 16 + src/graph/visitor/test/VisitorTestBase.h | 225 + src/interface/CMakeLists.txt | 37 + src/interface/common.thrift | 566 ++ src/interface/graph.thrift | 124 + src/interface/meta.thrift | 1315 ++++ src/interface/raftex.thrift | 146 + src/interface/storage.thrift | 938 +++ src/kvstore/CMakeLists.txt | 28 + src/kvstore/Common.h | 288 + src/kvstore/CompactionFilter.h | 115 + src/kvstore/DiskManager.cpp | 170 + src/kvstore/DiskManager.h | 134 + src/kvstore/EventListener.h | 390 + src/kvstore/KVEngine.h | 357 + src/kvstore/KVIterator.h | 50 + src/kvstore/KVStore.h | 472 ++ src/kvstore/LogEncoder.cpp | 249 + src/kvstore/LogEncoder.h | 221 + src/kvstore/NebulaSnapshotManager.cpp | 150 + src/kvstore/NebulaSnapshotManager.h | 65 + src/kvstore/NebulaStore.cpp | 1453 ++++ src/kvstore/NebulaStore.h | 893 +++ src/kvstore/Part.cpp | 587 ++ src/kvstore/Part.h | 357 + src/kvstore/PartManager.cpp | 254 + src/kvstore/PartManager.h | 559 ++ src/kvstore/RateLimiter.cpp | 10 + src/kvstore/RateLimiter.h | 58 + src/kvstore/RocksEngine.cpp | 701 ++ src/kvstore/RocksEngine.h | 612 ++ src/kvstore/RocksEngineConfig.cpp | 402 + src/kvstore/RocksEngineConfig.h | 101 + src/kvstore/listener/CMakeLists.txt | 11 + src/kvstore/listener/Listener.cpp | 185 + src/kvstore/listener/Listener.h | 283 + .../listener/elasticsearch/ESListener.cpp | 400 + .../listener/elasticsearch/ESListener.h | 133 + src/kvstore/listener/test/CMakeLists.txt | 66 + .../listener/test/NebulaListenerTest.cpp | 853 +++ src/kvstore/plugins/hbase/HBaseStore.cpp | 414 ++ src/kvstore/plugins/hbase/HBaseStore.h | 253 + src/kvstore/raftex/CMakeLists.txt | 15 + src/kvstore/raftex/Host.cpp | 535 ++ src/kvstore/raftex/Host.h | 295 + src/kvstore/raftex/RaftLogIterator.cpp | 47 + src/kvstore/raftex/RaftLogIterator.h | 69 + src/kvstore/raftex/RaftPart.cpp | 2271 ++++++ src/kvstore/raftex/RaftPart.h | 891 +++ src/kvstore/raftex/RaftexService.cpp | 178 + src/kvstore/raftex/RaftexService.h | 146 + src/kvstore/raftex/SnapshotManager.cpp | 140 + src/kvstore/raftex/SnapshotManager.h | 102 + src/kvstore/raftex/test/CMakeLists.txt | 153 + .../raftex/test/LeaderElectionTest.cpp | 133 + .../raftex/test/LeaderTransferTest.cpp | 102 + src/kvstore/raftex/test/LearnerTest.cpp | 143 + src/kvstore/raftex/test/LogAppendTest.cpp | 143 + src/kvstore/raftex/test/LogCASTest.cpp | 312 + src/kvstore/raftex/test/LogCommandTest.cpp | 195 + src/kvstore/raftex/test/MemberChangeTest.cpp | 139 + src/kvstore/raftex/test/RaftCase.cpp | 253 + src/kvstore/raftex/test/RaftexTestBase.cpp | 356 + src/kvstore/raftex/test/RaftexTestBase.h | 141 + src/kvstore/raftex/test/TestShard.cpp | 267 + src/kvstore/raftex/test/TestShard.h | 203 + src/kvstore/stats/CMakeLists.txt | 9 + src/kvstore/stats/KVStats.cpp | 41 + src/kvstore/stats/KVStats.h | 28 + src/kvstore/test/CMakeLists.txt | 230 + src/kvstore/test/CommonTest.cpp | 69 + src/kvstore/test/DiskManagerTest.cpp | 190 + src/kvstore/test/LogEncoderTest.cpp | 159 + src/kvstore/test/MultiVersionBenchmark.cpp | 111 + src/kvstore/test/NebulaStoreTest.cpp | 1259 ++++ src/kvstore/test/PartPerformanceTest.cpp | 244 + src/kvstore/test/PartTest.cpp | 261 + src/kvstore/test/RateLimiterTest.cpp | 64 + src/kvstore/test/RocksEngineConfigTest.cpp | 220 + src/kvstore/test/RocksEngineTest.cpp | 920 +++ src/kvstore/test/SnapshotLinkTest.cpp | 100 + src/kvstore/wal/AtomicLogBuffer.cpp | 217 + src/kvstore/wal/AtomicLogBuffer.h | 372 + src/kvstore/wal/CMakeLists.txt | 12 + src/kvstore/wal/FileBasedWal.cpp | 733 ++ src/kvstore/wal/FileBasedWal.h | 293 + src/kvstore/wal/Wal.h | 107 + src/kvstore/wal/WalFileInfo.h | 135 + src/kvstore/wal/WalFileIterator.cpp | 228 + src/kvstore/wal/WalFileIterator.h | 99 + src/kvstore/wal/test/AtomicLogBufferTest.cpp | 158 + src/kvstore/wal/test/CMakeLists.txt | 81 + src/kvstore/wal/test/FileBasedWalTest.cpp | 614 ++ src/kvstore/wal/test/InMemoryLogBuffer.cpp | 83 + src/kvstore/wal/test/InMemoryLogBuffer.h | 70 + src/kvstore/wal/test/InMemoryLogBufferList.h | 178 + .../wal/test/InMemoryLogBufferTest.cpp | 105 + src/kvstore/wal/test/LogBufferBenchmark.cpp | 465 ++ src/kvstore/wal/test/WalFileIterTest.cpp | 112 + src/meta/ActiveHostsMan.cpp | 275 + src/meta/ActiveHostsMan.h | 228 + src/meta/CMakeLists.txt | 190 + src/meta/KVBasedClusterIdMan.h | 163 + src/meta/MetaServiceHandler.cpp | 565 ++ src/meta/MetaServiceHandler.h | 248 + src/meta/MetaServiceUtils.cpp | 356 + src/meta/MetaServiceUtils.h | 106 + src/meta/MetaVersionMan.cpp | 228 + src/meta/MetaVersionMan.h | 56 + src/meta/RootUserMan.h | 83 + src/meta/http/CMakeLists.txt | 10 + src/meta/http/MetaHttpReplaceHostHandler.cpp | 287 + src/meta/http/MetaHttpReplaceHostHandler.h | 72 + src/meta/http/test/CMakeLists.txt | 21 + .../http/test/MetaHttpReplaceHandlerTest.cpp | 267 + src/meta/processors/BaseProcessor-inl.h | 599 ++ src/meta/processors/BaseProcessor.h | 482 ++ src/meta/processors/Common.h | 41 + src/meta/processors/admin/AdminClient.cpp | 817 ++ src/meta/processors/admin/AdminClient.h | 330 + .../processors/admin/AgentHBProcessor.cpp | 141 + src/meta/processors/admin/AgentHBProcessor.h | 62 + .../processors/admin/ClearSpaceProcessor.cpp | 82 + .../processors/admin/ClearSpaceProcessor.h | 37 + .../admin/CreateBackupProcessor.cpp | 265 + .../processors/admin/CreateBackupProcessor.h | 47 + .../admin/CreateSnapshotProcessor.cpp | 148 + .../admin/CreateSnapshotProcessor.h | 43 + .../admin/DropSnapshotProcessor.cpp | 79 + .../processors/admin/DropSnapshotProcessor.h | 43 + .../admin/GetMetaDirInfoProcessor.cpp | 28 + .../admin/GetMetaDirInfoProcessor.h | 34 + src/meta/processors/admin/HBProcessor.cpp | 135 + src/meta/processors/admin/HBProcessor.h | 71 + .../admin/ListClusterInfoProcessor.cpp | 158 + .../admin/ListClusterInfoProcessor.h | 36 + .../admin/ListSnapshotsProcessor.cpp | 46 + .../processors/admin/ListSnapshotsProcessor.h | 34 + .../processors/admin/RestoreProcessor.cpp | 231 + src/meta/processors/admin/RestoreProcessor.h | 56 + .../admin/SaveGraphVersionProcessor.cpp | 36 + .../admin/SaveGraphVersionProcessor.h | 27 + src/meta/processors/admin/SnapShot.cpp | 166 + src/meta/processors/admin/SnapShot.h | 77 + .../admin/VerifyClientVersionProcessor.cpp | 31 + .../admin/VerifyClientVersionProcessor.h | 27 + .../processors/config/GetConfigProcessor.cpp | 66 + .../processors/config/GetConfigProcessor.h | 38 + .../config/ListConfigsProcessor.cpp | 42 + .../processors/config/ListConfigsProcessor.h | 34 + .../processors/config/RegConfigProcessor.cpp | 54 + .../processors/config/RegConfigProcessor.h | 34 + .../processors/config/SetConfigProcessor.cpp | 82 + .../processors/config/SetConfigProcessor.h | 49 + .../processors/id/GetSegmentIdProcessor.cpp | 37 + .../processors/id/GetSegmentIdProcessor.h | 40 + .../processors/id/GetWorkerIdProcessor.cpp | 48 + src/meta/processors/id/GetWorkerIdProcessor.h | 46 + .../index/CreateEdgeIndexProcessor.cpp | 230 + .../index/CreateEdgeIndexProcessor.h | 43 + .../index/CreateTagIndexProcessor.cpp | 229 + .../index/CreateTagIndexProcessor.h | 38 + .../index/DropEdgeIndexProcessor.cpp | 75 + .../processors/index/DropEdgeIndexProcessor.h | 36 + .../index/DropTagIndexProcessor.cpp | 76 + .../processors/index/DropTagIndexProcessor.h | 37 + .../processors/index/FTIndexProcessor.cpp | 238 + src/meta/processors/index/FTIndexProcessor.h | 69 + .../index/GetEdgeIndexProcessor.cpp | 56 + .../processors/index/GetEdgeIndexProcessor.h | 36 + .../processors/index/GetTagIndexProcessor.cpp | 57 + .../processors/index/GetTagIndexProcessor.h | 36 + .../index/ListEdgeIndexesProcessor.cpp | 63 + .../index/ListEdgeIndexesProcessor.h | 35 + .../index/ListTagIndexesProcessor.cpp | 63 + .../index/ListTagIndexesProcessor.h | 35 + src/meta/processors/job/AdminJobProcessor.cpp | 157 + src/meta/processors/job/AdminJobProcessor.h | 52 + .../processors/job/BalanceJobExecutor.cpp | 191 + src/meta/processors/job/BalanceJobExecutor.h | 176 + src/meta/processors/job/BalancePlan.cpp | 277 + src/meta/processors/job/BalancePlan.h | 162 + src/meta/processors/job/BalanceTask.cpp | 263 + src/meta/processors/job/BalanceTask.h | 145 + .../processors/job/CompactJobExecutor.cpp | 43 + src/meta/processors/job/CompactJobExecutor.h | 39 + .../processors/job/DataBalanceJobExecutor.cpp | 222 + .../processors/job/DataBalanceJobExecutor.h | 69 + .../processors/job/DownloadJobExecutor.cpp | 115 + src/meta/processors/job/DownloadJobExecutor.h | 48 + src/meta/processors/job/FlushJobExecutor.cpp | 38 + src/meta/processors/job/FlushJobExecutor.h | 39 + src/meta/processors/job/GetStatsProcessor.cpp | 46 + src/meta/processors/job/GetStatsProcessor.h | 33 + src/meta/processors/job/IngestJobExecutor.cpp | 55 + src/meta/processors/job/IngestJobExecutor.h | 37 + src/meta/processors/job/JobDescription.cpp | 121 + src/meta/processors/job/JobDescription.h | 188 + src/meta/processors/job/JobExecutor.cpp | 94 + src/meta/processors/job/JobExecutor.h | 111 + src/meta/processors/job/JobManager.cpp | 1048 +++ src/meta/processors/job/JobManager.h | 344 + src/meta/processors/job/JobStatus.cpp | 56 + src/meta/processors/job/JobStatus.h | 33 + .../job/LeaderBalanceJobExecutor.cpp | 563 ++ .../processors/job/LeaderBalanceJobExecutor.h | 130 + .../job/ListEdgeIndexStatusProcessor.cpp | 69 + .../job/ListEdgeIndexStatusProcessor.h | 33 + .../job/ListTagIndexStatusProcessor.cpp | 70 + .../job/ListTagIndexStatusProcessor.h | 33 + src/meta/processors/job/MetaJobExecutor.cpp | 58 + src/meta/processors/job/MetaJobExecutor.h | 103 + .../processors/job/RebuildEdgeJobExecutor.cpp | 36 + .../processors/job/RebuildEdgeJobExecutor.h | 31 + .../processors/job/RebuildFTJobExecutor.cpp | 36 + .../processors/job/RebuildFTJobExecutor.h | 38 + .../processors/job/RebuildJobExecutor.cpp | 79 + src/meta/processors/job/RebuildJobExecutor.h | 38 + .../processors/job/RebuildTagJobExecutor.cpp | 36 + .../processors/job/RebuildTagJobExecutor.h | 31 + .../processors/job/ReportTaskProcessor.cpp | 27 + src/meta/processors/job/ReportTaskProcessor.h | 29 + .../job/SimpleConcurrentJobExecutor.cpp | 43 + .../job/SimpleConcurrentJobExecutor.h | 31 + src/meta/processors/job/StatsJobExecutor.cpp | 214 + src/meta/processors/job/StatsJobExecutor.h | 68 + .../processors/job/StorageJobExecutor.cpp | 249 + src/meta/processors/job/StorageJobExecutor.h | 127 + src/meta/processors/job/TaskDescription.cpp | 96 + src/meta/processors/job/TaskDescription.h | 127 + .../processors/job/ZoneBalanceJobExecutor.cpp | 349 + .../processors/job/ZoneBalanceJobExecutor.h | 84 + .../processors/listener/ListenerProcessor.cpp | 153 + .../processors/listener/ListenerProcessor.h | 71 + .../processors/parts/AlterSpaceProcessor.cpp | 80 + .../processors/parts/AlterSpaceProcessor.h | 38 + .../parts/CreateSpaceAsProcessor.cpp | 270 + .../processors/parts/CreateSpaceAsProcessor.h | 58 + .../processors/parts/CreateSpaceProcessor.cpp | 347 + .../processors/parts/CreateSpaceProcessor.h | 64 + .../processors/parts/DropSpaceProcessor.cpp | 157 + .../processors/parts/DropSpaceProcessor.h | 39 + .../parts/GetPartsAllocProcessor.cpp | 94 + .../processors/parts/GetPartsAllocProcessor.h | 42 + .../processors/parts/GetSpaceProcessor.cpp | 52 + src/meta/processors/parts/GetSpaceProcessor.h | 34 + .../processors/parts/ListHostsProcessor.cpp | 383 + .../processors/parts/ListHostsProcessor.h | 98 + .../processors/parts/ListPartsProcessor.cpp | 125 + .../processors/parts/ListPartsProcessor.h | 49 + .../processors/parts/ListSpacesProcessor.cpp | 42 + .../processors/parts/ListSpacesProcessor.h | 34 + .../processors/schema/AlterEdgeProcessor.cpp | 171 + .../processors/schema/AlterEdgeProcessor.h | 33 + .../processors/schema/AlterTagProcessor.cpp | 168 + .../processors/schema/AlterTagProcessor.h | 33 + .../processors/schema/CreateEdgeProcessor.cpp | 96 + .../processors/schema/CreateEdgeProcessor.h | 36 + .../processors/schema/CreateTagProcessor.cpp | 98 + .../processors/schema/CreateTagProcessor.h | 41 + .../processors/schema/DropEdgeProcessor.cpp | 113 + .../processors/schema/DropEdgeProcessor.h | 37 + .../processors/schema/DropTagProcessor.cpp | 113 + src/meta/processors/schema/DropTagProcessor.h | 37 + .../processors/schema/GetEdgeProcessor.cpp | 71 + src/meta/processors/schema/GetEdgeProcessor.h | 34 + .../processors/schema/GetTagProcessor.cpp | 71 + src/meta/processors/schema/GetTagProcessor.h | 32 + .../processors/schema/ListEdgesProcessor.cpp | 68 + .../processors/schema/ListEdgesProcessor.h | 40 + .../processors/schema/ListTagsProcessor.cpp | 68 + .../processors/schema/ListTagsProcessor.h | 40 + src/meta/processors/schema/SchemaUtil.cpp | 195 + src/meta/processors/schema/SchemaUtil.h | 49 + .../processors/service/ServiceProcessor.cpp | 95 + .../processors/service/ServiceProcessor.h | 70 + .../session/SessionManagerProcessor.cpp | 286 + .../session/SessionManagerProcessor.h | 119 + .../user/AuthenticationProcessor.cpp | 373 + .../processors/user/AuthenticationProcessor.h | 167 + .../zone/AddHostsIntoZoneProcessor.cpp | 107 + .../zone/AddHostsIntoZoneProcessor.h | 33 + .../processors/zone/AddHostsProcessor.cpp | 117 + src/meta/processors/zone/AddHostsProcessor.h | 32 + .../processors/zone/DivideZoneProcessor.cpp | 179 + .../processors/zone/DivideZoneProcessor.h | 46 + .../processors/zone/DropHostsProcessor.cpp | 189 + src/meta/processors/zone/DropHostsProcessor.h | 46 + .../processors/zone/DropZoneProcessor.cpp | 140 + src/meta/processors/zone/DropZoneProcessor.h | 49 + src/meta/processors/zone/GetZoneProcessor.cpp | 36 + src/meta/processors/zone/GetZoneProcessor.h | 32 + .../processors/zone/ListZonesProcessor.cpp | 45 + src/meta/processors/zone/ListZonesProcessor.h | 32 + .../processors/zone/MergeZoneProcessor.cpp | 245 + src/meta/processors/zone/MergeZoneProcessor.h | 42 + .../processors/zone/RenameZoneProcessor.cpp | 70 + .../processors/zone/RenameZoneProcessor.h | 33 + src/meta/stats/CMakeLists.txt | 9 + src/meta/stats/MetaStats.cpp | 14 + src/meta/stats/MetaStats.h | 16 + src/meta/test/ActiveHostsManTest.cpp | 210 + src/meta/test/AdminClientTest.cpp | 432 ++ src/meta/test/AgentHBProcessorTest.cpp | 97 + src/meta/test/AuthProcessorTest.cpp | 624 ++ src/meta/test/BalancerTest.cpp | 1491 ++++ src/meta/test/CMakeLists.txt | 320 + src/meta/test/ClusterIdManTest.cpp | 44 + src/meta/test/ConfigManTest.cpp | 661 ++ src/meta/test/CreateBackupProcessorTest.cpp | 305 + src/meta/test/GetStatsTest.cpp | 619 ++ src/meta/test/HBProcessorTest.cpp | 74 + src/meta/test/IdProcessorTest.cpp | 59 + src/meta/test/IndexProcessorTest.cpp | 2459 ++++++ src/meta/test/JobManagerTest.cpp | 2171 ++++++ src/meta/test/ListClusterInfoTest.cpp | 96 + src/meta/test/MetaClientTest.cpp | 2819 +++++++ src/meta/test/MockAdminClient.h | 79 + src/meta/test/MockHdfsHelper.h | 54 + src/meta/test/ProcessorTest.cpp | 5073 +++++++++++++ src/meta/test/RestoreProcessorTest.cpp | 437 ++ src/meta/test/SnapshotProcessorsTest.cpp | 562 ++ src/meta/test/TestUtils.h | 678 ++ src/meta/test/VerifyClientVersionTest.cpp | 76 + src/meta/upgrade/CMakeLists.txt | 16 + src/meta/upgrade/MetaDataUpgrade.cpp | 168 + src/meta/upgrade/MetaDataUpgrade.h | 69 + src/meta/upgrade/v2/CMakeLists.txt | 24 + src/meta/upgrade/v2/MetaServiceUtilsV2.cpp | 32 + src/meta/upgrade/v2/MetaServiceUtilsV2.h | 38 + src/meta/upgrade/v2/meta.thrift | 81 + src/mock/AdHocIndexManager.cpp | 190 + src/mock/AdHocIndexManager.h | 62 + src/mock/AdHocSchemaManager.cpp | 312 + src/mock/AdHocSchemaManager.h | 152 + src/mock/CMakeLists.txt | 17 + src/mock/FakeHttpServer.cpp | 100 + src/mock/FakeHttpServer.h | 75 + src/mock/LocalServer.h | 45 + src/mock/MockCluster.cpp | 303 + src/mock/MockCluster.h | 148 + src/mock/MockData.cpp | 1240 ++++ src/mock/MockData.h | 215 + src/mock/RpcServer.h | 52 + src/parser/.gitignore | 6 + src/parser/AdminSentences.cpp | 445 ++ src/parser/AdminSentences.h | 888 +++ src/parser/CMakeLists.txt | 51 + src/parser/Clauses.cpp | 267 + src/parser/Clauses.h | 543 ++ src/parser/EdgeKey.cpp | 48 + src/parser/EdgeKey.h | 118 + src/parser/ExplainSentence.cpp | 29 + src/parser/ExplainSentence.h | 46 + src/parser/GQLParser.cpp | 10 + src/parser/GQLParser.h | 87 + src/parser/GraphScanner.h | 195 + src/parser/MaintainSentences.cpp | 513 ++ src/parser/MaintainSentences.h | 1197 +++ src/parser/MatchPath.cpp | 108 + src/parser/MatchPath.h | 395 + src/parser/MatchSentence.cpp | 158 + src/parser/MatchSentence.h | 337 + src/parser/MutateSentences.cpp | 286 + src/parser/MutateSentences.h | 614 ++ src/parser/ProcessControlSentences.cpp | 24 + src/parser/ProcessControlSentences.h | 34 + src/parser/Sentence.h | 305 + src/parser/SequentialSentences.cpp | 22 + src/parser/SequentialSentences.h | 51 + src/parser/TraverseSentences.cpp | 302 + src/parser/TraverseSentences.h | 694 ++ src/parser/UserSentences.cpp | 111 + src/parser/UserSentences.h | 225 + src/parser/parser.yy | 4098 ++++++++++ src/parser/scanner.lex | 589 ++ src/parser/test/CMakeLists.txt | 91 + src/parser/test/ExpressionParsingTest.cpp | 453 ++ src/parser/test/ParserBenchmark.cpp | 185 + src/parser/test/ParserTest.cpp | 3247 ++++++++ src/parser/test/ScannerTest.cpp | 641 ++ src/parser/test/fuzzing/CMakeLists.txt | 11 + src/parser/test/fuzzing/ParserFuzzer.cpp | 16 + src/parser/test/fuzzing/README.md | 11 + src/parser/test/fuzzing/nebula.dict | 179 + src/storage/BaseProcessor-inl.h | 202 + src/storage/BaseProcessor.h | 198 + src/storage/CMakeLists.txt | 111 + src/storage/CommonUtils.cpp | 85 + src/storage/CommonUtils.h | 282 + src/storage/CompactionFilter.h | 238 + src/storage/GraphStorageLocalServer.cpp | 147 + src/storage/GraphStorageLocalServer.h | 66 + src/storage/GraphStorageServiceHandler.cpp | 222 + src/storage/GraphStorageServiceHandler.h | 83 + src/storage/InternalStorageServiceHandler.cpp | 41 + src/storage/InternalStorageServiceHandler.h | 40 + src/storage/MergeOperator.h | 48 + src/storage/StorageAdminServiceHandler.cpp | 110 + src/storage/StorageAdminServiceHandler.h | 59 + src/storage/StorageFlags.cpp | 50 + src/storage/StorageFlags.h | 39 + src/storage/StorageServer.cpp | 425 ++ src/storage/StorageServer.h | 133 + src/storage/admin/AdminProcessor.h | 443 ++ src/storage/admin/AdminTask.cpp | 55 + src/storage/admin/AdminTask.h | 248 + src/storage/admin/AdminTaskManager.cpp | 389 + src/storage/admin/AdminTaskManager.h | 119 + src/storage/admin/AdminTaskProcessor.cpp | 55 + src/storage/admin/AdminTaskProcessor.h | 53 + src/storage/admin/ClearSpaceProcessor.cpp | 32 + src/storage/admin/ClearSpaceProcessor.h | 53 + src/storage/admin/CompactTask.cpp | 43 + src/storage/admin/CompactTask.h | 38 + .../admin/CreateCheckpointProcessor.cpp | 45 + src/storage/admin/CreateCheckpointProcessor.h | 52 + src/storage/admin/DownloadTask.cpp | 66 + src/storage/admin/DownloadTask.h | 42 + src/storage/admin/DropCheckpointProcessor.cpp | 39 + src/storage/admin/DropCheckpointProcessor.h | 50 + src/storage/admin/FlushTask.cpp | 40 + src/storage/admin/FlushTask.h | 36 + src/storage/admin/GetLeaderProcessor.cpp | 33 + src/storage/admin/GetLeaderProcessor.h | 41 + src/storage/admin/IngestTask.cpp | 51 + src/storage/admin/IngestTask.h | 30 + src/storage/admin/RebuildEdgeIndexTask.cpp | 169 + src/storage/admin/RebuildEdgeIndexTask.h | 38 + src/storage/admin/RebuildFTIndexTask.cpp | 76 + src/storage/admin/RebuildFTIndexTask.h | 39 + src/storage/admin/RebuildIndexTask.cpp | 280 + src/storage/admin/RebuildIndexTask.h | 84 + src/storage/admin/RebuildTagIndexTask.cpp | 155 + src/storage/admin/RebuildTagIndexTask.h | 60 + src/storage/admin/SendBlockSignProcessor.cpp | 36 + src/storage/admin/SendBlockSignProcessor.h | 51 + src/storage/admin/StatsTask.cpp | 372 + src/storage/admin/StatsTask.h | 77 + src/storage/admin/StopAdminTaskProcessor.cpp | 34 + src/storage/admin/StopAdminTaskProcessor.h | 54 + .../context/StorageExpressionContext.cpp | 136 + .../context/StorageExpressionContext.h | 334 + src/storage/exec/AggregateNode.h | 165 + src/storage/exec/DeDupNode.h | 66 + src/storage/exec/EdgeNode.h | 250 + src/storage/exec/FilterNode.h | 129 + src/storage/exec/GetDstBySrcNode.h | 102 + src/storage/exec/GetNeighborsNode.h | 236 + src/storage/exec/GetPropNode.h | 254 + src/storage/exec/HashJoinNode.h | 168 + src/storage/exec/IndexAggregateNode.cpp | 125 + src/storage/exec/IndexAggregateNode.h | 51 + src/storage/exec/IndexDedupNode.cpp | 93 + src/storage/exec/IndexDedupNode.h | 83 + src/storage/exec/IndexEdgeScanNode.cpp | 151 + src/storage/exec/IndexEdgeScanNode.h | 52 + src/storage/exec/IndexExprContext.h | 113 + src/storage/exec/IndexLimitNode.cpp | 51 + src/storage/exec/IndexLimitNode.h | 30 + src/storage/exec/IndexNode.cpp | 30 + src/storage/exec/IndexNode.h | 345 + src/storage/exec/IndexProjectionNode.cpp | 69 + src/storage/exec/IndexProjectionNode.h | 65 + src/storage/exec/IndexScanNode.cpp | 567 ++ src/storage/exec/IndexScanNode.h | 615 ++ src/storage/exec/IndexSelectionNode.cpp | 58 + src/storage/exec/IndexSelectionNode.h | 107 + src/storage/exec/IndexTopNNode.cpp | 132 + src/storage/exec/IndexTopNNode.h | 100 + src/storage/exec/IndexVertexScanNode.cpp | 123 + src/storage/exec/IndexVertexScanNode.h | 62 + src/storage/exec/MultiTagNode.h | 120 + src/storage/exec/QueryUtils.h | 286 + src/storage/exec/RelNode.h | 170 + src/storage/exec/ScanNode.h | 364 + src/storage/exec/StorageIterator.h | 244 + src/storage/exec/StoragePlan.h | 97 + src/storage/exec/TagNode.h | 173 + src/storage/exec/UpdateNode.h | 881 +++ src/storage/exec/UpdateResultNode.h | 96 + src/storage/http/StorageHttpAdminHandler.cpp | 116 + src/storage/http/StorageHttpAdminHandler.h | 47 + .../http/StorageHttpPropertyHandler.cpp | 106 + src/storage/http/StorageHttpPropertyHandler.h | 46 + src/storage/http/StorageHttpStatsHandler.cpp | 59 + src/storage/http/StorageHttpStatsHandler.h | 31 + src/storage/index/LookupProcessor.cpp | 481 ++ src/storage/index/LookupProcessor.h | 62 + src/storage/kv/GetProcessor.cpp | 55 + src/storage/kv/GetProcessor.h | 37 + src/storage/kv/PutProcessor.cpp | 32 + src/storage/kv/PutProcessor.h | 36 + src/storage/kv/RemoveProcessor.cpp | 32 + src/storage/kv/RemoveProcessor.h | 37 + src/storage/mutate/AddEdgesProcessor.cpp | 447 ++ src/storage/mutate/AddEdgesProcessor.h | 69 + src/storage/mutate/AddVerticesProcessor.cpp | 397 + src/storage/mutate/AddVerticesProcessor.h | 63 + src/storage/mutate/DeleteEdgesProcessor.cpp | 233 + src/storage/mutate/DeleteEdgesProcessor.h | 52 + src/storage/mutate/DeleteTagsProcessor.cpp | 159 + src/storage/mutate/DeleteTagsProcessor.h | 42 + .../mutate/DeleteVerticesProcessor.cpp | 192 + src/storage/mutate/DeleteVerticesProcessor.h | 43 + src/storage/mutate/UpdateEdgeProcessor.cpp | 299 + src/storage/mutate/UpdateEdgeProcessor.h | 89 + src/storage/mutate/UpdateVertexProcessor.cpp | 281 + src/storage/mutate/UpdateVertexProcessor.h | 88 + src/storage/query/GetDstBySrcProcessor.cpp | 357 + src/storage/query/GetDstBySrcProcessor.h | 71 + src/storage/query/GetNeighborsProcessor.cpp | 497 ++ src/storage/query/GetNeighborsProcessor.h | 93 + src/storage/query/GetPropProcessor.cpp | 364 + src/storage/query/GetPropProcessor.h | 93 + src/storage/query/QueryBaseProcessor-inl.h | 699 ++ src/storage/query/QueryBaseProcessor.h | 229 + src/storage/query/ScanEdgeProcessor.cpp | 231 + src/storage/query/ScanEdgeProcessor.h | 92 + src/storage/query/ScanVertexProcessor.cpp | 231 + src/storage/query/ScanVertexProcessor.h | 94 + src/storage/stats/CMakeLists.txt | 9 + src/storage/stats/StorageStats.cpp | 30 + src/storage/stats/StorageStats.h | 26 + .../AddAndUpdateVertexAndEdgeBenchmark.cpp | 582 ++ src/storage/test/AddEdgesTest.cpp | 106 + src/storage/test/AddVerticesTest.cpp | 106 + src/storage/test/AdminTaskManagerTest.cpp | 814 ++ src/storage/test/CMakeLists.txt | 785 ++ src/storage/test/ChainAddEdgesTest.cpp | 189 + src/storage/test/ChainDeleteEdgesTest.cpp | 494 ++ src/storage/test/ChainResumeEdgeTest.cpp | 632 ++ src/storage/test/ChainTestUtils.h | 625 ++ src/storage/test/ChainUpdateEdgeTest.cpp | 268 + src/storage/test/CheckpointTest.cpp | 68 + src/storage/test/CompactionTest.cpp | 506 ++ src/storage/test/ConsistUtilTest.cpp | 30 + src/storage/test/DeleteEdgesTest.cpp | 135 + src/storage/test/DeleteTagsTest.cpp | 189 + src/storage/test/DeleteVerticesTest.cpp | 135 + .../test/ElasticSearchBulkInsertTest.cpp | 267 + src/storage/test/GetDstBySrcTest.cpp | 249 + src/storage/test/GetNeighborsBenchmark.cpp | 538 ++ src/storage/test/GetNeighborsTest.cpp | 2099 ++++++ src/storage/test/GetPropTest.cpp | 1039 +++ src/storage/test/IndexScanLimitTest.cpp | 419 ++ src/storage/test/IndexScanTopNTest.cpp | 565 ++ src/storage/test/IndexTest.cpp | 3627 +++++++++ src/storage/test/IndexTestUtil.h | 643 ++ src/storage/test/IndexWithTTLTest.cpp | 870 +++ src/storage/test/IndexWriteTest.cpp | 493 ++ src/storage/test/KVClientTest.cpp | 145 + src/storage/test/KVTest.cpp | 82 + src/storage/test/KillQueryTest.cpp | 220 + src/storage/test/LookupIndexTest.cpp | 3565 +++++++++ src/storage/test/MemoryLockBenchmark.cpp | 76 + src/storage/test/MemoryLockTest.cpp | 133 + src/storage/test/MockHdfsHelper.h | 73 + .../test/PrefixBloomFilterBenchmark.cpp | 134 + src/storage/test/QueryTestUtils.h | 916 +++ src/storage/test/RebuildIndexTest.cpp | 588 ++ src/storage/test/ScanEdgePropBenchmark.cpp | 282 + src/storage/test/ScanEdgeTest.cpp | 402 + src/storage/test/ScanVertexTest.cpp | 562 ++ src/storage/test/ShuffleIpTest.cpp | 38 + src/storage/test/StatsTaskTest.cpp | 402 + src/storage/test/StorageDAGBenchmark.cpp | 249 + src/storage/test/StorageDAGTest.cpp | 83 + .../test/StorageHttpAdminHandlerTest.cpp | 90 + .../test/StorageHttpPropertyHandlerTest.cpp | 100 + .../test/StorageHttpStatsHandlerTest.cpp | 109 + .../test/StorageIndexWriteBenchmark.cpp | 391 + src/storage/test/TestUtils.h | 288 + src/storage/test/TossTestExecutor.h | 391 + src/storage/test/TossTestUtils.h | 247 + src/storage/test/UpdateEdgeTest.cpp | 2133 ++++++ src/storage/test/UpdateVertexTest.cpp | 1491 ++++ .../ChainAddEdgesGroupProcessor.cpp | 84 + .../transaction/ChainAddEdgesGroupProcessor.h | 35 + .../ChainAddEdgesLocalProcessor.cpp | 490 ++ .../transaction/ChainAddEdgesLocalProcessor.h | 168 + .../ChainAddEdgesRemoteProcessor.cpp | 84 + .../ChainAddEdgesRemoteProcessor.h | 36 + src/storage/transaction/ChainBaseProcessor.h | 57 + .../ChainDeleteEdgesGroupProcessor.cpp | 82 + .../ChainDeleteEdgesGroupProcessor.h | 36 + .../ChainDeleteEdgesLocalProcessor.cpp | 377 + .../ChainDeleteEdgesLocalProcessor.h | 119 + .../ChainDeleteEdgesRemoteProcessor.cpp | 72 + .../ChainDeleteEdgesRemoteProcessor.h | 37 + .../ChainDeleteEdgesResumeProcessor.cpp | 59 + .../ChainDeleteEdgesResumeProcessor.h | 34 + .../ChainDeleteEdgesResumeRemoteProcessor.cpp | 76 + .../ChainDeleteEdgesResumeRemoteProcessor.h | 36 + .../transaction/ChainProcessorFactory.cpp | 118 + .../transaction/ChainProcessorFactory.h | 31 + .../ChainResumeAddDoublePrimeProcessor.cpp | 78 + .../ChainResumeAddDoublePrimeProcessor.h | 36 + .../ChainResumeAddPrimeProcessor.cpp | 76 + .../ChainResumeAddPrimeProcessor.h | 34 + .../ChainResumeUpdateDoublePrimeProcessor.cpp | 63 + .../ChainResumeUpdateDoublePrimeProcessor.h | 43 + .../ChainResumeUpdatePrimeProcessor.cpp | 61 + .../ChainResumeUpdatePrimeProcessor.h | 41 + .../ChainUpdateEdgeLocalProcessor.cpp | 297 + .../ChainUpdateEdgeLocalProcessor.h | 97 + .../ChainUpdateEdgeRemoteProcessor.cpp | 51 + .../ChainUpdateEdgeRemoteProcessor.h | 38 + src/storage/transaction/ConsistTypes.h | 43 + src/storage/transaction/ConsistUtil.cpp | 202 + src/storage/transaction/ConsistUtil.h | 107 + .../transaction/TransactionManager.cpp | 287 + src/storage/transaction/TransactionManager.h | 162 + src/tools/CMakeLists.txt | 74 + src/tools/db-dump/CMakeLists.txt | 32 + src/tools/db-dump/DbDumpTool.cpp | 89 + src/tools/db-dump/DbDumper.cpp | 587 ++ src/tools/db-dump/DbDumper.h | 96 + src/tools/db-upgrade/CMakeLists.txt | 34 + src/tools/db-upgrade/DbUpgrader.cpp | 1253 ++++ src/tools/db-upgrade/DbUpgrader.h | 223 + src/tools/db-upgrade/DbUpgraderTool.cpp | 204 + src/tools/db-upgrade/NebulaKeyUtilsV1.cpp | 109 + src/tools/db-upgrade/NebulaKeyUtilsV1.h | 252 + src/tools/db-upgrade/NebulaKeyUtilsV2.cpp | 225 + src/tools/db-upgrade/NebulaKeyUtilsV2.h | 304 + src/tools/db-upgrade/NebulaKeyUtilsV3.cpp | 26 + src/tools/db-upgrade/NebulaKeyUtilsV3.h | 20 + src/tools/meta-dump/CMakeLists.txt | 32 + src/tools/meta-dump/MetaDumpTool.cpp | 317 + src/tools/simple-kv-verify/CMakeLists.txt | 19 + .../simple-kv-verify/SimpleKVVerifyTool.cpp | 153 + src/tools/storage-perf/CMakeLists.txt | 51 + src/tools/storage-perf/README.md | 31 + .../storage-perf/StorageIntegrityTool.cpp | 210 + src/tools/storage-perf/StoragePerfTool.cpp | 454 ++ src/version/CMakeLists.txt | 14 + src/version/Version.cpp.in | 38 + src/version/Version.h | 19 + src/webservice/CMakeLists.txt | 23 + src/webservice/Common.cpp | 20 + src/webservice/Common.h | 45 + src/webservice/GetFlagsHandler.cpp | 201 + src/webservice/GetFlagsHandler.h | 50 + src/webservice/GetStatsHandler.cpp | 123 + src/webservice/GetStatsHandler.h | 48 + src/webservice/NotFoundHandler.cpp | 49 + src/webservice/NotFoundHandler.h | 33 + src/webservice/Router.cpp | 141 + src/webservice/Router.h | 109 + src/webservice/SetFlagsHandler.cpp | 120 + src/webservice/SetFlagsHandler.h | 38 + src/webservice/StatusHandler.cpp | 73 + src/webservice/StatusHandler.h | 45 + src/webservice/WebService.cpp | 159 + src/webservice/WebService.h | 63 + src/webservice/test/CMakeLists.txt | 97 + src/webservice/test/FlagsAccessTest.cpp | 150 + src/webservice/test/RouterTest.cpp | 162 + src/webservice/test/StatsReaderTest.cpp | 209 + src/webservice/test/StatusHandlerTest.cpp | 76 + src/webservice/test/TestUtils.h | 42 + tests/.gitignore | 3 + tests/.pylintrc | 570 ++ tests/Makefile | 158 + tests/README.md | 234 + tests/__init__.py | 0 tests/admin/__init__.py | 0 tests/admin/test_charset.py | 51 + tests/admin/test_configs.py | 149 + tests/admin/test_listener.py | 41 + tests/admin/test_parts.py | 62 + tests/admin/test_permission.py | 1041 +++ tests/admin/test_show_hosts.py | 41 + tests/admin/test_space.py | 217 + tests/admin/test_users.py | 210 + tests/bench/__init__.py | 0 tests/bench/data_generate.py | 63 + tests/bench/delete.py | 68 + tests/bench/insert.py | 72 + tests/bench/lookup.py | 74 + tests/cert/test.2.crt | 22 + tests/cert/test.2.csr | 18 + tests/cert/test.2.key | 30 + tests/cert/test.2.password | 1 + tests/cert/test.ca.key | 30 + tests/cert/test.ca.password | 1 + tests/cert/test.ca.pem | 24 + tests/cert/test.ca.srl | 1 + tests/cert/test.derive.crt | 23 + tests/cert/test.derive.csr | 19 + tests/cert/test.derive.key | 27 + tests/common/__init__.py | 0 tests/common/comparator.py | 385 + tests/common/configs.py | 61 + tests/common/constants.py | 15 + tests/common/csv_import.py | 205 + tests/common/dataset_printer.py | 130 + tests/common/nebula_service.py | 738 ++ tests/common/nebula_test_suite.py | 418 ++ tests/common/path_value.py | 31 + tests/common/plan_differ.py | 297 + tests/common/types.py | 196 + tests/common/utils.py | 474 ++ tests/conftest.py | 279 + tests/data/ldbc_v0_3_3/config.yaml | 42 + tests/data/nba.ngql | 375 + tests/data/nba/bachelor.csv | 2 + tests/data/nba/config.yaml | 135 + tests/data/nba/like.csv | 83 + tests/data/nba/null.csv | 5 + tests/data/nba/player.csv | 53 + tests/data/nba/serve.csv | 153 + tests/data/nba/team.csv | 31 + tests/data/nba/teammate.csv | 11 + tests/data/nba_int_vid/config.yaml | 145 + tests/data/ngdata/Edge_label_limits.csv | 26 + tests/data/ngdata/Edge_ttl_0.csv | 291 + tests/data/ngdata/Label_0.csv | 61 + tests/data/ngdata/Label_1.csv | 57 + tests/data/ngdata/Label_10.csv | 56 + tests/data/ngdata/Label_11.csv | 66 + tests/data/ngdata/Label_2.csv | 56 + tests/data/ngdata/Label_3.csv | 60 + tests/data/ngdata/Label_4.csv | 57 + tests/data/ngdata/Label_5.csv | 65 + tests/data/ngdata/Label_6.csv | 51 + tests/data/ngdata/Label_7.csv | 63 + tests/data/ngdata/Label_8.csv | 61 + tests/data/ngdata/Label_9.csv | 64 + tests/data/ngdata/Label_same_prop_0.csv | 67 + tests/data/ngdata/Label_same_prop_1.csv | 58 + tests/data/ngdata/Label_ttl_0.csv | 60 + tests/data/ngdata/README.md | 3 + tests/data/ngdata/Rel_0.csv | 291 + tests/data/ngdata/Rel_1.csv | 298 + tests/data/ngdata/Rel_2.csv | 288 + tests/data/ngdata/Rel_3.csv | 286 + tests/data/ngdata/Rel_4.csv | 302 + tests/data/ngdata/Rel_5.csv | 305 + tests/data/ngdata/config.yaml | 775 ++ tests/data/ngdata/gen_graph.sh | 15 + tests/data/student/config.yaml | 139 + tests/data/student/is_colleagues.csv | 8 + tests/data/student/is_friend.csv | 8 + tests/data/student/is_schoolmate.csv | 27 + tests/data/student/is_teacher.csv | 12 + tests/data/student/person.csv | 31 + tests/data/student/student.csv | 21 + tests/data/student/teacher.csv | 11 + tests/job/__init__.py | 0 tests/job/test_session.py | 455 ++ tests/job/test_snapshot.py | 46 + tests/maintain/__init__.py | 0 tests/maintain/test_comments.py | 27 + tests/maintain/test_zone.py | 13 + tests/nebula-test-run.py | 267 + tests/pytest.ini | 25 + tests/query/__init__.py | 0 tests/query/bugs/__init__.py | 0 tests/query/bugs/fixed_bigint_2031.py | 44 + tests/query/bugs/fixed_delete_vertex_1996.py | 36 + tests/query/bugs/fixed_issue2020.py | 47 + tests/query/bugs/fixed_negative_ttl.py | 52 + tests/query/bugs/fixed_over_all_pr1962.py | 54 + tests/query/bugs/fixed_pr2042.py | 37 + tests/query/bugs/fixed_update_issue1827.py | 46 + ...emove_unreasonable_error_message_pr1983.py | 38 + tests/query/bugs/test_fixed_issue1888.py | 59 + tests/query/bugs/test_fixed_issue1987.py | 76 + tests/query/bugs/test_fixed_issue2009.py | 66 + ...er_schema_altered_issue1185_1154_pr1606.py | 154 + ...st_query_after_schema_rebuilt_issue1843.py | 200 + tests/query/stateless/__init__.py | 0 tests/query/stateless/test_admin.py | 76 + tests/query/stateless/test_fetch.py | 46 + tests/query/stateless/test_findpath.py | 126 + tests/query/stateless/test_go.py | 229 + tests/query/stateless/test_groupby.py | 128 + tests/query/stateless/test_if_exists.py | 93 + tests/query/stateless/test_index.py | 172 + tests/query/stateless/test_keyword.py | 932 +++ tests/query/stateless/test_query_reversely.py | 78 + tests/query/stateless/test_range.py | 52 + tests/query/stateless/test_schema.py | 351 + tests/query/stateless/test_set.py | 150 + tests/query/stateless/test_update.py | 287 + tests/query/stateless/test_where.py | 125 + tests/query/stateless/test_yield.py | 185 + tests/requirements.txt | 19 + tests/tck/__init__.py | 0 tests/tck/cluster/Example.feature | 59 + tests/tck/cluster/terminate.feature | 10 + tests/tck/conftest.py | 998 +++ .../tck/features/admin/Authentication.feature | 128 + tests/tck/features/admin/Hosts.feature | 62 + tests/tck/features/admin/Sessions.feature | 74 + tests/tck/features/aggregate/Agg.feature | 935 +++ tests/tck/features/basic/Parser.feature | 16 + tests/tck/features/basic/data.feature | 65 + .../bugfix/AggPatternExpression.feature | 18 + .../features/bugfix/AliasTypeDeduce.feature | 49 + .../bugfix/ArgumentPlanNodeDep.feature | 28 + .../bugfix/BoolCheckingInExpr.feature | 16 + .../features/bugfix/BuildPathMistake.feature | 18 + tests/tck/features/bugfix/CompareDate.feature | 30 + .../features/bugfix/ContainsFilter.feature | 33 + .../bugfix/CrashWhenNullPathExpr.feature | 17 + .../CreateDurationAsDefaultValue.feature | 23 + .../tck/features/bugfix/DropRootUser.feature | 12 + .../features/bugfix/DupAliasInMatch.feature | 45 + .../tck/features/bugfix/ExtractFilter.feature | 17 + .../features/bugfix/FindStartError.feature | 16 + .../tck/features/bugfix/FixIterCrash.feature | 23 + .../bugfix/GetEdgesTransformLimitRule.feature | 23 + tests/tck/features/bugfix/InnerVar.feature | 25 + .../InsertMismatchedTypeDateTime.feature | 42 + .../bugfix/LackFilterGetEdges.feature | 48 + .../bugfix/ListSlicingWithNull.feature | 14 + tests/tck/features/bugfix/LookupIn.feature | 80 + .../bugfix/MTSafeConcurrencyVariables.feature | 14 + tests/tck/features/bugfix/MatchCrash.feature | 18 + .../features/bugfix/MatchJoinOnEdge.feature | 69 + .../bugfix/MatchNodeLabelFilter.feature | 62 + .../MatchNotFilterTheUndeclaredTag.feature | 96 + .../bugfix/MatchReturnEmptyTag.feature | 126 + .../features/bugfix/MatchUsedInPipe.feature | 137 + .../features/bugfix/NaNInfinityFloat.feature | 43 + .../features/bugfix/NullInputOfLength.feature | 18 + tests/tck/features/bugfix/NullProp.feature | 20 + .../bugfix/PredicationPushDownInGo.feature | 32 + .../bugfix/PrunePropertiesError.feature | 27 + .../bugfix/PushFilterDownProject.feature | 30 + .../features/bugfix/RewriteEdgeFilter.feature | 16 + tests/tck/features/bugfix/RoundFloat.feature | 55 + .../bugfix/StringFlaotAddition.feature | 68 + .../bugfix/SubgraphBeforePipe.feature | 349 + .../features/bugfix/SubscriptInUpdate.feature | 50 + .../features/bugfix/SyntaxErrorCrash.feature | 32 + .../bugfix/TestYieldConstantAfterPipe.feature | 41 + .../features/bugfix/TimeDefaultValue.feature | 35 + .../bugfix/TruncatedStringIndex.feature | 75 + tests/tck/features/bugfix/TypoError.feature | 34 + .../features/bugfix/UseSpaceAndMatch.feature | 41 + .../bugfix/VariableExpression.feature | 25 + .../features/bugfix/VidInvalidLength.feature | 151 + tests/tck/features/ddl/Ddl.feature | 298 + .../features/delete/DeleteEdge.IntVid.feature | 268 + tests/tck/features/delete/DeleteEdge.feature | 268 + .../features/delete/DeleteTag.IntVid.feature | 312 + tests/tck/features/delete/DeleteTag.feature | 312 + .../delete/DeleteVertex.IntVid.feature | 317 + .../tck/features/delete/DeleteVertex.feature | 310 + .../delete/DeleteVertexWithoutEdge.feature | 135 + .../explain/ExplainAndProfile.feature | 90 + .../tck/features/expression/Attribute.feature | 161 + .../features/expression/Attribute1.feature | 33 + .../expression/BugFixWithngdata.feature | 18 + tests/tck/features/expression/Case.feature | 482 ++ tests/tck/features/expression/Depth.feature | 143 + .../tck/features/expression/EndsWith.feature | 142 + .../features/expression/FixedString.feature | 54 + .../features/expression/FunctionCall.feature | 233 + tests/tck/features/expression/In.feature | 156 + .../tck/features/expression/LabelExpr.feature | 36 + .../expression/ListComprehension.feature | 108 + .../expression/ListRangeSubscript.feature | 28 + .../expression/LogicalExpression.feature | 44 + tests/tck/features/expression/NotIn.feature | 152 + tests/tck/features/expression/Null.feature | 66 + .../tck/features/expression/Predicate.feature | 361 + tests/tck/features/expression/Reduce.feature | 106 + tests/tck/features/expression/Regex.feature | 124 + .../expression/RelationalExpr.feature | 253 + .../features/expression/StartsWith.feature | 150 + .../expression/TimeComparison.feature | 23 + .../expression/TimeComputation.feature | 109 + .../tck/features/expression/UnaryExpr.feature | 103 + .../expression/function/Mathematical.feature | 28 + .../function/TypeConversion.feature | 120 + .../features/fetch/FetchEdges.intVid.feature | 322 + .../features/fetch/FetchEdges.strVid.feature | 326 + tests/tck/features/fetch/FetchEmpty.feature | 174 + .../fetch/FetchVertices.intVid.feature | 418 ++ .../fetch/FetchVertices.strVid.feature | 529 ++ .../fulltext_index/FulltextIndexScan.feature | 372 + .../fulltext_index/FultextIndexDDL.feature | 109 + tests/tck/features/function/coalesce.feature | 44 + .../features/function/json_extract.feature | 59 + tests/tck/features/function/round.feature | 236 + tests/tck/features/geo/GeoBase.feature | 789 ++ tests/tck/features/go/GO.IntVid.feature | 1834 +++++ tests/tck/features/go/GO.feature | 2067 ++++++ .../tck/features/go/GoYieldVertexEdge.feature | 1929 +++++ .../features/go/GroupbyLimit.IntVid.feature | 346 + tests/tck/features/go/GroupbyLimit.feature | 399 + tests/tck/features/go/Orderby.feature | 233 + tests/tck/features/go/SampleLimit.feature | 74 + .../features/go/SampleLimit.intVid.feature | 64 + tests/tck/features/go/SimpleCase.feature | 606 ++ tests/tck/features/index/Index.IntVid.feature | 698 ++ tests/tck/features/index/Index.feature | 3619 +++++++++ tests/tck/features/index/TagEdgeIndex.feature | 520 ++ tests/tck/features/insert/BoundCheck.feature | 63 + .../tck/features/insert/Insert.IntVid.feature | 628 ++ tests/tck/features/insert/Insert.feature | 689 ++ .../insert/InsertEdgeOnDiffParts.feature | 39 + .../features/insert/InsertIfNotExists.feature | 542 ++ .../features/insert/insertVertexOnly.feature | 42 + tests/tck/features/lookup/ByIndex.feature | 604 ++ .../features/lookup/ByIndex.intVid.feature | 603 ++ .../features/lookup/EdgeIndexFullScan.feature | 352 + .../tck/features/lookup/LookUp.IntVid.feature | 848 +++ tests/tck/features/lookup/LookUp.feature | 1066 +++ tests/tck/features/lookup/LookUpLimit.feature | 187 + tests/tck/features/lookup/LookUpTopN.feature | 263 + tests/tck/features/lookup/LookupEdge.feature | 129 + tests/tck/features/lookup/LookupEdge2.feature | 60 + tests/tck/features/lookup/LookupTag.feature | 130 + tests/tck/features/lookup/LookupTag2.feature | 91 + tests/tck/features/lookup/Output.feature | 99 + .../tck/features/lookup/Output.intVid.feature | 96 + .../features/lookup/TagIndexFullScan.feature | 432 ++ tests/tck/features/lookup/WithYield.feature | 113 + .../features/lookup/WithYield.intVid.feature | 114 + .../features/match/AllShortestPaths.feature | 1034 +++ tests/tck/features/match/Base.IntVid.feature | 1217 +++ tests/tck/features/match/Base.feature | 2391 ++++++ .../tck/features/match/IndexSelecting.feature | 170 + .../features/match/MatchById.IntVid.feature | 1124 +++ tests/tck/features/match/MatchById.feature | 1177 +++ .../features/match/MatchByVariable.feature | 1728 +++++ tests/tck/features/match/MatchGroupBy.feature | 238 + .../match/MultiLineMultiQueryParts.feature | 387 + .../features/match/MultiQueryParts.feature | 323 + tests/tck/features/match/Path.feature | 513 ++ tests/tck/features/match/PathExpr.feature | 524 ++ .../match/PathExprRefLocalVariable.feature | 299 + .../features/match/PipeAndVariable.feature | 338 + .../tck/features/match/RedefinedNode.feature | 211 + .../features/match/SameTagPropname.feature | 214 + tests/tck/features/match/Scan.feature | 419 ++ tests/tck/features/match/SeekByEdge.feature | 1519 ++++ tests/tck/features/match/SeekById.feature | 867 +++ .../features/match/SeekById.intVid.feature | 677 ++ tests/tck/features/match/SeekByTag.feature | 113 + .../features/match/SeekByTag.intVid.feature | 113 + .../match/SelfReflectiveEdges.feature | 46 + .../features/match/SingleShorestPath.feature | 798 ++ .../match/StartFromAnyNode.IntVid.feature | 526 ++ .../features/match/StartFromAnyNode.feature | 526 ++ tests/tck/features/match/Unwind.feature | 271 + .../match/VariableLengthPattern.feature | 518 ++ .../VariableLengthPattern.intVid.feature | 355 + tests/tck/features/match/With.feature | 523 ++ tests/tck/features/match/ZeroStep.feature | 493 ++ .../features/match/ZeroStep.intVid.feature | 456 ++ tests/tck/features/match/ZeroStep2.feature | 238 + tests/tck/features/mutate/ClearSpace.feature | 125 + .../mutate/InsertDurationType.feature | 132 + .../mutate/InsertWithTimeType.feature | 222 + .../optimizer/CasesUsingTestSpace.feature | 40 + .../optimizer/CollapseProjectRule.feature | 107 + .../optimizer/CombineFilterRule.feature | 28 + .../EliminateAppendVerticesRule.feature | 41 + .../optimizer/ElimintateInvalidProp.feature | 47 + .../EmbedEdgeAllPredIntoTraverseRule.feature | 709 ++ .../features/optimizer/IndexScanRule.feature | 149 + .../MergeGetNbrsDedupProjectRule.feature | 43 + .../MergeGetVerticesDedupProjectRule.feature | 29 + .../optimizer/PrunePropertiesRule.feature | 999 +++ .../optimizer/PushEFilterDownRule.feature | 93 + .../PushFilterDownAggregateRule.feature | 83 + .../optimizer/PushFilterDownBugFixes.feature | 54 + .../PushFilterDownCrossJoinRule.feature | 33 + .../PushFilterDownExpandAllRule.feature | 172 + .../PushFilterDownHashInnerJoinRule.feature | 280 + .../PushFilterDownHashLeftJoinRule.feature | 164 + .../optimizer/PushFilterDownNodeRule.feature | 73 + .../PushFilterDownProjectRule.feature | 59 + .../PushFilterDownTraverseRule.feature | 227 + .../PushLimitDownExpandAllRule.feature | 75 + .../PushLimitDownProjectRule.feature | 34 + .../PushLimitDownScanEdgesRule.feature | 45 + .../PushLimitDownScanVerticesRule.feature | 43 + .../optimizer/PushSampleDownRule.feature | 129 + .../RemoveAppendVerticesBelowJoinRule.feature | 91 + .../RemoveUselessProjectRule.feature | 71 + tests/tck/features/optimizer/TopNRule.feature | 86 + tests/tck/features/parser/Example.feature | 66 + tests/tck/features/parser/nebula.feature | 58 + .../tck/features/path/AllPath.IntVid.feature | 346 + tests/tck/features/path/AllPath.feature | 551 ++ tests/tck/features/path/NoLoop.IntVid.feature | 244 + tests/tck/features/path/NoLoop.feature | 297 + .../features/path/ShortestPath.IntVid.feature | 598 ++ tests/tck/features/path/ShortestPath.feature | 710 ++ .../features/path/singleShortestPath.feature | 151 + tests/tck/features/schema/Comment.feature | 229 + .../tck/features/schema/CreateSpaceAs.feature | 209 + tests/tck/features/schema/Schema.feature | 989 +++ tests/tck/features/set/Set.IntVid.feature | 299 + tests/tck/features/set/Set.feature | 521 ++ .../features/subgraph/subgraph.IntVid.feature | 1019 +++ tests/tck/features/subgraph/subgraph.feature | 1074 +++ .../subgraph/subgraphWithFilter.feature | 138 + tests/tck/features/ttl/TTL.feature | 536 ++ tests/tck/features/ttl/TTL2.feature | 512 ++ .../tck/features/update/Update.IntVid.feature | 651 ++ tests/tck/features/update/Update.feature | 1209 +++ tests/tck/features/user/User.feature | 770 ++ .../VerifyClientVersion.feature | 14 + .../tck/features/yield/NoSpaceChosen.feature | 116 + tests/tck/features/yield/join.feature | 382 + tests/tck/features/yield/parameter.feature | 458 ++ tests/tck/features/yield/return.feature | 144 + tests/tck/features/yield/yield.IntVid.feature | 674 ++ tests/tck/features/yield/yield.feature | 707 ++ tests/tck/job/Job.feature | 197 + .../Read.feature | 581 ++ .../interactive_workload/ComplexReads.feature | 358 + .../interactive_workload/ShortReads.feature | 131 + .../features/expressions/list/List2.feature | 182 + .../features/expressions/map/Map1.feature | 30 + .../features/expressions/map/Map2.feature | 64 + .../features/expressions/path/Path1.feature | 20 + .../features/expressions/path/Path2.feature | 58 + .../features/expressions/path/Path3.feature | 66 + .../slowquery/KillSlowQueryBaseTest.feature | 101 + .../KillSlowQueryViaDiffrentService.feature | 64 + .../KillSlowQueryViaSameService.feature | 43 + .../PermissionViaDifferentService.feature | 146 + .../PermissionViaSameService.feature | 99 + tests/tck/steps/__init__.py | 0 tests/tck/steps/conftest.py | 45 + tests/tck/steps/test_cluster.py | 29 + tests/tck/steps/test_jobs.py | 8 + ...t_kill_permission_via_different_service.py | 7 + .../test_kill_permission_via_same_service.py | 7 + .../steps/test_kill_slow_query_base_test.py | 7 + ...t_kill_slow_query_via_different_service.py | 8 + .../test_kill_slow_query_via_same_service.py | 8 + tests/tck/steps/test_ldbc.py | 7 + tests/tck/steps/test_tck.py | 8 + tests/tck/utils/.gitignore | 3 + tests/tck/utils/__init__.py | 0 tests/tck/utils/mmh2.py | 55 + tests/tck/utils/nbv.py | 705 ++ tests/tck/utils/table.py | 54 + third-party/cxx-compiler-abi-version.sh | 26 + third-party/install-cmake.sh | 70 + third-party/install-gcc.sh | 115 + third-party/install-third-party.sh | 103 + udf/.gitignore | 1 + udf/standard_deviation.cpp | 91 + udf/standard_deviation.h | 60 + 2280 files changed, 428629 insertions(+), 1 deletion(-) create mode 100644 .clang-format create mode 100644 .clang-tidy create mode 100644 .dockerignore create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/enhancement.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/Security.md create mode 100644 .github/actions/tagname-action/README.md create mode 100644 .github/actions/tagname-action/action.yml create mode 100644 .github/actions/upload-assets-action/README.md create mode 100644 .github/actions/upload-assets-action/action.yml create mode 100644 .github/actions/upload-to-oss-action/README.md create mode 100644 .github/actions/upload-to-oss-action/action.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/auto_cherry_pick.yml create mode 100644 .github/workflows/check_label.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/rc.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 .licenserc.yaml create mode 100644 .linters/cpp/checkKeyword.py create mode 100644 .linters/cpp/cpplint.py create mode 100644 .linters/cpp/hooks/pre-commit.sh create mode 100644 .linters/java/nebula_java_style_checks.xml create mode 100644 CMakeLists.txt create mode 100644 CONTRIBUTING.md create mode 100644 Coding_Style_Guide.md create mode 100644 LICENSE create mode 100644 README-CN.md create mode 100644 cmake/CPackage.cmake create mode 100644 cmake/ClangTidy.cmake create mode 100644 cmake/FetchModule.cmake create mode 100644 cmake/FindBreakpad.cmake create mode 100644 cmake/FindBzip2.cmake create mode 100644 cmake/FindDoubleConversion.cmake create mode 100644 cmake/FindFatal.cmake create mode 100644 cmake/FindFbthrift.cmake create mode 100644 cmake/FindFizz.cmake create mode 100644 cmake/FindFolly.cmake create mode 100644 cmake/FindGflags.cmake create mode 100644 cmake/FindGlog.cmake create mode 100644 cmake/FindGoogletest.cmake create mode 100644 cmake/FindJemalloc.cmake create mode 100644 cmake/FindLibevent.cmake create mode 100644 cmake/FindLibunwind.cmake create mode 100644 cmake/FindMoldLinker.cmake create mode 100644 cmake/FindPCHSupport.cmake create mode 100644 cmake/FindProxygen.cmake create mode 100644 cmake/FindRocksdb.cmake create mode 100644 cmake/FindSnappy.cmake create mode 100644 cmake/FindSodium.cmake create mode 100644 cmake/FindWangle.cmake create mode 100644 cmake/FindZstd.cmake create mode 100644 cmake/ThriftGenerate.cmake create mode 100644 cmake/nebula/ABIConfig.cmake create mode 100644 cmake/nebula/CompilerLauncher.cmake create mode 100644 cmake/nebula/GeneralCMakeConfig.cmake create mode 100644 cmake/nebula/GeneralCMakeOptions.cmake create mode 100644 cmake/nebula/GeneralCompilerConfig.cmake create mode 100644 cmake/nebula/GitHooksConfig.cmake create mode 100644 cmake/nebula/GitInfoConfig.cmake create mode 100644 cmake/nebula/IncludeWhatYouUse.cmake create mode 100644 cmake/nebula/InstallThirdParty.cmake create mode 100644 cmake/nebula/LinkerConfig.cmake create mode 100644 cmake/nebula/MakeBisonRelocatable.cmake create mode 100644 cmake/nebula/NebulaCMakeMacros.cmake create mode 100644 cmake/nebula/NebulaCustomTargets.cmake create mode 100644 cmake/nebula/PlatformCheck.cmake create mode 100644 cmake/nebula/SanitizerConfig.cmake create mode 100644 cmake/nebula/ThirdPartyConfig.cmake create mode 100644 cmake/nebula/TimeTrace.cmake create mode 100644 codecov.yml create mode 100644 conf/CMakeLists.txt create mode 100644 conf/nebula-graphd.conf.default create mode 100644 conf/nebula-graphd.conf.production create mode 100644 conf/nebula-metad.conf.default create mode 100644 conf/nebula-metad.conf.production create mode 100644 conf/nebula-standalone.conf.default create mode 100644 conf/nebula-storaged-listener.conf.default create mode 100644 conf/nebula-storaged-listener.conf.production create mode 100644 conf/nebula-storaged.conf.default create mode 100644 conf/nebula-storaged.conf.production create mode 100644 conf/tuned/README.md create mode 100644 conf/tuned/nebula/tuned.conf create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.graphd create mode 100644 docker/Dockerfile.metad create mode 100644 docker/Dockerfile.storaged create mode 100644 docker/Dockerfile.tools create mode 100644 docker/README.md create mode 100644 docs/rfcs/0000-template.md create mode 100644 package/package.sh create mode 100644 package/postinst create mode 100644 package/rpm_postinst create mode 100644 resources/CMakeLists.txt create mode 100644 resources/README.md create mode 100644 resources/date_time_zonespec.csv create mode 100644 resources/gflags.json create mode 100644 scripts/CMakeLists.txt create mode 100644 scripts/README.md create mode 100644 scripts/meta-transfer-tools.sh create mode 100644 scripts/nebula-graphd.service create mode 100644 scripts/nebula-metad.service create mode 100644 scripts/nebula-standalone.service create mode 100644 scripts/nebula-storaged-listener.service create mode 100644 scripts/nebula-storaged.service create mode 100644 scripts/nebula.service create mode 100644 scripts/services.sh create mode 100644 scripts/utils.sh create mode 100644 src/CMakeLists.txt create mode 100644 src/clients/CMakeLists.txt create mode 100644 src/clients/graph/README.md create mode 100644 src/clients/meta/CMakeLists.txt create mode 100644 src/clients/meta/FileBasedClusterIdMan.cpp create mode 100644 src/clients/meta/FileBasedClusterIdMan.h create mode 100644 src/clients/meta/MetaClient.cpp create mode 100644 src/clients/meta/MetaClient.h create mode 100644 src/clients/meta/stats/CMakeLists.txt create mode 100644 src/clients/meta/stats/MetaClientStats.cpp create mode 100644 src/clients/meta/stats/MetaClientStats.h create mode 100644 src/clients/meta/test/CMakeLists.txt create mode 100644 src/clients/meta/test/FileBasedClusterIdManTest.cpp create mode 100644 src/clients/storage/CMakeLists.txt create mode 100644 src/clients/storage/GeneralStorageClient.h create mode 100644 src/clients/storage/InternalStorageClient.cpp create mode 100644 src/clients/storage/InternalStorageClient.h create mode 100644 src/clients/storage/StorageClient.cpp create mode 100644 src/clients/storage/StorageClient.h create mode 100644 src/clients/storage/StorageClientBase-inl.h create mode 100644 src/clients/storage/StorageClientBase.cpp create mode 100644 src/clients/storage/StorageClientBase.h create mode 100644 src/clients/storage/stats/CMakeLists.txt create mode 100644 src/clients/storage/stats/StorageClientStats.cpp create mode 100644 src/clients/storage/stats/StorageClientStats.h create mode 100644 src/codec/CMakeLists.txt create mode 100644 src/codec/Common.h create mode 100644 src/codec/NebulaCodecImpl.cpp create mode 100644 src/codec/RowReaderV2.cpp create mode 100644 src/codec/RowReaderV2.h create mode 100644 src/codec/RowReaderWrapper.cpp create mode 100644 src/codec/RowReaderWrapper.h create mode 100644 src/codec/RowWriterV2.cpp create mode 100644 src/codec/RowWriterV2.h create mode 100644 src/codec/include/NebulaCodec.h create mode 100644 src/codec/test/CMakeLists.txt create mode 100644 src/codec/test/RowReaderV2Test.cpp create mode 100644 src/codec/test/RowWriterBenchmark.cpp create mode 100644 src/codec/test/RowWriterV1-inl.h create mode 100644 src/codec/test/RowWriterV1.cpp create mode 100644 src/codec/test/RowWriterV1.h create mode 100644 src/codec/test/RowWriterV2Test.cpp create mode 100644 src/common/CMakeLists.txt create mode 100644 src/common/algorithm/CMakeLists.txt create mode 100644 src/common/algorithm/ReservoirSampling.cpp create mode 100644 src/common/algorithm/ReservoirSampling.h create mode 100644 src/common/algorithm/test/CMakeLists.txt create mode 100644 src/common/algorithm/test/ReservoirSamplingTest.cpp create mode 100644 src/common/base/Arena.cpp create mode 100644 src/common/base/Arena.h create mode 100644 src/common/base/Base.cpp create mode 100644 src/common/base/Base.h create mode 100644 src/common/base/CMakeLists.txt create mode 100644 src/common/base/CheckPointer.h create mode 100644 src/common/base/CollectNSucceeded-inl.h create mode 100644 src/common/base/CollectNSucceeded.h create mode 100644 src/common/base/CommonMacro.h create mode 100644 src/common/base/ConcurrentLRUCache.h create mode 100644 src/common/base/Cord.cpp create mode 100644 src/common/base/Cord.h create mode 100644 src/common/base/EitherOr.h create mode 100644 src/common/base/ErrorOr.h create mode 100644 src/common/base/GdbDebugScript.cpp create mode 100644 src/common/base/ICord.h create mode 100644 src/common/base/Logging.h create mode 100644 src/common/base/MurmurHash2.h create mode 100644 src/common/base/ObjectPool.h create mode 100644 src/common/base/SanitizerOptions.cpp create mode 100644 src/common/base/SignalHandler.cpp create mode 100644 src/common/base/SignalHandler.h create mode 100644 src/common/base/Status.cpp create mode 100644 src/common/base/Status.h create mode 100644 src/common/base/StatusOr.h create mode 100644 src/common/base/test/ArenaBenchmark.cpp create mode 100644 src/common/base/test/ArenaTest.cpp create mode 100644 src/common/base/test/CMakeLists.txt create mode 100644 src/common/base/test/ConcurrentLRUCacheTest.cpp create mode 100644 src/common/base/test/CordTest.cpp create mode 100644 src/common/base/test/CordTestT.cpp create mode 100644 src/common/base/test/EitherOrTest.cpp create mode 100644 src/common/base/test/ErrorOrTest.cpp create mode 100644 src/common/base/test/HashBenchmark.cpp create mode 100644 src/common/base/test/ICordBenchmark.cpp create mode 100644 src/common/base/test/ICordTest.cpp create mode 100644 src/common/base/test/LoggingBenchmark.cpp create mode 100644 src/common/base/test/MurmurHash2Test.cpp create mode 100644 src/common/base/test/ObjectPoolTest.cpp create mode 100644 src/common/base/test/RangeVsTransformBenchmark.cpp create mode 100644 src/common/base/test/SignalHandlerTest.cpp create mode 100644 src/common/base/test/StatusOrTest.cpp create mode 100644 src/common/base/test/StatusTest.cpp create mode 100644 src/common/charset/CMakeLists.txt create mode 100644 src/common/charset/Charset.cpp create mode 100644 src/common/charset/Charset.h create mode 100644 src/common/charset/test/CMakeLists.txt create mode 100644 src/common/charset/test/CharsetTest.cpp create mode 100644 src/common/conf/CMakeLists.txt create mode 100644 src/common/conf/Configuration.cpp create mode 100644 src/common/conf/Configuration.h create mode 100644 src/common/conf/test/CMakeLists.txt create mode 100644 src/common/conf/test/ConfigurationTest.cpp create mode 100644 src/common/context/ExpressionContext.h create mode 100644 src/common/cpp/helpers.h create mode 100644 src/common/datatypes/CMakeLists.txt create mode 100644 src/common/datatypes/CommonCpp2Ops.h create mode 100644 src/common/datatypes/DataSet.h create mode 100644 src/common/datatypes/DataSetOps-inl.h create mode 100644 src/common/datatypes/Date.cpp create mode 100644 src/common/datatypes/Date.h create mode 100644 src/common/datatypes/DateOps-inl.h create mode 100644 src/common/datatypes/Duration.cpp create mode 100644 src/common/datatypes/Duration.h create mode 100644 src/common/datatypes/DurationOps-inl.h create mode 100644 src/common/datatypes/Edge.cpp create mode 100644 src/common/datatypes/Edge.h create mode 100644 src/common/datatypes/EdgeOps-inl.h create mode 100644 src/common/datatypes/Geography.cpp create mode 100644 src/common/datatypes/Geography.h create mode 100644 src/common/datatypes/GeographyOps-inl.h create mode 100644 src/common/datatypes/HostAddr.cpp create mode 100644 src/common/datatypes/HostAddr.h create mode 100644 src/common/datatypes/HostAddrOps-inl.h create mode 100644 src/common/datatypes/KeyValue.h create mode 100644 src/common/datatypes/KeyValueOps-inl.h create mode 100644 src/common/datatypes/List.cpp create mode 100644 src/common/datatypes/List.h create mode 100644 src/common/datatypes/ListOps-inl.h create mode 100644 src/common/datatypes/Map.cpp create mode 100644 src/common/datatypes/Map.h create mode 100644 src/common/datatypes/MapOps-inl.h create mode 100644 src/common/datatypes/Path.cpp create mode 100644 src/common/datatypes/Path.h create mode 100644 src/common/datatypes/PathOps-inl.h create mode 100644 src/common/datatypes/Set.cpp create mode 100644 src/common/datatypes/Set.h create mode 100644 src/common/datatypes/SetOps-inl.h create mode 100644 src/common/datatypes/Value.cpp create mode 100644 src/common/datatypes/Value.h create mode 100644 src/common/datatypes/ValueOps-inl.h create mode 100644 src/common/datatypes/Vertex.cpp create mode 100644 src/common/datatypes/Vertex.h create mode 100644 src/common/datatypes/VertexOps-inl.h create mode 100644 src/common/datatypes/test/CMakeLists.txt create mode 100644 src/common/datatypes/test/DataSetTest.cpp create mode 100644 src/common/datatypes/test/DateTest.cpp create mode 100644 src/common/datatypes/test/EdgeBenchmark.cpp create mode 100644 src/common/datatypes/test/EdgeTest.cpp create mode 100644 src/common/datatypes/test/GeographyTest.cpp create mode 100644 src/common/datatypes/test/HostAddrTest.cpp create mode 100644 src/common/datatypes/test/PathTest.cpp create mode 100644 src/common/datatypes/test/ValueBenchmark.cpp create mode 100644 src/common/datatypes/test/ValueTest.cpp create mode 100644 src/common/datatypes/test/ValueToJsonTest.cpp create mode 100644 src/common/expression/AggregateExpression.cpp create mode 100644 src/common/expression/AggregateExpression.h create mode 100644 src/common/expression/ArithmeticExpression.cpp create mode 100644 src/common/expression/ArithmeticExpression.h create mode 100644 src/common/expression/AttributeExpression.cpp create mode 100644 src/common/expression/AttributeExpression.h create mode 100644 src/common/expression/BinaryExpression.cpp create mode 100644 src/common/expression/BinaryExpression.h create mode 100644 src/common/expression/CMakeLists.txt create mode 100644 src/common/expression/CaseExpression.cpp create mode 100644 src/common/expression/CaseExpression.h create mode 100644 src/common/expression/ColumnExpression.cpp create mode 100644 src/common/expression/ColumnExpression.h create mode 100644 src/common/expression/ConstantExpression.cpp create mode 100644 src/common/expression/ConstantExpression.h create mode 100644 src/common/expression/ContainerExpression.cpp create mode 100644 src/common/expression/ContainerExpression.h create mode 100644 src/common/expression/EdgeExpression.cpp create mode 100644 src/common/expression/EdgeExpression.h create mode 100644 src/common/expression/ExprVisitor.h create mode 100644 src/common/expression/ExprVisitorImpl.cpp create mode 100644 src/common/expression/ExprVisitorImpl.h create mode 100644 src/common/expression/Expression.cpp create mode 100644 src/common/expression/Expression.h create mode 100644 src/common/expression/FunctionCallExpression.cpp create mode 100644 src/common/expression/FunctionCallExpression.h create mode 100644 src/common/expression/LabelAttributeExpression.cpp create mode 100644 src/common/expression/LabelAttributeExpression.h create mode 100644 src/common/expression/LabelExpression.cpp create mode 100644 src/common/expression/LabelExpression.h create mode 100644 src/common/expression/ListComprehensionExpression.cpp create mode 100644 src/common/expression/ListComprehensionExpression.h create mode 100644 src/common/expression/LogicalExpression.cpp create mode 100644 src/common/expression/LogicalExpression.h create mode 100644 src/common/expression/MatchPathPatternExpression.cpp create mode 100644 src/common/expression/MatchPathPatternExpression.h create mode 100644 src/common/expression/PathBuildExpression.cpp create mode 100644 src/common/expression/PathBuildExpression.h create mode 100644 src/common/expression/PredicateExpression.cpp create mode 100644 src/common/expression/PredicateExpression.h create mode 100644 src/common/expression/PropertyExpression.cpp create mode 100644 src/common/expression/PropertyExpression.h create mode 100644 src/common/expression/ReduceExpression.cpp create mode 100644 src/common/expression/ReduceExpression.h create mode 100644 src/common/expression/RelationalExpression.cpp create mode 100644 src/common/expression/RelationalExpression.h create mode 100644 src/common/expression/SubscriptExpression.cpp create mode 100644 src/common/expression/SubscriptExpression.h create mode 100644 src/common/expression/TextSearchExpression.cpp create mode 100644 src/common/expression/TextSearchExpression.h create mode 100644 src/common/expression/TypeCastingExpression.cpp create mode 100644 src/common/expression/TypeCastingExpression.h create mode 100644 src/common/expression/UUIDExpression.cpp create mode 100644 src/common/expression/UUIDExpression.h create mode 100644 src/common/expression/UnaryExpression.cpp create mode 100644 src/common/expression/UnaryExpression.h create mode 100644 src/common/expression/VariableExpression.cpp create mode 100644 src/common/expression/VariableExpression.h create mode 100644 src/common/expression/VertexExpression.cpp create mode 100644 src/common/expression/VertexExpression.h create mode 100644 src/common/expression/test/AggregateExpressionBenchmark.cpp create mode 100644 src/common/expression/test/AggregateExpressionTest.cpp create mode 100644 src/common/expression/test/ArithmeticExpressionTest.cpp create mode 100644 src/common/expression/test/AttributeExpressionTest.cpp create mode 100644 src/common/expression/test/CMakeLists.txt create mode 100644 src/common/expression/test/CaseExpressionTest.cpp create mode 100644 src/common/expression/test/ColumnExpressionTest.cpp create mode 100644 src/common/expression/test/ConstantExpressionTest.cpp create mode 100644 src/common/expression/test/ContainerExpressionTest.cpp create mode 100644 src/common/expression/test/EncodeDecodeTest.cpp create mode 100644 src/common/expression/test/ExpressionBenchmark.cpp create mode 100644 src/common/expression/test/ExpressionContextMock.cpp create mode 100644 src/common/expression/test/ExpressionContextMock.h create mode 100644 src/common/expression/test/ExpressionTest.cpp create mode 100644 src/common/expression/test/FunctionCallExpressionBenchmark.cpp create mode 100644 src/common/expression/test/FunctionCallExpressionTest.cpp create mode 100644 src/common/expression/test/LabelExpressionTest.cpp create mode 100644 src/common/expression/test/ListComprehensionExpressionTest.cpp create mode 100644 src/common/expression/test/LogicalExpressionTest.cpp create mode 100644 src/common/expression/test/PathBuildExpressionTest.cpp create mode 100644 src/common/expression/test/PredicateExpressionTest.cpp create mode 100644 src/common/expression/test/PropertyExpressionTest.cpp create mode 100644 src/common/expression/test/ReduceExpressionTest.cpp create mode 100644 src/common/expression/test/RelationalExpressionTest.cpp create mode 100644 src/common/expression/test/SubscriptExpressionTest.cpp create mode 100644 src/common/expression/test/TestBase.cpp create mode 100644 src/common/expression/test/TestBase.h create mode 100644 src/common/expression/test/TypeCastingExpressionTest.cpp create mode 100644 src/common/expression/test/UnaryExpressionTest.cpp create mode 100644 src/common/expression/test/VersionedVariableExpressionTest.cpp create mode 100644 src/common/fs/CMakeLists.txt create mode 100644 src/common/fs/FileUtils.cpp create mode 100644 src/common/fs/FileUtils.h create mode 100644 src/common/fs/TempDir.cpp create mode 100644 src/common/fs/TempDir.h create mode 100644 src/common/fs/TempFile.cpp create mode 100644 src/common/fs/TempFile.h create mode 100644 src/common/fs/test/CMakeLists.txt create mode 100644 src/common/fs/test/FileUtilsTest.cpp create mode 100644 src/common/fs/test/TempDirTest.cpp create mode 100644 src/common/fs/test/TempFileTest.cpp create mode 100644 src/common/function/AggFunctionManager.cpp create mode 100644 src/common/function/AggFunctionManager.h create mode 100644 src/common/function/CMakeLists.txt create mode 100644 src/common/function/FunctionManager.cpp create mode 100644 src/common/function/FunctionManager.h create mode 100644 src/common/function/FunctionUdfManager.cpp create mode 100644 src/common/function/FunctionUdfManager.h create mode 100644 src/common/function/GraphFunction.h create mode 100644 src/common/function/test/AggFunctionManagerTest.cpp create mode 100644 src/common/function/test/CMakeLists.txt create mode 100644 src/common/function/test/FunctionManagerTest.cpp create mode 100644 src/common/function/test/TwiceTimezoneConversionTest.cpp create mode 100644 src/common/geo/CMakeLists.txt create mode 100644 src/common/geo/GeoFunction.cpp create mode 100644 src/common/geo/GeoFunction.h create mode 100644 src/common/geo/GeoIndex.cpp create mode 100644 src/common/geo/GeoIndex.h create mode 100644 src/common/geo/GeoUtils.h create mode 100644 src/common/geo/io/CMakeLists.txt create mode 100644 src/common/geo/io/wkb/ByteOrder.h create mode 100644 src/common/geo/io/wkb/ByteOrderDataIOStream.cpp create mode 100644 src/common/geo/io/wkb/ByteOrderDataIOStream.h create mode 100644 src/common/geo/io/wkb/CMakeLists.txt create mode 100644 src/common/geo/io/wkb/WKBReader.cpp create mode 100644 src/common/geo/io/wkb/WKBReader.h create mode 100644 src/common/geo/io/wkb/WKBWriter.cpp create mode 100644 src/common/geo/io/wkb/WKBWriter.h create mode 100644 src/common/geo/io/wkb/test/CMakeLists.txt create mode 100644 src/common/geo/io/wkb/test/WKBTest.cpp create mode 100644 src/common/geo/io/wkt/CMakeLists.txt create mode 100644 src/common/geo/io/wkt/WKTReader.cpp create mode 100644 src/common/geo/io/wkt/WKTReader.h create mode 100644 src/common/geo/io/wkt/WKTScanner.h create mode 100644 src/common/geo/io/wkt/WKTWriter.cpp create mode 100644 src/common/geo/io/wkt/WKTWriter.h create mode 100644 src/common/geo/io/wkt/test/CMakeLists.txt create mode 100644 src/common/geo/io/wkt/test/WKTTest.cpp create mode 100644 src/common/geo/io/wkt/wkt_parser.yy create mode 100644 src/common/geo/io/wkt/wkt_scanner.lex create mode 100644 src/common/geo/test/CMakeLists.txt create mode 100644 src/common/geo/test/GeoFunctionTest.cpp create mode 100644 src/common/geo/test/GeoIndexTest.cpp create mode 100644 src/common/graph/AuthResponseOps-inl.h create mode 100644 src/common/graph/CMakeLists.txt create mode 100644 src/common/graph/ExecutionResponseOps-inl.h create mode 100644 src/common/graph/GraphCpp2Ops.h create mode 100644 src/common/graph/PairOps-inl.h create mode 100644 src/common/graph/PlanDescriptionOps-inl.h create mode 100644 src/common/graph/PlanNodeBranchInfoOps-inl.h create mode 100644 src/common/graph/PlanNodeDescriptionOps-inl.h create mode 100644 src/common/graph/ProfilingStatsOps-inl.h create mode 100644 src/common/graph/Response.cpp create mode 100644 src/common/graph/Response.h create mode 100644 src/common/graph/tests/CMakeLists.txt create mode 100644 src/common/graph/tests/ResponseEncodeDecodeTest.cpp create mode 100644 src/common/hdfs/CMakeLists.txt create mode 100644 src/common/hdfs/HdfsCommandHelper.cpp create mode 100644 src/common/hdfs/HdfsCommandHelper.h create mode 100644 src/common/hdfs/HdfsHelper.h create mode 100644 src/common/http/CMakeLists.txt create mode 100644 src/common/http/HttpClient.cpp create mode 100644 src/common/http/HttpClient.h create mode 100644 src/common/http/test/CMakeLists.txt create mode 100644 src/common/http/test/HttpClientTest.cpp create mode 100644 src/common/id/CMakeLists.txt create mode 100644 src/common/id/SegmentId.cpp create mode 100644 src/common/id/SegmentId.h create mode 100644 src/common/id/Snowflake.cpp create mode 100644 src/common/id/Snowflake.h create mode 100644 src/common/id/UUID.cpp create mode 100644 src/common/id/UUID.h create mode 100644 src/common/id/test/CMakeLists.txt create mode 100644 src/common/id/test/MockMetaClient.h create mode 100644 src/common/id/test/SegmentIdBenchmark.cpp create mode 100644 src/common/id/test/SegmentIdTest.cpp create mode 100644 src/common/id/test/SnowflakeBenchmark.cpp create mode 100644 src/common/id/test/SnowflakeTest.cpp create mode 100644 src/common/log/CMakeLists.txt create mode 100644 src/common/log/LogMonitor.cpp create mode 100644 src/common/log/LogMonitor.h create mode 100644 src/common/memory/CMakeLists.txt create mode 100644 src/common/memory/Memory.h create mode 100644 src/common/memory/MemoryTracker.cpp create mode 100644 src/common/memory/MemoryTracker.h create mode 100644 src/common/memory/MemoryUtils.cpp create mode 100644 src/common/memory/MemoryUtils.h create mode 100644 src/common/memory/NewDelete.cpp create mode 100644 src/common/memory/test/CMakeLists.txt create mode 100644 src/common/memory/test/MemoryUtilsTest.cpp create mode 100644 src/common/memory/test/SysInfoReadBenchmark.cpp create mode 100644 src/common/meta/CMakeLists.txt create mode 100644 src/common/meta/ClusterIdManBase.h create mode 100644 src/common/meta/Common.h create mode 100644 src/common/meta/GflagsManager.cpp create mode 100644 src/common/meta/GflagsManager.h create mode 100644 src/common/meta/IndexManager.cpp create mode 100644 src/common/meta/IndexManager.h create mode 100644 src/common/meta/NebulaSchemaProvider.cpp create mode 100644 src/common/meta/NebulaSchemaProvider.h create mode 100644 src/common/meta/SchemaManager.cpp create mode 100644 src/common/meta/SchemaManager.h create mode 100644 src/common/meta/SchemaProviderIf.cpp create mode 100644 src/common/meta/SchemaProviderIf.h create mode 100644 src/common/meta/ServerBasedIndexManager.cpp create mode 100644 src/common/meta/ServerBasedIndexManager.h create mode 100644 src/common/meta/ServerBasedSchemaManager.cpp create mode 100644 src/common/meta/ServerBasedSchemaManager.h create mode 100644 src/common/network/CMakeLists.txt create mode 100644 src/common/network/NetworkUtils.cpp create mode 100644 src/common/network/NetworkUtils.h create mode 100644 src/common/network/test/CMakeLists.txt create mode 100644 src/common/network/test/NetworkUtilsBenchmark.cpp create mode 100644 src/common/network/test/NetworkUtilsTest.cpp create mode 100644 src/common/plugin/CMakeLists.txt create mode 100644 src/common/plugin/fulltext/CMakeLists.txt create mode 100644 src/common/plugin/fulltext/elasticsearch/ESAdapter.cpp create mode 100644 src/common/plugin/fulltext/elasticsearch/ESAdapter.h create mode 100644 src/common/plugin/fulltext/elasticsearch/ESClient.cpp create mode 100644 src/common/plugin/fulltext/elasticsearch/ESClient.h create mode 100644 src/common/plugin/fulltext/test/CMakeLists.txt create mode 100644 src/common/plugin/fulltext/test/ESQueryResultExample.h create mode 100644 src/common/plugin/fulltext/test/ElasticsearchTest.cpp create mode 100644 src/common/process/CMakeLists.txt create mode 100644 src/common/process/ProcessUtils.cpp create mode 100644 src/common/process/ProcessUtils.h create mode 100644 src/common/process/test/CMakeLists.txt create mode 100644 src/common/process/test/ProcessTest.cpp create mode 100644 src/common/session/SessionManager.h create mode 100644 src/common/ssl/CMakeLists.txt create mode 100644 src/common/ssl/SSLConfig.cpp create mode 100644 src/common/ssl/SSLConfig.h create mode 100644 src/common/stats/CMakeLists.txt create mode 100644 src/common/stats/StatsManager-inl.h create mode 100644 src/common/stats/StatsManager.cpp create mode 100644 src/common/stats/StatsManager.h create mode 100644 src/common/stats/test/CMakeLists.txt create mode 100644 src/common/stats/test/StatsManagerBenchmark.cpp create mode 100644 src/common/stats/test/StatsManagerCrossLevelTest.cpp create mode 100644 src/common/stats/test/StatsManagerRateTest.cpp create mode 100644 src/common/stats/test/StatsManagerTest.cpp create mode 100644 src/common/test/ServerContext.h create mode 100644 src/common/thread/CMakeLists.txt create mode 100644 src/common/thread/GenericThreadPool.cpp create mode 100644 src/common/thread/GenericThreadPool.h create mode 100644 src/common/thread/GenericWorker.cpp create mode 100644 src/common/thread/GenericWorker.h create mode 100644 src/common/thread/NamedThread.cpp create mode 100644 src/common/thread/NamedThread.h create mode 100644 src/common/thread/test/CMakeLists.txt create mode 100644 src/common/thread/test/GenericThreadPoolTest.cpp create mode 100644 src/common/thread/test/GenericWorkerTest.cpp create mode 100644 src/common/thread/test/ThreadTest.cpp create mode 100644 src/common/thrift/CMakeLists.txt create mode 100644 src/common/thrift/ThriftClientManager-inl.h create mode 100644 src/common/thrift/ThriftClientManager.cpp create mode 100644 src/common/thrift/ThriftClientManager.h create mode 100644 src/common/thrift/ThriftCpp2OpsHelper.h create mode 100644 src/common/thrift/ThriftLocalClientManager.h create mode 100644 src/common/thrift/ThriftTypes.h create mode 100644 src/common/time/CMakeLists.txt create mode 100644 src/common/time/Constants.h create mode 100644 src/common/time/Duration.cpp create mode 100644 src/common/time/Duration.h create mode 100644 src/common/time/ScopedTimer.h create mode 100644 src/common/time/TimeConversion.cpp create mode 100644 src/common/time/TimeConversion.h create mode 100644 src/common/time/TimeUtils.cpp create mode 100644 src/common/time/TimeUtils.h create mode 100644 src/common/time/TimezoneInfo.cpp create mode 100644 src/common/time/TimezoneInfo.h create mode 100644 src/common/time/WallClock.cpp create mode 100644 src/common/time/WallClock.h create mode 100644 src/common/time/detail/TscHelper.cpp create mode 100644 src/common/time/detail/TscHelper.h create mode 100644 src/common/time/parser/CMakeLists.txt create mode 100644 src/common/time/parser/DatetimeReader.cpp create mode 100644 src/common/time/parser/DatetimeReader.h create mode 100644 src/common/time/parser/DatetimeScanner.h create mode 100644 src/common/time/parser/Result.h create mode 100644 src/common/time/parser/datetime_parser.yy create mode 100644 src/common/time/parser/datetime_scanner.lex create mode 100644 src/common/time/parser/test/CMakeLists.txt create mode 100644 src/common/time/parser/test/DateTimeParserTest.cpp create mode 100644 src/common/time/test/CMakeLists.txt create mode 100644 src/common/time/test/DurationBenchmark.cpp create mode 100644 src/common/time/test/DurationTest.cpp create mode 100644 src/common/time/test/ScopedTimerTest.cpp create mode 100644 src/common/time/test/TimeUtilsTest.cpp create mode 100644 src/common/time/test/TimezoneInfoTest.cpp create mode 100644 src/common/time/test/WallClockBenchmark.cpp create mode 100644 src/common/time/test/WallClockTest.cpp create mode 100644 src/common/utils/CMakeLists.txt create mode 100644 src/common/utils/DefaultValueContext.h create mode 100644 src/common/utils/IndexKeyUtils.cpp create mode 100644 src/common/utils/IndexKeyUtils.h create mode 100644 src/common/utils/LogIterator.h create mode 100644 src/common/utils/LogStrListIterator.cpp create mode 100644 src/common/utils/LogStrListIterator.h create mode 100644 src/common/utils/MemoryLockCore.h create mode 100644 src/common/utils/MemoryLockWrapper.h create mode 100644 src/common/utils/MetaKeyUtils.cpp create mode 100644 src/common/utils/MetaKeyUtils.h create mode 100644 src/common/utils/NebulaKeyUtils.cpp create mode 100644 src/common/utils/NebulaKeyUtils.h create mode 100644 src/common/utils/OperationKeyUtils.cpp create mode 100644 src/common/utils/OperationKeyUtils.h create mode 100644 src/common/utils/Types.h create mode 100644 src/common/utils/Utils.h create mode 100644 src/common/utils/test/CMakeLists.txt create mode 100644 src/common/utils/test/IndexKeyUtilsTest.cpp create mode 100644 src/common/utils/test/MetaKeyUtilsTest.cpp create mode 100644 src/common/utils/test/NebulaKeyUtilsTest.cpp create mode 100644 src/common/utils/test/OperationKeyUtilsTest.cpp create mode 100644 src/console/CMakeLists.txt create mode 100644 src/console/compile-console.sh create mode 100644 src/daemons/CMakeLists.txt create mode 100644 src/daemons/GraphDaemon.cpp create mode 100644 src/daemons/MetaDaemon.cpp create mode 100644 src/daemons/MetaDaemonInit.cpp create mode 100644 src/daemons/MetaDaemonInit.h create mode 100644 src/daemons/SetupBreakpad.cpp create mode 100644 src/daemons/SetupLogging.cpp create mode 100644 src/daemons/SetupLogging.h create mode 100644 src/daemons/StandAloneDaemon.cpp create mode 100644 src/daemons/StorageDaemon.cpp create mode 100644 src/graph/CMakeLists.txt create mode 100644 src/graph/context/CMakeLists.txt create mode 100644 src/graph/context/ExecutionContext.cpp create mode 100644 src/graph/context/ExecutionContext.h create mode 100644 src/graph/context/Iterator.h create mode 100644 src/graph/context/QueryContext.cpp create mode 100644 src/graph/context/QueryContext.h create mode 100644 src/graph/context/QueryExpressionContext.cpp create mode 100644 src/graph/context/QueryExpressionContext.h create mode 100644 src/graph/context/Result.cpp create mode 100644 src/graph/context/Result.h create mode 100644 src/graph/context/Symbols.cpp create mode 100644 src/graph/context/Symbols.h create mode 100644 src/graph/context/ValidateContext.h create mode 100644 src/graph/context/ast/AstContext.h create mode 100644 src/graph/context/ast/CypherAstContext.h create mode 100644 src/graph/context/ast/QueryAstContext.h create mode 100644 src/graph/context/iterator/DefaultIter.h create mode 100644 src/graph/context/iterator/GetNbrsRespDataSetIter.cpp create mode 100644 src/graph/context/iterator/GetNbrsRespDataSetIter.h create mode 100644 src/graph/context/iterator/GetNeighborsIter.cpp create mode 100644 src/graph/context/iterator/GetNeighborsIter.h create mode 100644 src/graph/context/iterator/Iterator.cpp create mode 100644 src/graph/context/iterator/Iterator.h create mode 100644 src/graph/context/iterator/PropIter.cpp create mode 100644 src/graph/context/iterator/PropIter.h create mode 100644 src/graph/context/iterator/SequentialIter.cpp create mode 100644 src/graph/context/iterator/SequentialIter.h create mode 100644 src/graph/context/test/CMakeLists.txt create mode 100644 src/graph/context/test/ExecutionContextTest.cpp create mode 100644 src/graph/context/test/ExpressionContextTest.cpp create mode 100644 src/graph/context/test/InputVarPropBenchmark.cpp create mode 100644 src/graph/context/test/IteratorBenchmark.cpp create mode 100644 src/graph/context/test/IteratorTest.cpp create mode 100644 src/graph/executor/CMakeLists.txt create mode 100644 src/graph/executor/ExecutionError.h create mode 100644 src/graph/executor/Executor.cpp create mode 100644 src/graph/executor/Executor.h create mode 100644 src/graph/executor/StorageAccessExecutor.cpp create mode 100644 src/graph/executor/StorageAccessExecutor.h create mode 100644 src/graph/executor/admin/AddHostsExecutor.cpp create mode 100644 src/graph/executor/admin/AddHostsExecutor.h create mode 100644 src/graph/executor/admin/ChangePasswordExecutor.cpp create mode 100644 src/graph/executor/admin/ChangePasswordExecutor.h create mode 100644 src/graph/executor/admin/CharsetExecutor.cpp create mode 100644 src/graph/executor/admin/CharsetExecutor.h create mode 100644 src/graph/executor/admin/ConfigExecutor.cpp create mode 100644 src/graph/executor/admin/ConfigExecutor.h create mode 100644 src/graph/executor/admin/CreateUserExecutor.cpp create mode 100644 src/graph/executor/admin/CreateUserExecutor.h create mode 100644 src/graph/executor/admin/DescribeUserExecutor.cpp create mode 100644 src/graph/executor/admin/DescribeUserExecutor.h create mode 100644 src/graph/executor/admin/DropHostsExecutor.cpp create mode 100644 src/graph/executor/admin/DropHostsExecutor.h create mode 100644 src/graph/executor/admin/DropUserExecutor.cpp create mode 100644 src/graph/executor/admin/DropUserExecutor.h create mode 100644 src/graph/executor/admin/GrantRoleExecutor.cpp create mode 100644 src/graph/executor/admin/GrantRoleExecutor.h create mode 100644 src/graph/executor/admin/KillQueryExecutor.cpp create mode 100644 src/graph/executor/admin/KillQueryExecutor.h create mode 100644 src/graph/executor/admin/ListRolesExecutor.cpp create mode 100644 src/graph/executor/admin/ListRolesExecutor.h create mode 100644 src/graph/executor/admin/ListUserRolesExecutor.cpp create mode 100644 src/graph/executor/admin/ListUserRolesExecutor.h create mode 100644 src/graph/executor/admin/ListUsersExecutor.cpp create mode 100644 src/graph/executor/admin/ListUsersExecutor.h create mode 100644 src/graph/executor/admin/ListenerExecutor.cpp create mode 100644 src/graph/executor/admin/ListenerExecutor.h create mode 100644 src/graph/executor/admin/PartExecutor.cpp create mode 100644 src/graph/executor/admin/PartExecutor.h create mode 100644 src/graph/executor/admin/RevokeRoleExecutor.cpp create mode 100644 src/graph/executor/admin/RevokeRoleExecutor.h create mode 100644 src/graph/executor/admin/SessionExecutor.cpp create mode 100644 src/graph/executor/admin/SessionExecutor.h create mode 100644 src/graph/executor/admin/ShowHostsExecutor.cpp create mode 100644 src/graph/executor/admin/ShowHostsExecutor.h create mode 100644 src/graph/executor/admin/ShowMetaLeaderExecutor.cpp create mode 100644 src/graph/executor/admin/ShowMetaLeaderExecutor.h create mode 100644 src/graph/executor/admin/ShowQueriesExecutor.cpp create mode 100644 src/graph/executor/admin/ShowQueriesExecutor.h create mode 100644 src/graph/executor/admin/ShowServiceClientsExecutor.cpp create mode 100644 src/graph/executor/admin/ShowServiceClientsExecutor.h create mode 100644 src/graph/executor/admin/ShowStatsExecutor.cpp create mode 100644 src/graph/executor/admin/ShowStatsExecutor.h create mode 100644 src/graph/executor/admin/SignInServiceExecutor.cpp create mode 100644 src/graph/executor/admin/SignInServiceExecutor.h create mode 100644 src/graph/executor/admin/SignOutServiceExecutor.cpp create mode 100644 src/graph/executor/admin/SignOutServiceExecutor.h create mode 100644 src/graph/executor/admin/SnapshotExecutor.cpp create mode 100644 src/graph/executor/admin/SnapshotExecutor.h create mode 100644 src/graph/executor/admin/SpaceExecutor.cpp create mode 100644 src/graph/executor/admin/SpaceExecutor.h create mode 100644 src/graph/executor/admin/SubmitJobExecutor.cpp create mode 100644 src/graph/executor/admin/SubmitJobExecutor.h create mode 100644 src/graph/executor/admin/SwitchSpaceExecutor.cpp create mode 100644 src/graph/executor/admin/SwitchSpaceExecutor.h create mode 100644 src/graph/executor/admin/UpdateUserExecutor.cpp create mode 100644 src/graph/executor/admin/UpdateUserExecutor.h create mode 100644 src/graph/executor/admin/ZoneExecutor.cpp create mode 100644 src/graph/executor/admin/ZoneExecutor.h create mode 100644 src/graph/executor/algo/AllPathsExecutor.cpp create mode 100644 src/graph/executor/algo/AllPathsExecutor.h create mode 100644 src/graph/executor/algo/BFSShortestPathExecutor.cpp create mode 100644 src/graph/executor/algo/BFSShortestPathExecutor.h create mode 100644 src/graph/executor/algo/BatchShortestPath.cpp create mode 100644 src/graph/executor/algo/BatchShortestPath.h create mode 100644 src/graph/executor/algo/CartesianProductExecutor.cpp create mode 100644 src/graph/executor/algo/CartesianProductExecutor.h create mode 100644 src/graph/executor/algo/MultiShortestPathExecutor.cpp create mode 100644 src/graph/executor/algo/MultiShortestPathExecutor.h create mode 100644 src/graph/executor/algo/ShortestPathBase.cpp create mode 100644 src/graph/executor/algo/ShortestPathBase.h create mode 100644 src/graph/executor/algo/ShortestPathExecutor.cpp create mode 100644 src/graph/executor/algo/ShortestPathExecutor.h create mode 100644 src/graph/executor/algo/SingleShortestPath.cpp create mode 100644 src/graph/executor/algo/SingleShortestPath.h create mode 100644 src/graph/executor/algo/SubgraphExecutor.cpp create mode 100644 src/graph/executor/algo/SubgraphExecutor.h create mode 100644 src/graph/executor/logic/ArgumentExecutor.cpp create mode 100644 src/graph/executor/logic/ArgumentExecutor.h create mode 100644 src/graph/executor/logic/LoopExecutor.cpp create mode 100644 src/graph/executor/logic/LoopExecutor.h create mode 100644 src/graph/executor/logic/PassThroughExecutor.cpp create mode 100644 src/graph/executor/logic/PassThroughExecutor.h create mode 100644 src/graph/executor/logic/SelectExecutor.cpp create mode 100644 src/graph/executor/logic/SelectExecutor.h create mode 100644 src/graph/executor/logic/StartExecutor.cpp create mode 100644 src/graph/executor/logic/StartExecutor.h create mode 100644 src/graph/executor/maintain/EdgeExecutor.cpp create mode 100644 src/graph/executor/maintain/EdgeExecutor.h create mode 100644 src/graph/executor/maintain/EdgeIndexExecutor.cpp create mode 100644 src/graph/executor/maintain/EdgeIndexExecutor.h create mode 100644 src/graph/executor/maintain/FTIndexExecutor.cpp create mode 100644 src/graph/executor/maintain/FTIndexExecutor.h create mode 100644 src/graph/executor/maintain/TagExecutor.cpp create mode 100644 src/graph/executor/maintain/TagExecutor.h create mode 100644 src/graph/executor/maintain/TagIndexExecutor.cpp create mode 100644 src/graph/executor/maintain/TagIndexExecutor.h create mode 100644 src/graph/executor/mutate/DeleteExecutor.cpp create mode 100644 src/graph/executor/mutate/DeleteExecutor.h create mode 100644 src/graph/executor/mutate/InsertExecutor.cpp create mode 100644 src/graph/executor/mutate/InsertExecutor.h create mode 100644 src/graph/executor/mutate/UpdateExecutor.cpp create mode 100644 src/graph/executor/mutate/UpdateExecutor.h create mode 100644 src/graph/executor/query/AggregateExecutor.cpp create mode 100644 src/graph/executor/query/AggregateExecutor.h create mode 100644 src/graph/executor/query/AppendVerticesExecutor.cpp create mode 100644 src/graph/executor/query/AppendVerticesExecutor.h create mode 100644 src/graph/executor/query/AssignExecutor.cpp create mode 100644 src/graph/executor/query/AssignExecutor.h create mode 100644 src/graph/executor/query/DataCollectExecutor.cpp create mode 100644 src/graph/executor/query/DataCollectExecutor.h create mode 100644 src/graph/executor/query/DedupExecutor.cpp create mode 100644 src/graph/executor/query/DedupExecutor.h create mode 100644 src/graph/executor/query/ExpandAllExecutor.cpp create mode 100644 src/graph/executor/query/ExpandAllExecutor.h create mode 100644 src/graph/executor/query/ExpandExecutor.cpp create mode 100644 src/graph/executor/query/ExpandExecutor.h create mode 100644 src/graph/executor/query/FilterExecutor.cpp create mode 100644 src/graph/executor/query/FilterExecutor.h create mode 100644 src/graph/executor/query/FulltextIndexScanExecutor.cpp create mode 100644 src/graph/executor/query/FulltextIndexScanExecutor.h create mode 100644 src/graph/executor/query/GetEdgesExecutor.cpp create mode 100644 src/graph/executor/query/GetEdgesExecutor.h create mode 100644 src/graph/executor/query/GetNeighborsExecutor.cpp create mode 100644 src/graph/executor/query/GetNeighborsExecutor.h create mode 100644 src/graph/executor/query/GetPropExecutor.h create mode 100644 src/graph/executor/query/GetVerticesExecutor.cpp create mode 100644 src/graph/executor/query/GetVerticesExecutor.h create mode 100644 src/graph/executor/query/IndexScanExecutor.cpp create mode 100644 src/graph/executor/query/IndexScanExecutor.h create mode 100644 src/graph/executor/query/InnerJoinExecutor.cpp create mode 100644 src/graph/executor/query/InnerJoinExecutor.h create mode 100644 src/graph/executor/query/IntersectExecutor.cpp create mode 100644 src/graph/executor/query/IntersectExecutor.h create mode 100644 src/graph/executor/query/JoinExecutor.cpp create mode 100644 src/graph/executor/query/JoinExecutor.h create mode 100644 src/graph/executor/query/LeftJoinExecutor.cpp create mode 100644 src/graph/executor/query/LeftJoinExecutor.h create mode 100644 src/graph/executor/query/LimitExecutor.cpp create mode 100644 src/graph/executor/query/LimitExecutor.h create mode 100644 src/graph/executor/query/MinusExecutor.cpp create mode 100644 src/graph/executor/query/MinusExecutor.h create mode 100644 src/graph/executor/query/PatternApplyExecutor.cpp create mode 100644 src/graph/executor/query/PatternApplyExecutor.h create mode 100644 src/graph/executor/query/ProjectExecutor.cpp create mode 100644 src/graph/executor/query/ProjectExecutor.h create mode 100644 src/graph/executor/query/RollUpApplyExecutor.cpp create mode 100644 src/graph/executor/query/RollUpApplyExecutor.h create mode 100644 src/graph/executor/query/SampleExecutor.cpp create mode 100644 src/graph/executor/query/SampleExecutor.h create mode 100644 src/graph/executor/query/ScanEdgesExecutor.cpp create mode 100644 src/graph/executor/query/ScanEdgesExecutor.h create mode 100644 src/graph/executor/query/ScanVerticesExecutor.cpp create mode 100644 src/graph/executor/query/ScanVerticesExecutor.h create mode 100644 src/graph/executor/query/SetExecutor.cpp create mode 100644 src/graph/executor/query/SetExecutor.h create mode 100644 src/graph/executor/query/SortExecutor.cpp create mode 100644 src/graph/executor/query/SortExecutor.h create mode 100644 src/graph/executor/query/TopNExecutor.cpp create mode 100644 src/graph/executor/query/TopNExecutor.h create mode 100644 src/graph/executor/query/TraverseExecutor.cpp create mode 100644 src/graph/executor/query/TraverseExecutor.h create mode 100644 src/graph/executor/query/UnionAllVersionVarExecutor.cpp create mode 100644 src/graph/executor/query/UnionAllVersionVarExecutor.h create mode 100644 src/graph/executor/query/UnionExecutor.cpp create mode 100644 src/graph/executor/query/UnionExecutor.h create mode 100644 src/graph/executor/query/UnwindExecutor.cpp create mode 100644 src/graph/executor/query/UnwindExecutor.h create mode 100644 src/graph/executor/query/ValueExecutor.cpp create mode 100644 src/graph/executor/query/ValueExecutor.h create mode 100644 src/graph/executor/test/AggregateTest.cpp create mode 100644 src/graph/executor/test/AssignTest.cpp create mode 100644 src/graph/executor/test/CMakeLists.txt create mode 100644 src/graph/executor/test/CartesianProductTest.cpp create mode 100644 src/graph/executor/test/DataCollectTest.cpp create mode 100644 src/graph/executor/test/DedupTest.cpp create mode 100644 src/graph/executor/test/FilterTest.cpp create mode 100644 src/graph/executor/test/FindPathTest.cpp create mode 100644 src/graph/executor/test/GetNeighborsTest.cpp create mode 100644 src/graph/executor/test/JobTest.cpp create mode 100644 src/graph/executor/test/JoinTest.cpp create mode 100644 src/graph/executor/test/LimitTest.cpp create mode 100644 src/graph/executor/test/LogicExecutorsTest.cpp create mode 100644 src/graph/executor/test/ProjectTest.cpp create mode 100644 src/graph/executor/test/QueryTestBase.h create mode 100644 src/graph/executor/test/SampleTest.cpp create mode 100644 src/graph/executor/test/SetExecutorTest.cpp create mode 100644 src/graph/executor/test/ShowQueriesTest.cpp create mode 100644 src/graph/executor/test/SortTest.cpp create mode 100644 src/graph/executor/test/StorageServerStub.cpp create mode 100644 src/graph/executor/test/TestMain.cpp create mode 100644 src/graph/executor/test/TopNTest.cpp create mode 100644 src/graph/executor/test/UnwindTest.cpp create mode 100644 src/graph/gc/CMakeLists.txt create mode 100644 src/graph/gc/GC.cpp create mode 100644 src/graph/gc/GC.h create mode 100644 src/graph/optimizer/CMakeLists.txt create mode 100644 src/graph/optimizer/OptContext.cpp create mode 100644 src/graph/optimizer/OptContext.h create mode 100644 src/graph/optimizer/OptGroup.cpp create mode 100644 src/graph/optimizer/OptGroup.h create mode 100644 src/graph/optimizer/OptRule.cpp create mode 100644 src/graph/optimizer/OptRule.h create mode 100644 src/graph/optimizer/Optimizer.cpp create mode 100644 src/graph/optimizer/Optimizer.h create mode 100644 src/graph/optimizer/rule/CollapseProjectRule.cpp create mode 100644 src/graph/optimizer/rule/CollapseProjectRule.h create mode 100644 src/graph/optimizer/rule/CombineFilterRule.cpp create mode 100644 src/graph/optimizer/rule/CombineFilterRule.h create mode 100644 src/graph/optimizer/rule/EdgeIndexFullScanRule.cpp create mode 100644 src/graph/optimizer/rule/EdgeIndexFullScanRule.h create mode 100644 src/graph/optimizer/rule/EliminateAppendVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/EliminateAppendVerticesRule.h create mode 100644 src/graph/optimizer/rule/EliminateFilterRule.cpp create mode 100644 src/graph/optimizer/rule/EliminateFilterRule.h create mode 100644 src/graph/optimizer/rule/EliminateRowCollectRule.cpp create mode 100644 src/graph/optimizer/rule/EliminateRowCollectRule.h create mode 100644 src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp create mode 100644 src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h create mode 100644 src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/GeoPredicateEdgeIndexScanRule.h create mode 100644 src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp create mode 100644 src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.h create mode 100644 src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/GeoPredicateTagIndexScanRule.h create mode 100644 src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp create mode 100644 src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.h create mode 100644 src/graph/optimizer/rule/GetEdgesTransformRule.cpp create mode 100644 src/graph/optimizer/rule/GetEdgesTransformRule.h create mode 100644 src/graph/optimizer/rule/GetEdgesTransformUtils.h create mode 100644 src/graph/optimizer/rule/IndexFullScanBaseRule.cpp create mode 100644 src/graph/optimizer/rule/IndexFullScanBaseRule.h create mode 100644 src/graph/optimizer/rule/IndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/IndexScanRule.h create mode 100644 src/graph/optimizer/rule/MergeGetNbrsAndDedupRule.cpp create mode 100644 src/graph/optimizer/rule/MergeGetNbrsAndDedupRule.h create mode 100644 src/graph/optimizer/rule/MergeGetNbrsAndProjectRule.cpp create mode 100644 src/graph/optimizer/rule/MergeGetNbrsAndProjectRule.h create mode 100644 src/graph/optimizer/rule/MergeGetVerticesAndDedupRule.cpp create mode 100644 src/graph/optimizer/rule/MergeGetVerticesAndDedupRule.h create mode 100644 src/graph/optimizer/rule/MergeGetVerticesAndProjectRule.cpp create mode 100644 src/graph/optimizer/rule/MergeGetVerticesAndProjectRule.h create mode 100644 src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/MergeLimitAndFulltextIndexScanRule.h create mode 100644 src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp create mode 100644 src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.h create mode 100644 src/graph/optimizer/rule/OptimizeTagIndexScanByFilterRule.cpp create mode 100644 src/graph/optimizer/rule/OptimizeTagIndexScanByFilterRule.h create mode 100644 src/graph/optimizer/rule/PushEFilterDownRule.cpp create mode 100644 src/graph/optimizer/rule/PushEFilterDownRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownAggregateRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownAggregateRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownAllPathsRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownAllPathsRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownCrossJoinRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownCrossJoinRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownExpandAllRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownExpandAllRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownGetNbrsRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownGetNbrsRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownHashInnerJoinRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownHashInnerJoinRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownHashLeftJoinRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownHashLeftJoinRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownInnerJoinRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownInnerJoinRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownNodeRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownNodeRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownProjectRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownProjectRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownScanVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownScanVerticesRule.h create mode 100644 src/graph/optimizer/rule/PushFilterDownTraverseRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterDownTraverseRule.h create mode 100644 src/graph/optimizer/rule/PushFilterThroughAppendVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushFilterThroughAppendVerticesRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownAllPathsRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownAllPathsRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownExpandAllRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownExpandAllRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownFulltextIndexScanRule2.h create mode 100644 src/graph/optimizer/rule/PushLimitDownGetEdgesRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownGetEdgesRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownGetNeighborsRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownGetNeighborsRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownGetVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownGetVerticesRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownIndexScanRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownProjectRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownProjectRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownScanAppendVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownScanAppendVerticesRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownScanEdgesAppendVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownScanEdgesAppendVerticesRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownScanEdgesRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownScanEdgesRule.h create mode 100644 src/graph/optimizer/rule/PushStepLimitDownGetNeighborsRule.cpp create mode 100644 src/graph/optimizer/rule/PushStepLimitDownGetNeighborsRule.h create mode 100644 src/graph/optimizer/rule/PushStepSampleDownGetNeighborsRule.cpp create mode 100644 src/graph/optimizer/rule/PushStepSampleDownGetNeighborsRule.h create mode 100644 src/graph/optimizer/rule/PushTopNDownIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/PushTopNDownIndexScanRule.h create mode 100644 src/graph/optimizer/rule/PushVFilterDownScanVerticesRule.cpp create mode 100644 src/graph/optimizer/rule/PushVFilterDownScanVerticesRule.h create mode 100644 src/graph/optimizer/rule/RemoveAppendVerticesBelowJoinRule.cpp create mode 100644 src/graph/optimizer/rule/RemoveAppendVerticesBelowJoinRule.h create mode 100644 src/graph/optimizer/rule/RemoveNoopProjectRule.cpp create mode 100644 src/graph/optimizer/rule/RemoveNoopProjectRule.h create mode 100644 src/graph/optimizer/rule/TagIndexFullScanRule.cpp create mode 100644 src/graph/optimizer/rule/TagIndexFullScanRule.h create mode 100644 src/graph/optimizer/rule/TopNRule.cpp create mode 100644 src/graph/optimizer/rule/TopNRule.h create mode 100644 src/graph/optimizer/rule/UnionAllEdgeIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/UnionAllEdgeIndexScanRule.h create mode 100644 src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp create mode 100644 src/graph/optimizer/rule/UnionAllIndexScanBaseRule.h create mode 100644 src/graph/optimizer/rule/UnionAllTagIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/UnionAllTagIndexScanRule.h create mode 100644 src/graph/optimizer/test/CMakeLists.txt create mode 100644 src/graph/optimizer/test/IndexScanRuleTest.cpp create mode 100644 src/graph/planner/CMakeLists.txt create mode 100644 src/graph/planner/Planner.cpp create mode 100644 src/graph/planner/Planner.h create mode 100644 src/graph/planner/PlannersRegister.cpp create mode 100644 src/graph/planner/PlannersRegister.h create mode 100644 src/graph/planner/SequentialPlanner.cpp create mode 100644 src/graph/planner/SequentialPlanner.h create mode 100644 src/graph/planner/match/ArgumentFinder.cpp create mode 100644 src/graph/planner/match/ArgumentFinder.h create mode 100644 src/graph/planner/match/CypherClausePlanner.h create mode 100644 src/graph/planner/match/EdgeIndexSeek.h create mode 100644 src/graph/planner/match/LabelIndexSeek.cpp create mode 100644 src/graph/planner/match/LabelIndexSeek.h create mode 100644 src/graph/planner/match/MatchClausePlanner.cpp create mode 100644 src/graph/planner/match/MatchClausePlanner.h create mode 100644 src/graph/planner/match/MatchPathPlanner.cpp create mode 100644 src/graph/planner/match/MatchPathPlanner.h create mode 100644 src/graph/planner/match/MatchPlanner.cpp create mode 100644 src/graph/planner/match/MatchPlanner.h create mode 100644 src/graph/planner/match/MatchSolver.cpp create mode 100644 src/graph/planner/match/MatchSolver.h create mode 100644 src/graph/planner/match/OrderByClausePlanner.cpp create mode 100644 src/graph/planner/match/OrderByClausePlanner.h create mode 100644 src/graph/planner/match/PaginationPlanner.cpp create mode 100644 src/graph/planner/match/PaginationPlanner.h create mode 100644 src/graph/planner/match/PropIndexSeek.cpp create mode 100644 src/graph/planner/match/PropIndexSeek.h create mode 100644 src/graph/planner/match/ReturnClausePlanner.cpp create mode 100644 src/graph/planner/match/ReturnClausePlanner.h create mode 100644 src/graph/planner/match/ScanSeek.cpp create mode 100644 src/graph/planner/match/ScanSeek.h create mode 100644 src/graph/planner/match/SegmentsConnector.cpp create mode 100644 src/graph/planner/match/SegmentsConnector.h create mode 100644 src/graph/planner/match/ShortestPathPlanner.cpp create mode 100644 src/graph/planner/match/ShortestPathPlanner.h create mode 100644 src/graph/planner/match/StartVidFinder.cpp create mode 100644 src/graph/planner/match/StartVidFinder.h create mode 100644 src/graph/planner/match/UnwindClausePlanner.cpp create mode 100644 src/graph/planner/match/UnwindClausePlanner.h create mode 100644 src/graph/planner/match/VariablePropIndexSeek.cpp create mode 100644 src/graph/planner/match/VariablePropIndexSeek.h create mode 100644 src/graph/planner/match/VariableVertexIdSeek.cpp create mode 100644 src/graph/planner/match/VariableVertexIdSeek.h create mode 100644 src/graph/planner/match/VertexIdSeek.cpp create mode 100644 src/graph/planner/match/VertexIdSeek.h create mode 100644 src/graph/planner/match/WhereClausePlanner.cpp create mode 100644 src/graph/planner/match/WhereClausePlanner.h create mode 100644 src/graph/planner/match/WithClausePlanner.cpp create mode 100644 src/graph/planner/match/WithClausePlanner.h create mode 100644 src/graph/planner/match/YieldClausePlanner.cpp create mode 100644 src/graph/planner/match/YieldClausePlanner.h create mode 100644 src/graph/planner/ngql/FetchEdgesPlanner.cpp create mode 100644 src/graph/planner/ngql/FetchEdgesPlanner.h create mode 100644 src/graph/planner/ngql/FetchVerticesPlanner.cpp create mode 100644 src/graph/planner/ngql/FetchVerticesPlanner.h create mode 100644 src/graph/planner/ngql/GoPlanner.cpp create mode 100644 src/graph/planner/ngql/GoPlanner.h create mode 100644 src/graph/planner/ngql/LookupPlanner.cpp create mode 100644 src/graph/planner/ngql/LookupPlanner.h create mode 100644 src/graph/planner/ngql/MaintainPlanner.cpp create mode 100644 src/graph/planner/ngql/MaintainPlanner.h create mode 100644 src/graph/planner/ngql/PathPlanner.cpp create mode 100644 src/graph/planner/ngql/PathPlanner.h create mode 100644 src/graph/planner/ngql/SubgraphPlanner.cpp create mode 100644 src/graph/planner/ngql/SubgraphPlanner.h create mode 100644 src/graph/planner/plan/Admin.cpp create mode 100644 src/graph/planner/plan/Admin.h create mode 100644 src/graph/planner/plan/Algo.cpp create mode 100644 src/graph/planner/plan/Algo.h create mode 100644 src/graph/planner/plan/ExecutionPlan.cpp create mode 100644 src/graph/planner/plan/ExecutionPlan.h create mode 100644 src/graph/planner/plan/Logic.cpp create mode 100644 src/graph/planner/plan/Logic.h create mode 100644 src/graph/planner/plan/Maintain.cpp create mode 100644 src/graph/planner/plan/Maintain.h create mode 100644 src/graph/planner/plan/Mutate.cpp create mode 100644 src/graph/planner/plan/Mutate.h create mode 100644 src/graph/planner/plan/PlanNode.cpp create mode 100644 src/graph/planner/plan/PlanNode.h create mode 100644 src/graph/planner/plan/PlanNodeVisitor.h create mode 100644 src/graph/planner/plan/Query.cpp create mode 100644 src/graph/planner/plan/Query.h create mode 100644 src/graph/planner/plan/Scan.h create mode 100644 src/graph/planner/test/CMakeLists.txt create mode 100644 src/graph/planner/test/ExecutionPlanTest.cpp create mode 100644 src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp create mode 100644 src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h create mode 100644 src/graph/scheduler/CMakeLists.txt create mode 100644 src/graph/scheduler/Scheduler.cpp create mode 100644 src/graph/scheduler/Scheduler.h create mode 100644 src/graph/service/Authenticator.h create mode 100644 src/graph/service/CMakeLists.txt create mode 100644 src/graph/service/CloudAuthenticator.cpp create mode 100644 src/graph/service/CloudAuthenticator.h create mode 100644 src/graph/service/GraphFlags.cpp create mode 100644 src/graph/service/GraphFlags.h create mode 100644 src/graph/service/GraphServer.cpp create mode 100644 src/graph/service/GraphServer.h create mode 100644 src/graph/service/GraphService.cpp create mode 100644 src/graph/service/GraphService.h create mode 100644 src/graph/service/PasswordAuthenticator.cpp create mode 100644 src/graph/service/PasswordAuthenticator.h create mode 100644 src/graph/service/PermissionCheck.cpp create mode 100644 src/graph/service/PermissionCheck.h create mode 100644 src/graph/service/PermissionManager.cpp create mode 100644 src/graph/service/PermissionManager.h create mode 100644 src/graph/service/QueryEngine.cpp create mode 100644 src/graph/service/QueryEngine.h create mode 100644 src/graph/service/QueryInstance.cpp create mode 100644 src/graph/service/QueryInstance.h create mode 100644 src/graph/service/RequestContext.h create mode 100644 src/graph/service/test/CMakeLists.txt create mode 100644 src/graph/service/test/StandAloneTestGraphFlags.cpp create mode 100644 src/graph/session/CMakeLists.txt create mode 100644 src/graph/session/ClientSession.cpp create mode 100644 src/graph/session/ClientSession.h create mode 100644 src/graph/session/GraphSessionManager.cpp create mode 100644 src/graph/session/GraphSessionManager.h create mode 100644 src/graph/stats/CMakeLists.txt create mode 100644 src/graph/stats/GraphStats.cpp create mode 100644 src/graph/stats/GraphStats.h create mode 100644 src/graph/util/AnonColGenerator.h create mode 100644 src/graph/util/AnonVarGenerator.h create mode 100644 src/graph/util/AstUtils.h create mode 100644 src/graph/util/CMakeLists.txt create mode 100644 src/graph/util/Constants.cpp create mode 100644 src/graph/util/Constants.h create mode 100644 src/graph/util/ExpressionUtils.cpp create mode 100644 src/graph/util/ExpressionUtils.h create mode 100644 src/graph/util/FTIndexUtils.cpp create mode 100644 src/graph/util/FTIndexUtils.h create mode 100644 src/graph/util/IdGenerator.cpp create mode 100644 src/graph/util/IdGenerator.h create mode 100644 src/graph/util/IndexUtil.cpp create mode 100644 src/graph/util/IndexUtil.h create mode 100644 src/graph/util/OptimizerUtils.cpp create mode 100644 src/graph/util/OptimizerUtils.h create mode 100644 src/graph/util/ParserUtil.cpp create mode 100644 src/graph/util/ParserUtil.h create mode 100644 src/graph/util/PlannerUtil.cpp create mode 100644 src/graph/util/PlannerUtil.h create mode 100644 src/graph/util/SchemaUtil.cpp create mode 100644 src/graph/util/SchemaUtil.h create mode 100644 src/graph/util/ToJson.cpp create mode 100644 src/graph/util/ToJson.h create mode 100644 src/graph/util/Utils.cpp create mode 100644 src/graph/util/Utils.h create mode 100644 src/graph/util/ValidateUtil.cpp create mode 100644 src/graph/util/ValidateUtil.h create mode 100644 src/graph/util/ZoneUtil.cpp create mode 100644 src/graph/util/ZoneUtil.h create mode 100644 src/graph/util/test/CMakeLists.txt create mode 100644 src/graph/util/test/ExpressionUtilsTest.cpp create mode 100644 src/graph/util/test/IdGeneratorTest.cpp create mode 100644 src/graph/validator/ACLValidator.cpp create mode 100644 src/graph/validator/ACLValidator.h create mode 100644 src/graph/validator/AdminJobValidator.cpp create mode 100644 src/graph/validator/AdminJobValidator.h create mode 100644 src/graph/validator/AdminValidator.cpp create mode 100644 src/graph/validator/AdminValidator.h create mode 100644 src/graph/validator/AssignmentValidator.cpp create mode 100644 src/graph/validator/AssignmentValidator.h create mode 100644 src/graph/validator/CMakeLists.txt create mode 100644 src/graph/validator/ExplainValidator.cpp create mode 100644 src/graph/validator/ExplainValidator.h create mode 100644 src/graph/validator/FetchEdgesValidator.cpp create mode 100644 src/graph/validator/FetchEdgesValidator.h create mode 100644 src/graph/validator/FetchVerticesValidator.cpp create mode 100644 src/graph/validator/FetchVerticesValidator.h create mode 100644 src/graph/validator/FindPathValidator.cpp create mode 100644 src/graph/validator/FindPathValidator.h create mode 100644 src/graph/validator/GetSubgraphValidator.cpp create mode 100644 src/graph/validator/GetSubgraphValidator.h create mode 100644 src/graph/validator/GoValidator.cpp create mode 100644 src/graph/validator/GoValidator.h create mode 100644 src/graph/validator/GroupByValidator.cpp create mode 100644 src/graph/validator/GroupByValidator.h create mode 100644 src/graph/validator/LimitValidator.cpp create mode 100644 src/graph/validator/LimitValidator.h create mode 100644 src/graph/validator/LookupValidator.cpp create mode 100644 src/graph/validator/LookupValidator.h create mode 100644 src/graph/validator/MaintainValidator.cpp create mode 100644 src/graph/validator/MaintainValidator.h create mode 100644 src/graph/validator/MatchValidator.cpp create mode 100644 src/graph/validator/MatchValidator.h create mode 100644 src/graph/validator/MutateValidator.cpp create mode 100644 src/graph/validator/MutateValidator.h create mode 100644 src/graph/validator/OrderByValidator.cpp create mode 100644 src/graph/validator/OrderByValidator.h create mode 100644 src/graph/validator/PipeValidator.cpp create mode 100644 src/graph/validator/PipeValidator.h create mode 100644 src/graph/validator/ReportError.h create mode 100644 src/graph/validator/SequentialValidator.cpp create mode 100644 src/graph/validator/SequentialValidator.h create mode 100644 src/graph/validator/SetValidator.cpp create mode 100644 src/graph/validator/SetValidator.h create mode 100644 src/graph/validator/UnwindValidator.cpp create mode 100644 src/graph/validator/UnwindValidator.h create mode 100644 src/graph/validator/UseValidator.cpp create mode 100644 src/graph/validator/UseValidator.h create mode 100644 src/graph/validator/Validator.cpp create mode 100644 src/graph/validator/Validator.h create mode 100644 src/graph/validator/YieldValidator.cpp create mode 100644 src/graph/validator/YieldValidator.h create mode 100644 src/graph/validator/test/ACLValidatorTest.cpp create mode 100644 src/graph/validator/test/AdminValidatorTest.cpp create mode 100644 src/graph/validator/test/CMakeLists.txt create mode 100644 src/graph/validator/test/ExplainValidatorTest.cpp create mode 100644 src/graph/validator/test/FetchEdgesTest.cpp create mode 100644 src/graph/validator/test/FetchVerticesTest.cpp create mode 100644 src/graph/validator/test/FindPathValidatorTest.cpp create mode 100644 src/graph/validator/test/GetSubgraphValidatorTest.cpp create mode 100644 src/graph/validator/test/GroupByValidatorTest.cpp create mode 100644 src/graph/validator/test/LookupValidatorTest.cpp create mode 100644 src/graph/validator/test/MaintainValidatorTest.cpp create mode 100644 src/graph/validator/test/MatchValidatorTest.cpp create mode 100644 src/graph/validator/test/MockIndexManager.cpp create mode 100644 src/graph/validator/test/MockIndexManager.h create mode 100644 src/graph/validator/test/MockSchemaManager.cpp create mode 100644 src/graph/validator/test/MockSchemaManager.h create mode 100644 src/graph/validator/test/MockSchemaManagerTest.cpp create mode 100644 src/graph/validator/test/MutateValidatorTest.cpp create mode 100644 src/graph/validator/test/QueryValidatorTest.cpp create mode 100644 src/graph/validator/test/SymbolsTest.cpp create mode 100644 src/graph/validator/test/ValidatorTestBase.cpp create mode 100644 src/graph/validator/test/ValidatorTestBase.h create mode 100644 src/graph/validator/test/YieldValidatorTest.cpp create mode 100644 src/graph/visitor/CMakeLists.txt create mode 100644 src/graph/visitor/DeduceAliasTypeVisitor.cpp create mode 100644 src/graph/visitor/DeduceAliasTypeVisitor.h create mode 100644 src/graph/visitor/DeducePropsVisitor.cpp create mode 100644 src/graph/visitor/DeducePropsVisitor.h create mode 100644 src/graph/visitor/DeduceTypeVisitor.cpp create mode 100644 src/graph/visitor/DeduceTypeVisitor.h create mode 100644 src/graph/visitor/EvaluableExprVisitor.cpp create mode 100644 src/graph/visitor/EvaluableExprVisitor.h create mode 100644 src/graph/visitor/ExtractFilterExprVisitor.cpp create mode 100644 src/graph/visitor/ExtractFilterExprVisitor.h create mode 100644 src/graph/visitor/ExtractGroupSuiteVisitor.cpp create mode 100644 src/graph/visitor/ExtractGroupSuiteVisitor.h create mode 100644 src/graph/visitor/ExtractPropExprVisitor.cpp create mode 100644 src/graph/visitor/ExtractPropExprVisitor.h create mode 100644 src/graph/visitor/FindVisitor.cpp create mode 100644 src/graph/visitor/FindVisitor.h create mode 100644 src/graph/visitor/FoldConstantExprVisitor.cpp create mode 100644 src/graph/visitor/FoldConstantExprVisitor.h create mode 100644 src/graph/visitor/PropertyTrackerVisitor.cpp create mode 100644 src/graph/visitor/PropertyTrackerVisitor.h create mode 100644 src/graph/visitor/PrunePropertiesVisitor.cpp create mode 100644 src/graph/visitor/PrunePropertiesVisitor.h create mode 100644 src/graph/visitor/RewriteVisitor.cpp create mode 100644 src/graph/visitor/RewriteVisitor.h create mode 100644 src/graph/visitor/ValidatePatternExpressionVisitor.cpp create mode 100644 src/graph/visitor/ValidatePatternExpressionVisitor.h create mode 100644 src/graph/visitor/VidExtractVisitor.cpp create mode 100644 src/graph/visitor/VidExtractVisitor.h create mode 100644 src/graph/visitor/test/CMakeLists.txt create mode 100644 src/graph/visitor/test/DeduceAliasTypeVisitorTest.cpp create mode 100644 src/graph/visitor/test/DeduceTypeVisitorTest.cpp create mode 100644 src/graph/visitor/test/ExtractFilterExprVisitorTest.cpp create mode 100644 src/graph/visitor/test/ExtractGroupSuiteVisitorTest.cpp create mode 100644 src/graph/visitor/test/FilterTransformTest.cpp create mode 100644 src/graph/visitor/test/FoldConstantExprVisitorTest.cpp create mode 100644 src/graph/visitor/test/RewriteRelExprVisitorTest.cpp create mode 100644 src/graph/visitor/test/RewriteUnaryNotExprVisitorTest.cpp create mode 100644 src/graph/visitor/test/TestMain.cpp create mode 100644 src/graph/visitor/test/VisitorTestBase.h create mode 100644 src/interface/CMakeLists.txt create mode 100644 src/interface/common.thrift create mode 100644 src/interface/graph.thrift create mode 100644 src/interface/meta.thrift create mode 100644 src/interface/raftex.thrift create mode 100644 src/interface/storage.thrift create mode 100644 src/kvstore/CMakeLists.txt create mode 100644 src/kvstore/Common.h create mode 100644 src/kvstore/CompactionFilter.h create mode 100644 src/kvstore/DiskManager.cpp create mode 100644 src/kvstore/DiskManager.h create mode 100644 src/kvstore/EventListener.h create mode 100644 src/kvstore/KVEngine.h create mode 100644 src/kvstore/KVIterator.h create mode 100644 src/kvstore/KVStore.h create mode 100644 src/kvstore/LogEncoder.cpp create mode 100644 src/kvstore/LogEncoder.h create mode 100644 src/kvstore/NebulaSnapshotManager.cpp create mode 100644 src/kvstore/NebulaSnapshotManager.h create mode 100644 src/kvstore/NebulaStore.cpp create mode 100644 src/kvstore/NebulaStore.h create mode 100644 src/kvstore/Part.cpp create mode 100644 src/kvstore/Part.h create mode 100644 src/kvstore/PartManager.cpp create mode 100644 src/kvstore/PartManager.h create mode 100644 src/kvstore/RateLimiter.cpp create mode 100644 src/kvstore/RateLimiter.h create mode 100644 src/kvstore/RocksEngine.cpp create mode 100644 src/kvstore/RocksEngine.h create mode 100644 src/kvstore/RocksEngineConfig.cpp create mode 100644 src/kvstore/RocksEngineConfig.h create mode 100644 src/kvstore/listener/CMakeLists.txt create mode 100644 src/kvstore/listener/Listener.cpp create mode 100644 src/kvstore/listener/Listener.h create mode 100644 src/kvstore/listener/elasticsearch/ESListener.cpp create mode 100644 src/kvstore/listener/elasticsearch/ESListener.h create mode 100644 src/kvstore/listener/test/CMakeLists.txt create mode 100644 src/kvstore/listener/test/NebulaListenerTest.cpp create mode 100644 src/kvstore/plugins/hbase/HBaseStore.cpp create mode 100644 src/kvstore/plugins/hbase/HBaseStore.h create mode 100644 src/kvstore/raftex/CMakeLists.txt create mode 100644 src/kvstore/raftex/Host.cpp create mode 100644 src/kvstore/raftex/Host.h create mode 100644 src/kvstore/raftex/RaftLogIterator.cpp create mode 100644 src/kvstore/raftex/RaftLogIterator.h create mode 100644 src/kvstore/raftex/RaftPart.cpp create mode 100644 src/kvstore/raftex/RaftPart.h create mode 100644 src/kvstore/raftex/RaftexService.cpp create mode 100644 src/kvstore/raftex/RaftexService.h create mode 100644 src/kvstore/raftex/SnapshotManager.cpp create mode 100644 src/kvstore/raftex/SnapshotManager.h create mode 100644 src/kvstore/raftex/test/CMakeLists.txt create mode 100644 src/kvstore/raftex/test/LeaderElectionTest.cpp create mode 100644 src/kvstore/raftex/test/LeaderTransferTest.cpp create mode 100644 src/kvstore/raftex/test/LearnerTest.cpp create mode 100644 src/kvstore/raftex/test/LogAppendTest.cpp create mode 100644 src/kvstore/raftex/test/LogCASTest.cpp create mode 100644 src/kvstore/raftex/test/LogCommandTest.cpp create mode 100644 src/kvstore/raftex/test/MemberChangeTest.cpp create mode 100644 src/kvstore/raftex/test/RaftCase.cpp create mode 100644 src/kvstore/raftex/test/RaftexTestBase.cpp create mode 100644 src/kvstore/raftex/test/RaftexTestBase.h create mode 100644 src/kvstore/raftex/test/TestShard.cpp create mode 100644 src/kvstore/raftex/test/TestShard.h create mode 100644 src/kvstore/stats/CMakeLists.txt create mode 100644 src/kvstore/stats/KVStats.cpp create mode 100644 src/kvstore/stats/KVStats.h create mode 100644 src/kvstore/test/CMakeLists.txt create mode 100644 src/kvstore/test/CommonTest.cpp create mode 100644 src/kvstore/test/DiskManagerTest.cpp create mode 100644 src/kvstore/test/LogEncoderTest.cpp create mode 100644 src/kvstore/test/MultiVersionBenchmark.cpp create mode 100644 src/kvstore/test/NebulaStoreTest.cpp create mode 100644 src/kvstore/test/PartPerformanceTest.cpp create mode 100644 src/kvstore/test/PartTest.cpp create mode 100644 src/kvstore/test/RateLimiterTest.cpp create mode 100644 src/kvstore/test/RocksEngineConfigTest.cpp create mode 100644 src/kvstore/test/RocksEngineTest.cpp create mode 100644 src/kvstore/test/SnapshotLinkTest.cpp create mode 100644 src/kvstore/wal/AtomicLogBuffer.cpp create mode 100644 src/kvstore/wal/AtomicLogBuffer.h create mode 100644 src/kvstore/wal/CMakeLists.txt create mode 100644 src/kvstore/wal/FileBasedWal.cpp create mode 100644 src/kvstore/wal/FileBasedWal.h create mode 100644 src/kvstore/wal/Wal.h create mode 100644 src/kvstore/wal/WalFileInfo.h create mode 100644 src/kvstore/wal/WalFileIterator.cpp create mode 100644 src/kvstore/wal/WalFileIterator.h create mode 100644 src/kvstore/wal/test/AtomicLogBufferTest.cpp create mode 100644 src/kvstore/wal/test/CMakeLists.txt create mode 100644 src/kvstore/wal/test/FileBasedWalTest.cpp create mode 100644 src/kvstore/wal/test/InMemoryLogBuffer.cpp create mode 100644 src/kvstore/wal/test/InMemoryLogBuffer.h create mode 100644 src/kvstore/wal/test/InMemoryLogBufferList.h create mode 100644 src/kvstore/wal/test/InMemoryLogBufferTest.cpp create mode 100644 src/kvstore/wal/test/LogBufferBenchmark.cpp create mode 100644 src/kvstore/wal/test/WalFileIterTest.cpp create mode 100644 src/meta/ActiveHostsMan.cpp create mode 100644 src/meta/ActiveHostsMan.h create mode 100644 src/meta/CMakeLists.txt create mode 100644 src/meta/KVBasedClusterIdMan.h create mode 100644 src/meta/MetaServiceHandler.cpp create mode 100644 src/meta/MetaServiceHandler.h create mode 100644 src/meta/MetaServiceUtils.cpp create mode 100644 src/meta/MetaServiceUtils.h create mode 100644 src/meta/MetaVersionMan.cpp create mode 100644 src/meta/MetaVersionMan.h create mode 100644 src/meta/RootUserMan.h create mode 100644 src/meta/http/CMakeLists.txt create mode 100644 src/meta/http/MetaHttpReplaceHostHandler.cpp create mode 100644 src/meta/http/MetaHttpReplaceHostHandler.h create mode 100644 src/meta/http/test/CMakeLists.txt create mode 100644 src/meta/http/test/MetaHttpReplaceHandlerTest.cpp create mode 100644 src/meta/processors/BaseProcessor-inl.h create mode 100644 src/meta/processors/BaseProcessor.h create mode 100644 src/meta/processors/Common.h create mode 100644 src/meta/processors/admin/AdminClient.cpp create mode 100644 src/meta/processors/admin/AdminClient.h create mode 100644 src/meta/processors/admin/AgentHBProcessor.cpp create mode 100644 src/meta/processors/admin/AgentHBProcessor.h create mode 100644 src/meta/processors/admin/ClearSpaceProcessor.cpp create mode 100644 src/meta/processors/admin/ClearSpaceProcessor.h create mode 100644 src/meta/processors/admin/CreateBackupProcessor.cpp create mode 100644 src/meta/processors/admin/CreateBackupProcessor.h create mode 100644 src/meta/processors/admin/CreateSnapshotProcessor.cpp create mode 100644 src/meta/processors/admin/CreateSnapshotProcessor.h create mode 100644 src/meta/processors/admin/DropSnapshotProcessor.cpp create mode 100644 src/meta/processors/admin/DropSnapshotProcessor.h create mode 100644 src/meta/processors/admin/GetMetaDirInfoProcessor.cpp create mode 100644 src/meta/processors/admin/GetMetaDirInfoProcessor.h create mode 100644 src/meta/processors/admin/HBProcessor.cpp create mode 100644 src/meta/processors/admin/HBProcessor.h create mode 100644 src/meta/processors/admin/ListClusterInfoProcessor.cpp create mode 100644 src/meta/processors/admin/ListClusterInfoProcessor.h create mode 100644 src/meta/processors/admin/ListSnapshotsProcessor.cpp create mode 100644 src/meta/processors/admin/ListSnapshotsProcessor.h create mode 100644 src/meta/processors/admin/RestoreProcessor.cpp create mode 100644 src/meta/processors/admin/RestoreProcessor.h create mode 100644 src/meta/processors/admin/SaveGraphVersionProcessor.cpp create mode 100644 src/meta/processors/admin/SaveGraphVersionProcessor.h create mode 100644 src/meta/processors/admin/SnapShot.cpp create mode 100644 src/meta/processors/admin/SnapShot.h create mode 100644 src/meta/processors/admin/VerifyClientVersionProcessor.cpp create mode 100644 src/meta/processors/admin/VerifyClientVersionProcessor.h create mode 100644 src/meta/processors/config/GetConfigProcessor.cpp create mode 100644 src/meta/processors/config/GetConfigProcessor.h create mode 100644 src/meta/processors/config/ListConfigsProcessor.cpp create mode 100644 src/meta/processors/config/ListConfigsProcessor.h create mode 100644 src/meta/processors/config/RegConfigProcessor.cpp create mode 100644 src/meta/processors/config/RegConfigProcessor.h create mode 100644 src/meta/processors/config/SetConfigProcessor.cpp create mode 100644 src/meta/processors/config/SetConfigProcessor.h create mode 100644 src/meta/processors/id/GetSegmentIdProcessor.cpp create mode 100644 src/meta/processors/id/GetSegmentIdProcessor.h create mode 100644 src/meta/processors/id/GetWorkerIdProcessor.cpp create mode 100644 src/meta/processors/id/GetWorkerIdProcessor.h create mode 100644 src/meta/processors/index/CreateEdgeIndexProcessor.cpp create mode 100644 src/meta/processors/index/CreateEdgeIndexProcessor.h create mode 100644 src/meta/processors/index/CreateTagIndexProcessor.cpp create mode 100644 src/meta/processors/index/CreateTagIndexProcessor.h create mode 100644 src/meta/processors/index/DropEdgeIndexProcessor.cpp create mode 100644 src/meta/processors/index/DropEdgeIndexProcessor.h create mode 100644 src/meta/processors/index/DropTagIndexProcessor.cpp create mode 100644 src/meta/processors/index/DropTagIndexProcessor.h create mode 100644 src/meta/processors/index/FTIndexProcessor.cpp create mode 100644 src/meta/processors/index/FTIndexProcessor.h create mode 100644 src/meta/processors/index/GetEdgeIndexProcessor.cpp create mode 100644 src/meta/processors/index/GetEdgeIndexProcessor.h create mode 100644 src/meta/processors/index/GetTagIndexProcessor.cpp create mode 100644 src/meta/processors/index/GetTagIndexProcessor.h create mode 100644 src/meta/processors/index/ListEdgeIndexesProcessor.cpp create mode 100644 src/meta/processors/index/ListEdgeIndexesProcessor.h create mode 100644 src/meta/processors/index/ListTagIndexesProcessor.cpp create mode 100644 src/meta/processors/index/ListTagIndexesProcessor.h create mode 100644 src/meta/processors/job/AdminJobProcessor.cpp create mode 100644 src/meta/processors/job/AdminJobProcessor.h create mode 100644 src/meta/processors/job/BalanceJobExecutor.cpp create mode 100644 src/meta/processors/job/BalanceJobExecutor.h create mode 100644 src/meta/processors/job/BalancePlan.cpp create mode 100644 src/meta/processors/job/BalancePlan.h create mode 100644 src/meta/processors/job/BalanceTask.cpp create mode 100644 src/meta/processors/job/BalanceTask.h create mode 100644 src/meta/processors/job/CompactJobExecutor.cpp create mode 100644 src/meta/processors/job/CompactJobExecutor.h create mode 100644 src/meta/processors/job/DataBalanceJobExecutor.cpp create mode 100644 src/meta/processors/job/DataBalanceJobExecutor.h create mode 100644 src/meta/processors/job/DownloadJobExecutor.cpp create mode 100644 src/meta/processors/job/DownloadJobExecutor.h create mode 100644 src/meta/processors/job/FlushJobExecutor.cpp create mode 100644 src/meta/processors/job/FlushJobExecutor.h create mode 100644 src/meta/processors/job/GetStatsProcessor.cpp create mode 100644 src/meta/processors/job/GetStatsProcessor.h create mode 100644 src/meta/processors/job/IngestJobExecutor.cpp create mode 100644 src/meta/processors/job/IngestJobExecutor.h create mode 100644 src/meta/processors/job/JobDescription.cpp create mode 100644 src/meta/processors/job/JobDescription.h create mode 100644 src/meta/processors/job/JobExecutor.cpp create mode 100644 src/meta/processors/job/JobExecutor.h create mode 100644 src/meta/processors/job/JobManager.cpp create mode 100644 src/meta/processors/job/JobManager.h create mode 100644 src/meta/processors/job/JobStatus.cpp create mode 100644 src/meta/processors/job/JobStatus.h create mode 100644 src/meta/processors/job/LeaderBalanceJobExecutor.cpp create mode 100644 src/meta/processors/job/LeaderBalanceJobExecutor.h create mode 100644 src/meta/processors/job/ListEdgeIndexStatusProcessor.cpp create mode 100644 src/meta/processors/job/ListEdgeIndexStatusProcessor.h create mode 100644 src/meta/processors/job/ListTagIndexStatusProcessor.cpp create mode 100644 src/meta/processors/job/ListTagIndexStatusProcessor.h create mode 100644 src/meta/processors/job/MetaJobExecutor.cpp create mode 100644 src/meta/processors/job/MetaJobExecutor.h create mode 100644 src/meta/processors/job/RebuildEdgeJobExecutor.cpp create mode 100644 src/meta/processors/job/RebuildEdgeJobExecutor.h create mode 100644 src/meta/processors/job/RebuildFTJobExecutor.cpp create mode 100644 src/meta/processors/job/RebuildFTJobExecutor.h create mode 100644 src/meta/processors/job/RebuildJobExecutor.cpp create mode 100644 src/meta/processors/job/RebuildJobExecutor.h create mode 100644 src/meta/processors/job/RebuildTagJobExecutor.cpp create mode 100644 src/meta/processors/job/RebuildTagJobExecutor.h create mode 100644 src/meta/processors/job/ReportTaskProcessor.cpp create mode 100644 src/meta/processors/job/ReportTaskProcessor.h create mode 100644 src/meta/processors/job/SimpleConcurrentJobExecutor.cpp create mode 100644 src/meta/processors/job/SimpleConcurrentJobExecutor.h create mode 100644 src/meta/processors/job/StatsJobExecutor.cpp create mode 100644 src/meta/processors/job/StatsJobExecutor.h create mode 100644 src/meta/processors/job/StorageJobExecutor.cpp create mode 100644 src/meta/processors/job/StorageJobExecutor.h create mode 100644 src/meta/processors/job/TaskDescription.cpp create mode 100644 src/meta/processors/job/TaskDescription.h create mode 100644 src/meta/processors/job/ZoneBalanceJobExecutor.cpp create mode 100644 src/meta/processors/job/ZoneBalanceJobExecutor.h create mode 100644 src/meta/processors/listener/ListenerProcessor.cpp create mode 100644 src/meta/processors/listener/ListenerProcessor.h create mode 100644 src/meta/processors/parts/AlterSpaceProcessor.cpp create mode 100644 src/meta/processors/parts/AlterSpaceProcessor.h create mode 100644 src/meta/processors/parts/CreateSpaceAsProcessor.cpp create mode 100644 src/meta/processors/parts/CreateSpaceAsProcessor.h create mode 100644 src/meta/processors/parts/CreateSpaceProcessor.cpp create mode 100644 src/meta/processors/parts/CreateSpaceProcessor.h create mode 100644 src/meta/processors/parts/DropSpaceProcessor.cpp create mode 100644 src/meta/processors/parts/DropSpaceProcessor.h create mode 100644 src/meta/processors/parts/GetPartsAllocProcessor.cpp create mode 100644 src/meta/processors/parts/GetPartsAllocProcessor.h create mode 100644 src/meta/processors/parts/GetSpaceProcessor.cpp create mode 100644 src/meta/processors/parts/GetSpaceProcessor.h create mode 100644 src/meta/processors/parts/ListHostsProcessor.cpp create mode 100644 src/meta/processors/parts/ListHostsProcessor.h create mode 100644 src/meta/processors/parts/ListPartsProcessor.cpp create mode 100644 src/meta/processors/parts/ListPartsProcessor.h create mode 100644 src/meta/processors/parts/ListSpacesProcessor.cpp create mode 100644 src/meta/processors/parts/ListSpacesProcessor.h create mode 100644 src/meta/processors/schema/AlterEdgeProcessor.cpp create mode 100644 src/meta/processors/schema/AlterEdgeProcessor.h create mode 100644 src/meta/processors/schema/AlterTagProcessor.cpp create mode 100644 src/meta/processors/schema/AlterTagProcessor.h create mode 100644 src/meta/processors/schema/CreateEdgeProcessor.cpp create mode 100644 src/meta/processors/schema/CreateEdgeProcessor.h create mode 100644 src/meta/processors/schema/CreateTagProcessor.cpp create mode 100644 src/meta/processors/schema/CreateTagProcessor.h create mode 100644 src/meta/processors/schema/DropEdgeProcessor.cpp create mode 100644 src/meta/processors/schema/DropEdgeProcessor.h create mode 100644 src/meta/processors/schema/DropTagProcessor.cpp create mode 100644 src/meta/processors/schema/DropTagProcessor.h create mode 100644 src/meta/processors/schema/GetEdgeProcessor.cpp create mode 100644 src/meta/processors/schema/GetEdgeProcessor.h create mode 100644 src/meta/processors/schema/GetTagProcessor.cpp create mode 100644 src/meta/processors/schema/GetTagProcessor.h create mode 100644 src/meta/processors/schema/ListEdgesProcessor.cpp create mode 100644 src/meta/processors/schema/ListEdgesProcessor.h create mode 100644 src/meta/processors/schema/ListTagsProcessor.cpp create mode 100644 src/meta/processors/schema/ListTagsProcessor.h create mode 100644 src/meta/processors/schema/SchemaUtil.cpp create mode 100644 src/meta/processors/schema/SchemaUtil.h create mode 100644 src/meta/processors/service/ServiceProcessor.cpp create mode 100644 src/meta/processors/service/ServiceProcessor.h create mode 100644 src/meta/processors/session/SessionManagerProcessor.cpp create mode 100644 src/meta/processors/session/SessionManagerProcessor.h create mode 100644 src/meta/processors/user/AuthenticationProcessor.cpp create mode 100644 src/meta/processors/user/AuthenticationProcessor.h create mode 100644 src/meta/processors/zone/AddHostsIntoZoneProcessor.cpp create mode 100644 src/meta/processors/zone/AddHostsIntoZoneProcessor.h create mode 100644 src/meta/processors/zone/AddHostsProcessor.cpp create mode 100644 src/meta/processors/zone/AddHostsProcessor.h create mode 100644 src/meta/processors/zone/DivideZoneProcessor.cpp create mode 100644 src/meta/processors/zone/DivideZoneProcessor.h create mode 100644 src/meta/processors/zone/DropHostsProcessor.cpp create mode 100644 src/meta/processors/zone/DropHostsProcessor.h create mode 100644 src/meta/processors/zone/DropZoneProcessor.cpp create mode 100644 src/meta/processors/zone/DropZoneProcessor.h create mode 100644 src/meta/processors/zone/GetZoneProcessor.cpp create mode 100644 src/meta/processors/zone/GetZoneProcessor.h create mode 100644 src/meta/processors/zone/ListZonesProcessor.cpp create mode 100644 src/meta/processors/zone/ListZonesProcessor.h create mode 100644 src/meta/processors/zone/MergeZoneProcessor.cpp create mode 100644 src/meta/processors/zone/MergeZoneProcessor.h create mode 100644 src/meta/processors/zone/RenameZoneProcessor.cpp create mode 100644 src/meta/processors/zone/RenameZoneProcessor.h create mode 100644 src/meta/stats/CMakeLists.txt create mode 100644 src/meta/stats/MetaStats.cpp create mode 100644 src/meta/stats/MetaStats.h create mode 100644 src/meta/test/ActiveHostsManTest.cpp create mode 100644 src/meta/test/AdminClientTest.cpp create mode 100644 src/meta/test/AgentHBProcessorTest.cpp create mode 100644 src/meta/test/AuthProcessorTest.cpp create mode 100644 src/meta/test/BalancerTest.cpp create mode 100644 src/meta/test/CMakeLists.txt create mode 100644 src/meta/test/ClusterIdManTest.cpp create mode 100644 src/meta/test/ConfigManTest.cpp create mode 100644 src/meta/test/CreateBackupProcessorTest.cpp create mode 100644 src/meta/test/GetStatsTest.cpp create mode 100644 src/meta/test/HBProcessorTest.cpp create mode 100644 src/meta/test/IdProcessorTest.cpp create mode 100644 src/meta/test/IndexProcessorTest.cpp create mode 100644 src/meta/test/JobManagerTest.cpp create mode 100644 src/meta/test/ListClusterInfoTest.cpp create mode 100644 src/meta/test/MetaClientTest.cpp create mode 100644 src/meta/test/MockAdminClient.h create mode 100644 src/meta/test/MockHdfsHelper.h create mode 100644 src/meta/test/ProcessorTest.cpp create mode 100644 src/meta/test/RestoreProcessorTest.cpp create mode 100644 src/meta/test/SnapshotProcessorsTest.cpp create mode 100644 src/meta/test/TestUtils.h create mode 100644 src/meta/test/VerifyClientVersionTest.cpp create mode 100644 src/meta/upgrade/CMakeLists.txt create mode 100644 src/meta/upgrade/MetaDataUpgrade.cpp create mode 100644 src/meta/upgrade/MetaDataUpgrade.h create mode 100644 src/meta/upgrade/v2/CMakeLists.txt create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.cpp create mode 100644 src/meta/upgrade/v2/MetaServiceUtilsV2.h create mode 100644 src/meta/upgrade/v2/meta.thrift create mode 100644 src/mock/AdHocIndexManager.cpp create mode 100644 src/mock/AdHocIndexManager.h create mode 100644 src/mock/AdHocSchemaManager.cpp create mode 100644 src/mock/AdHocSchemaManager.h create mode 100644 src/mock/CMakeLists.txt create mode 100644 src/mock/FakeHttpServer.cpp create mode 100644 src/mock/FakeHttpServer.h create mode 100644 src/mock/LocalServer.h create mode 100644 src/mock/MockCluster.cpp create mode 100644 src/mock/MockCluster.h create mode 100644 src/mock/MockData.cpp create mode 100644 src/mock/MockData.h create mode 100644 src/mock/RpcServer.h create mode 100644 src/parser/.gitignore create mode 100644 src/parser/AdminSentences.cpp create mode 100644 src/parser/AdminSentences.h create mode 100644 src/parser/CMakeLists.txt create mode 100644 src/parser/Clauses.cpp create mode 100644 src/parser/Clauses.h create mode 100644 src/parser/EdgeKey.cpp create mode 100644 src/parser/EdgeKey.h create mode 100644 src/parser/ExplainSentence.cpp create mode 100644 src/parser/ExplainSentence.h create mode 100644 src/parser/GQLParser.cpp create mode 100644 src/parser/GQLParser.h create mode 100644 src/parser/GraphScanner.h create mode 100644 src/parser/MaintainSentences.cpp create mode 100644 src/parser/MaintainSentences.h create mode 100644 src/parser/MatchPath.cpp create mode 100644 src/parser/MatchPath.h create mode 100644 src/parser/MatchSentence.cpp create mode 100644 src/parser/MatchSentence.h create mode 100644 src/parser/MutateSentences.cpp create mode 100644 src/parser/MutateSentences.h create mode 100644 src/parser/ProcessControlSentences.cpp create mode 100644 src/parser/ProcessControlSentences.h create mode 100644 src/parser/Sentence.h create mode 100644 src/parser/SequentialSentences.cpp create mode 100644 src/parser/SequentialSentences.h create mode 100644 src/parser/TraverseSentences.cpp create mode 100644 src/parser/TraverseSentences.h create mode 100644 src/parser/UserSentences.cpp create mode 100644 src/parser/UserSentences.h create mode 100644 src/parser/parser.yy create mode 100644 src/parser/scanner.lex create mode 100644 src/parser/test/CMakeLists.txt create mode 100644 src/parser/test/ExpressionParsingTest.cpp create mode 100644 src/parser/test/ParserBenchmark.cpp create mode 100644 src/parser/test/ParserTest.cpp create mode 100644 src/parser/test/ScannerTest.cpp create mode 100644 src/parser/test/fuzzing/CMakeLists.txt create mode 100644 src/parser/test/fuzzing/ParserFuzzer.cpp create mode 100644 src/parser/test/fuzzing/README.md create mode 100644 src/parser/test/fuzzing/nebula.dict create mode 100644 src/storage/BaseProcessor-inl.h create mode 100644 src/storage/BaseProcessor.h create mode 100644 src/storage/CMakeLists.txt create mode 100644 src/storage/CommonUtils.cpp create mode 100644 src/storage/CommonUtils.h create mode 100644 src/storage/CompactionFilter.h create mode 100644 src/storage/GraphStorageLocalServer.cpp create mode 100644 src/storage/GraphStorageLocalServer.h create mode 100644 src/storage/GraphStorageServiceHandler.cpp create mode 100644 src/storage/GraphStorageServiceHandler.h create mode 100644 src/storage/InternalStorageServiceHandler.cpp create mode 100644 src/storage/InternalStorageServiceHandler.h create mode 100644 src/storage/MergeOperator.h create mode 100644 src/storage/StorageAdminServiceHandler.cpp create mode 100644 src/storage/StorageAdminServiceHandler.h create mode 100644 src/storage/StorageFlags.cpp create mode 100644 src/storage/StorageFlags.h create mode 100644 src/storage/StorageServer.cpp create mode 100644 src/storage/StorageServer.h create mode 100644 src/storage/admin/AdminProcessor.h create mode 100644 src/storage/admin/AdminTask.cpp create mode 100644 src/storage/admin/AdminTask.h create mode 100644 src/storage/admin/AdminTaskManager.cpp create mode 100644 src/storage/admin/AdminTaskManager.h create mode 100644 src/storage/admin/AdminTaskProcessor.cpp create mode 100644 src/storage/admin/AdminTaskProcessor.h create mode 100644 src/storage/admin/ClearSpaceProcessor.cpp create mode 100644 src/storage/admin/ClearSpaceProcessor.h create mode 100644 src/storage/admin/CompactTask.cpp create mode 100644 src/storage/admin/CompactTask.h create mode 100644 src/storage/admin/CreateCheckpointProcessor.cpp create mode 100644 src/storage/admin/CreateCheckpointProcessor.h create mode 100644 src/storage/admin/DownloadTask.cpp create mode 100644 src/storage/admin/DownloadTask.h create mode 100644 src/storage/admin/DropCheckpointProcessor.cpp create mode 100644 src/storage/admin/DropCheckpointProcessor.h create mode 100644 src/storage/admin/FlushTask.cpp create mode 100644 src/storage/admin/FlushTask.h create mode 100644 src/storage/admin/GetLeaderProcessor.cpp create mode 100644 src/storage/admin/GetLeaderProcessor.h create mode 100644 src/storage/admin/IngestTask.cpp create mode 100644 src/storage/admin/IngestTask.h create mode 100644 src/storage/admin/RebuildEdgeIndexTask.cpp create mode 100644 src/storage/admin/RebuildEdgeIndexTask.h create mode 100644 src/storage/admin/RebuildFTIndexTask.cpp create mode 100644 src/storage/admin/RebuildFTIndexTask.h create mode 100644 src/storage/admin/RebuildIndexTask.cpp create mode 100644 src/storage/admin/RebuildIndexTask.h create mode 100644 src/storage/admin/RebuildTagIndexTask.cpp create mode 100644 src/storage/admin/RebuildTagIndexTask.h create mode 100644 src/storage/admin/SendBlockSignProcessor.cpp create mode 100644 src/storage/admin/SendBlockSignProcessor.h create mode 100644 src/storage/admin/StatsTask.cpp create mode 100644 src/storage/admin/StatsTask.h create mode 100644 src/storage/admin/StopAdminTaskProcessor.cpp create mode 100644 src/storage/admin/StopAdminTaskProcessor.h create mode 100644 src/storage/context/StorageExpressionContext.cpp create mode 100644 src/storage/context/StorageExpressionContext.h create mode 100644 src/storage/exec/AggregateNode.h create mode 100644 src/storage/exec/DeDupNode.h create mode 100644 src/storage/exec/EdgeNode.h create mode 100644 src/storage/exec/FilterNode.h create mode 100644 src/storage/exec/GetDstBySrcNode.h create mode 100644 src/storage/exec/GetNeighborsNode.h create mode 100644 src/storage/exec/GetPropNode.h create mode 100644 src/storage/exec/HashJoinNode.h create mode 100644 src/storage/exec/IndexAggregateNode.cpp create mode 100644 src/storage/exec/IndexAggregateNode.h create mode 100644 src/storage/exec/IndexDedupNode.cpp create mode 100644 src/storage/exec/IndexDedupNode.h create mode 100644 src/storage/exec/IndexEdgeScanNode.cpp create mode 100644 src/storage/exec/IndexEdgeScanNode.h create mode 100644 src/storage/exec/IndexExprContext.h create mode 100644 src/storage/exec/IndexLimitNode.cpp create mode 100644 src/storage/exec/IndexLimitNode.h create mode 100644 src/storage/exec/IndexNode.cpp create mode 100644 src/storage/exec/IndexNode.h create mode 100644 src/storage/exec/IndexProjectionNode.cpp create mode 100644 src/storage/exec/IndexProjectionNode.h create mode 100644 src/storage/exec/IndexScanNode.cpp create mode 100644 src/storage/exec/IndexScanNode.h create mode 100644 src/storage/exec/IndexSelectionNode.cpp create mode 100644 src/storage/exec/IndexSelectionNode.h create mode 100644 src/storage/exec/IndexTopNNode.cpp create mode 100644 src/storage/exec/IndexTopNNode.h create mode 100644 src/storage/exec/IndexVertexScanNode.cpp create mode 100644 src/storage/exec/IndexVertexScanNode.h create mode 100644 src/storage/exec/MultiTagNode.h create mode 100644 src/storage/exec/QueryUtils.h create mode 100644 src/storage/exec/RelNode.h create mode 100644 src/storage/exec/ScanNode.h create mode 100644 src/storage/exec/StorageIterator.h create mode 100644 src/storage/exec/StoragePlan.h create mode 100644 src/storage/exec/TagNode.h create mode 100644 src/storage/exec/UpdateNode.h create mode 100644 src/storage/exec/UpdateResultNode.h create mode 100644 src/storage/http/StorageHttpAdminHandler.cpp create mode 100644 src/storage/http/StorageHttpAdminHandler.h create mode 100644 src/storage/http/StorageHttpPropertyHandler.cpp create mode 100644 src/storage/http/StorageHttpPropertyHandler.h create mode 100644 src/storage/http/StorageHttpStatsHandler.cpp create mode 100644 src/storage/http/StorageHttpStatsHandler.h create mode 100644 src/storage/index/LookupProcessor.cpp create mode 100644 src/storage/index/LookupProcessor.h create mode 100644 src/storage/kv/GetProcessor.cpp create mode 100644 src/storage/kv/GetProcessor.h create mode 100644 src/storage/kv/PutProcessor.cpp create mode 100644 src/storage/kv/PutProcessor.h create mode 100644 src/storage/kv/RemoveProcessor.cpp create mode 100644 src/storage/kv/RemoveProcessor.h create mode 100644 src/storage/mutate/AddEdgesProcessor.cpp create mode 100644 src/storage/mutate/AddEdgesProcessor.h create mode 100644 src/storage/mutate/AddVerticesProcessor.cpp create mode 100644 src/storage/mutate/AddVerticesProcessor.h create mode 100644 src/storage/mutate/DeleteEdgesProcessor.cpp create mode 100644 src/storage/mutate/DeleteEdgesProcessor.h create mode 100644 src/storage/mutate/DeleteTagsProcessor.cpp create mode 100644 src/storage/mutate/DeleteTagsProcessor.h create mode 100644 src/storage/mutate/DeleteVerticesProcessor.cpp create mode 100644 src/storage/mutate/DeleteVerticesProcessor.h create mode 100644 src/storage/mutate/UpdateEdgeProcessor.cpp create mode 100644 src/storage/mutate/UpdateEdgeProcessor.h create mode 100644 src/storage/mutate/UpdateVertexProcessor.cpp create mode 100644 src/storage/mutate/UpdateVertexProcessor.h create mode 100644 src/storage/query/GetDstBySrcProcessor.cpp create mode 100644 src/storage/query/GetDstBySrcProcessor.h create mode 100644 src/storage/query/GetNeighborsProcessor.cpp create mode 100644 src/storage/query/GetNeighborsProcessor.h create mode 100644 src/storage/query/GetPropProcessor.cpp create mode 100644 src/storage/query/GetPropProcessor.h create mode 100644 src/storage/query/QueryBaseProcessor-inl.h create mode 100644 src/storage/query/QueryBaseProcessor.h create mode 100644 src/storage/query/ScanEdgeProcessor.cpp create mode 100644 src/storage/query/ScanEdgeProcessor.h create mode 100644 src/storage/query/ScanVertexProcessor.cpp create mode 100644 src/storage/query/ScanVertexProcessor.h create mode 100644 src/storage/stats/CMakeLists.txt create mode 100644 src/storage/stats/StorageStats.cpp create mode 100644 src/storage/stats/StorageStats.h create mode 100644 src/storage/test/AddAndUpdateVertexAndEdgeBenchmark.cpp create mode 100644 src/storage/test/AddEdgesTest.cpp create mode 100644 src/storage/test/AddVerticesTest.cpp create mode 100644 src/storage/test/AdminTaskManagerTest.cpp create mode 100644 src/storage/test/CMakeLists.txt create mode 100644 src/storage/test/ChainAddEdgesTest.cpp create mode 100644 src/storage/test/ChainDeleteEdgesTest.cpp create mode 100644 src/storage/test/ChainResumeEdgeTest.cpp create mode 100644 src/storage/test/ChainTestUtils.h create mode 100644 src/storage/test/ChainUpdateEdgeTest.cpp create mode 100644 src/storage/test/CheckpointTest.cpp create mode 100644 src/storage/test/CompactionTest.cpp create mode 100644 src/storage/test/ConsistUtilTest.cpp create mode 100644 src/storage/test/DeleteEdgesTest.cpp create mode 100644 src/storage/test/DeleteTagsTest.cpp create mode 100644 src/storage/test/DeleteVerticesTest.cpp create mode 100644 src/storage/test/ElasticSearchBulkInsertTest.cpp create mode 100644 src/storage/test/GetDstBySrcTest.cpp create mode 100644 src/storage/test/GetNeighborsBenchmark.cpp create mode 100644 src/storage/test/GetNeighborsTest.cpp create mode 100644 src/storage/test/GetPropTest.cpp create mode 100644 src/storage/test/IndexScanLimitTest.cpp create mode 100644 src/storage/test/IndexScanTopNTest.cpp create mode 100644 src/storage/test/IndexTest.cpp create mode 100644 src/storage/test/IndexTestUtil.h create mode 100644 src/storage/test/IndexWithTTLTest.cpp create mode 100644 src/storage/test/IndexWriteTest.cpp create mode 100644 src/storage/test/KVClientTest.cpp create mode 100644 src/storage/test/KVTest.cpp create mode 100644 src/storage/test/KillQueryTest.cpp create mode 100644 src/storage/test/LookupIndexTest.cpp create mode 100644 src/storage/test/MemoryLockBenchmark.cpp create mode 100644 src/storage/test/MemoryLockTest.cpp create mode 100644 src/storage/test/MockHdfsHelper.h create mode 100644 src/storage/test/PrefixBloomFilterBenchmark.cpp create mode 100644 src/storage/test/QueryTestUtils.h create mode 100644 src/storage/test/RebuildIndexTest.cpp create mode 100644 src/storage/test/ScanEdgePropBenchmark.cpp create mode 100644 src/storage/test/ScanEdgeTest.cpp create mode 100644 src/storage/test/ScanVertexTest.cpp create mode 100644 src/storage/test/ShuffleIpTest.cpp create mode 100644 src/storage/test/StatsTaskTest.cpp create mode 100644 src/storage/test/StorageDAGBenchmark.cpp create mode 100644 src/storage/test/StorageDAGTest.cpp create mode 100644 src/storage/test/StorageHttpAdminHandlerTest.cpp create mode 100644 src/storage/test/StorageHttpPropertyHandlerTest.cpp create mode 100644 src/storage/test/StorageHttpStatsHandlerTest.cpp create mode 100644 src/storage/test/StorageIndexWriteBenchmark.cpp create mode 100644 src/storage/test/TestUtils.h create mode 100644 src/storage/test/TossTestExecutor.h create mode 100644 src/storage/test/TossTestUtils.h create mode 100644 src/storage/test/UpdateEdgeTest.cpp create mode 100644 src/storage/test/UpdateVertexTest.cpp create mode 100644 src/storage/transaction/ChainAddEdgesGroupProcessor.cpp create mode 100644 src/storage/transaction/ChainAddEdgesGroupProcessor.h create mode 100644 src/storage/transaction/ChainAddEdgesLocalProcessor.cpp create mode 100644 src/storage/transaction/ChainAddEdgesLocalProcessor.h create mode 100644 src/storage/transaction/ChainAddEdgesRemoteProcessor.cpp create mode 100644 src/storage/transaction/ChainAddEdgesRemoteProcessor.h create mode 100644 src/storage/transaction/ChainBaseProcessor.h create mode 100644 src/storage/transaction/ChainDeleteEdgesGroupProcessor.cpp create mode 100644 src/storage/transaction/ChainDeleteEdgesGroupProcessor.h create mode 100644 src/storage/transaction/ChainDeleteEdgesLocalProcessor.cpp create mode 100644 src/storage/transaction/ChainDeleteEdgesLocalProcessor.h create mode 100644 src/storage/transaction/ChainDeleteEdgesRemoteProcessor.cpp create mode 100644 src/storage/transaction/ChainDeleteEdgesRemoteProcessor.h create mode 100644 src/storage/transaction/ChainDeleteEdgesResumeProcessor.cpp create mode 100644 src/storage/transaction/ChainDeleteEdgesResumeProcessor.h create mode 100644 src/storage/transaction/ChainDeleteEdgesResumeRemoteProcessor.cpp create mode 100644 src/storage/transaction/ChainDeleteEdgesResumeRemoteProcessor.h create mode 100644 src/storage/transaction/ChainProcessorFactory.cpp create mode 100644 src/storage/transaction/ChainProcessorFactory.h create mode 100644 src/storage/transaction/ChainResumeAddDoublePrimeProcessor.cpp create mode 100644 src/storage/transaction/ChainResumeAddDoublePrimeProcessor.h create mode 100644 src/storage/transaction/ChainResumeAddPrimeProcessor.cpp create mode 100644 src/storage/transaction/ChainResumeAddPrimeProcessor.h create mode 100644 src/storage/transaction/ChainResumeUpdateDoublePrimeProcessor.cpp create mode 100644 src/storage/transaction/ChainResumeUpdateDoublePrimeProcessor.h create mode 100644 src/storage/transaction/ChainResumeUpdatePrimeProcessor.cpp create mode 100644 src/storage/transaction/ChainResumeUpdatePrimeProcessor.h create mode 100644 src/storage/transaction/ChainUpdateEdgeLocalProcessor.cpp create mode 100644 src/storage/transaction/ChainUpdateEdgeLocalProcessor.h create mode 100644 src/storage/transaction/ChainUpdateEdgeRemoteProcessor.cpp create mode 100644 src/storage/transaction/ChainUpdateEdgeRemoteProcessor.h create mode 100644 src/storage/transaction/ConsistTypes.h create mode 100644 src/storage/transaction/ConsistUtil.cpp create mode 100644 src/storage/transaction/ConsistUtil.h create mode 100644 src/storage/transaction/TransactionManager.cpp create mode 100644 src/storage/transaction/TransactionManager.h create mode 100644 src/tools/CMakeLists.txt create mode 100644 src/tools/db-dump/CMakeLists.txt create mode 100644 src/tools/db-dump/DbDumpTool.cpp create mode 100644 src/tools/db-dump/DbDumper.cpp create mode 100644 src/tools/db-dump/DbDumper.h create mode 100644 src/tools/db-upgrade/CMakeLists.txt create mode 100644 src/tools/db-upgrade/DbUpgrader.cpp create mode 100644 src/tools/db-upgrade/DbUpgrader.h create mode 100644 src/tools/db-upgrade/DbUpgraderTool.cpp create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV1.cpp create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV1.h create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV2.cpp create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV2.h create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV3.cpp create mode 100644 src/tools/db-upgrade/NebulaKeyUtilsV3.h create mode 100644 src/tools/meta-dump/CMakeLists.txt create mode 100644 src/tools/meta-dump/MetaDumpTool.cpp create mode 100644 src/tools/simple-kv-verify/CMakeLists.txt create mode 100644 src/tools/simple-kv-verify/SimpleKVVerifyTool.cpp create mode 100644 src/tools/storage-perf/CMakeLists.txt create mode 100644 src/tools/storage-perf/README.md create mode 100644 src/tools/storage-perf/StorageIntegrityTool.cpp create mode 100644 src/tools/storage-perf/StoragePerfTool.cpp create mode 100644 src/version/CMakeLists.txt create mode 100644 src/version/Version.cpp.in create mode 100644 src/version/Version.h create mode 100644 src/webservice/CMakeLists.txt create mode 100644 src/webservice/Common.cpp create mode 100644 src/webservice/Common.h create mode 100644 src/webservice/GetFlagsHandler.cpp create mode 100644 src/webservice/GetFlagsHandler.h create mode 100644 src/webservice/GetStatsHandler.cpp create mode 100644 src/webservice/GetStatsHandler.h create mode 100644 src/webservice/NotFoundHandler.cpp create mode 100644 src/webservice/NotFoundHandler.h create mode 100644 src/webservice/Router.cpp create mode 100644 src/webservice/Router.h create mode 100644 src/webservice/SetFlagsHandler.cpp create mode 100644 src/webservice/SetFlagsHandler.h create mode 100644 src/webservice/StatusHandler.cpp create mode 100644 src/webservice/StatusHandler.h create mode 100644 src/webservice/WebService.cpp create mode 100644 src/webservice/WebService.h create mode 100644 src/webservice/test/CMakeLists.txt create mode 100644 src/webservice/test/FlagsAccessTest.cpp create mode 100644 src/webservice/test/RouterTest.cpp create mode 100644 src/webservice/test/StatsReaderTest.cpp create mode 100644 src/webservice/test/StatusHandlerTest.cpp create mode 100644 src/webservice/test/TestUtils.h create mode 100644 tests/.gitignore create mode 100644 tests/.pylintrc create mode 100644 tests/Makefile create mode 100644 tests/README.md create mode 100644 tests/__init__.py create mode 100644 tests/admin/__init__.py create mode 100644 tests/admin/test_charset.py create mode 100644 tests/admin/test_configs.py create mode 100644 tests/admin/test_listener.py create mode 100644 tests/admin/test_parts.py create mode 100644 tests/admin/test_permission.py create mode 100644 tests/admin/test_show_hosts.py create mode 100644 tests/admin/test_space.py create mode 100644 tests/admin/test_users.py create mode 100644 tests/bench/__init__.py create mode 100644 tests/bench/data_generate.py create mode 100644 tests/bench/delete.py create mode 100644 tests/bench/insert.py create mode 100644 tests/bench/lookup.py create mode 100644 tests/cert/test.2.crt create mode 100644 tests/cert/test.2.csr create mode 100644 tests/cert/test.2.key create mode 100644 tests/cert/test.2.password create mode 100644 tests/cert/test.ca.key create mode 100644 tests/cert/test.ca.password create mode 100644 tests/cert/test.ca.pem create mode 100644 tests/cert/test.ca.srl create mode 100644 tests/cert/test.derive.crt create mode 100644 tests/cert/test.derive.csr create mode 100644 tests/cert/test.derive.key create mode 100644 tests/common/__init__.py create mode 100644 tests/common/comparator.py create mode 100644 tests/common/configs.py create mode 100644 tests/common/constants.py create mode 100644 tests/common/csv_import.py create mode 100644 tests/common/dataset_printer.py create mode 100644 tests/common/nebula_service.py create mode 100644 tests/common/nebula_test_suite.py create mode 100644 tests/common/path_value.py create mode 100644 tests/common/plan_differ.py create mode 100644 tests/common/types.py create mode 100644 tests/common/utils.py create mode 100644 tests/conftest.py create mode 100644 tests/data/ldbc_v0_3_3/config.yaml create mode 100644 tests/data/nba.ngql create mode 100644 tests/data/nba/bachelor.csv create mode 100644 tests/data/nba/config.yaml create mode 100644 tests/data/nba/like.csv create mode 100644 tests/data/nba/null.csv create mode 100644 tests/data/nba/player.csv create mode 100644 tests/data/nba/serve.csv create mode 100644 tests/data/nba/team.csv create mode 100644 tests/data/nba/teammate.csv create mode 100644 tests/data/nba_int_vid/config.yaml create mode 100644 tests/data/ngdata/Edge_label_limits.csv create mode 100644 tests/data/ngdata/Edge_ttl_0.csv create mode 100644 tests/data/ngdata/Label_0.csv create mode 100644 tests/data/ngdata/Label_1.csv create mode 100644 tests/data/ngdata/Label_10.csv create mode 100644 tests/data/ngdata/Label_11.csv create mode 100644 tests/data/ngdata/Label_2.csv create mode 100644 tests/data/ngdata/Label_3.csv create mode 100644 tests/data/ngdata/Label_4.csv create mode 100644 tests/data/ngdata/Label_5.csv create mode 100644 tests/data/ngdata/Label_6.csv create mode 100644 tests/data/ngdata/Label_7.csv create mode 100644 tests/data/ngdata/Label_8.csv create mode 100644 tests/data/ngdata/Label_9.csv create mode 100644 tests/data/ngdata/Label_same_prop_0.csv create mode 100644 tests/data/ngdata/Label_same_prop_1.csv create mode 100644 tests/data/ngdata/Label_ttl_0.csv create mode 100644 tests/data/ngdata/README.md create mode 100644 tests/data/ngdata/Rel_0.csv create mode 100644 tests/data/ngdata/Rel_1.csv create mode 100644 tests/data/ngdata/Rel_2.csv create mode 100644 tests/data/ngdata/Rel_3.csv create mode 100644 tests/data/ngdata/Rel_4.csv create mode 100644 tests/data/ngdata/Rel_5.csv create mode 100644 tests/data/ngdata/config.yaml create mode 100644 tests/data/ngdata/gen_graph.sh create mode 100644 tests/data/student/config.yaml create mode 100644 tests/data/student/is_colleagues.csv create mode 100644 tests/data/student/is_friend.csv create mode 100644 tests/data/student/is_schoolmate.csv create mode 100644 tests/data/student/is_teacher.csv create mode 100644 tests/data/student/person.csv create mode 100644 tests/data/student/student.csv create mode 100644 tests/data/student/teacher.csv create mode 100644 tests/job/__init__.py create mode 100644 tests/job/test_session.py create mode 100644 tests/job/test_snapshot.py create mode 100644 tests/maintain/__init__.py create mode 100644 tests/maintain/test_comments.py create mode 100644 tests/maintain/test_zone.py create mode 100644 tests/nebula-test-run.py create mode 100644 tests/pytest.ini create mode 100644 tests/query/__init__.py create mode 100644 tests/query/bugs/__init__.py create mode 100644 tests/query/bugs/fixed_bigint_2031.py create mode 100644 tests/query/bugs/fixed_delete_vertex_1996.py create mode 100644 tests/query/bugs/fixed_issue2020.py create mode 100644 tests/query/bugs/fixed_negative_ttl.py create mode 100644 tests/query/bugs/fixed_over_all_pr1962.py create mode 100644 tests/query/bugs/fixed_pr2042.py create mode 100644 tests/query/bugs/fixed_update_issue1827.py create mode 100644 tests/query/bugs/remove_unreasonable_error_message_pr1983.py create mode 100644 tests/query/bugs/test_fixed_issue1888.py create mode 100644 tests/query/bugs/test_fixed_issue1987.py create mode 100644 tests/query/bugs/test_fixed_issue2009.py create mode 100644 tests/query/bugs/test_query_after_schema_altered_issue1185_1154_pr1606.py create mode 100644 tests/query/bugs/test_query_after_schema_rebuilt_issue1843.py create mode 100644 tests/query/stateless/__init__.py create mode 100644 tests/query/stateless/test_admin.py create mode 100644 tests/query/stateless/test_fetch.py create mode 100644 tests/query/stateless/test_findpath.py create mode 100644 tests/query/stateless/test_go.py create mode 100644 tests/query/stateless/test_groupby.py create mode 100644 tests/query/stateless/test_if_exists.py create mode 100644 tests/query/stateless/test_index.py create mode 100644 tests/query/stateless/test_keyword.py create mode 100644 tests/query/stateless/test_query_reversely.py create mode 100644 tests/query/stateless/test_range.py create mode 100644 tests/query/stateless/test_schema.py create mode 100644 tests/query/stateless/test_set.py create mode 100644 tests/query/stateless/test_update.py create mode 100644 tests/query/stateless/test_where.py create mode 100644 tests/query/stateless/test_yield.py create mode 100644 tests/requirements.txt create mode 100644 tests/tck/__init__.py create mode 100644 tests/tck/cluster/Example.feature create mode 100644 tests/tck/cluster/terminate.feature create mode 100644 tests/tck/conftest.py create mode 100644 tests/tck/features/admin/Authentication.feature create mode 100644 tests/tck/features/admin/Hosts.feature create mode 100644 tests/tck/features/admin/Sessions.feature create mode 100644 tests/tck/features/aggregate/Agg.feature create mode 100644 tests/tck/features/basic/Parser.feature create mode 100644 tests/tck/features/basic/data.feature create mode 100644 tests/tck/features/bugfix/AggPatternExpression.feature create mode 100644 tests/tck/features/bugfix/AliasTypeDeduce.feature create mode 100644 tests/tck/features/bugfix/ArgumentPlanNodeDep.feature create mode 100644 tests/tck/features/bugfix/BoolCheckingInExpr.feature create mode 100644 tests/tck/features/bugfix/BuildPathMistake.feature create mode 100644 tests/tck/features/bugfix/CompareDate.feature create mode 100644 tests/tck/features/bugfix/ContainsFilter.feature create mode 100644 tests/tck/features/bugfix/CrashWhenNullPathExpr.feature create mode 100644 tests/tck/features/bugfix/CreateDurationAsDefaultValue.feature create mode 100644 tests/tck/features/bugfix/DropRootUser.feature create mode 100644 tests/tck/features/bugfix/DupAliasInMatch.feature create mode 100644 tests/tck/features/bugfix/ExtractFilter.feature create mode 100644 tests/tck/features/bugfix/FindStartError.feature create mode 100644 tests/tck/features/bugfix/FixIterCrash.feature create mode 100644 tests/tck/features/bugfix/GetEdgesTransformLimitRule.feature create mode 100644 tests/tck/features/bugfix/InnerVar.feature create mode 100644 tests/tck/features/bugfix/InsertMismatchedTypeDateTime.feature create mode 100644 tests/tck/features/bugfix/LackFilterGetEdges.feature create mode 100644 tests/tck/features/bugfix/ListSlicingWithNull.feature create mode 100644 tests/tck/features/bugfix/LookupIn.feature create mode 100644 tests/tck/features/bugfix/MTSafeConcurrencyVariables.feature create mode 100644 tests/tck/features/bugfix/MatchCrash.feature create mode 100644 tests/tck/features/bugfix/MatchJoinOnEdge.feature create mode 100644 tests/tck/features/bugfix/MatchNodeLabelFilter.feature create mode 100644 tests/tck/features/bugfix/MatchNotFilterTheUndeclaredTag.feature create mode 100644 tests/tck/features/bugfix/MatchReturnEmptyTag.feature create mode 100644 tests/tck/features/bugfix/MatchUsedInPipe.feature create mode 100644 tests/tck/features/bugfix/NaNInfinityFloat.feature create mode 100644 tests/tck/features/bugfix/NullInputOfLength.feature create mode 100644 tests/tck/features/bugfix/NullProp.feature create mode 100644 tests/tck/features/bugfix/PredicationPushDownInGo.feature create mode 100644 tests/tck/features/bugfix/PrunePropertiesError.feature create mode 100644 tests/tck/features/bugfix/PushFilterDownProject.feature create mode 100644 tests/tck/features/bugfix/RewriteEdgeFilter.feature create mode 100644 tests/tck/features/bugfix/RoundFloat.feature create mode 100644 tests/tck/features/bugfix/StringFlaotAddition.feature create mode 100644 tests/tck/features/bugfix/SubgraphBeforePipe.feature create mode 100644 tests/tck/features/bugfix/SubscriptInUpdate.feature create mode 100644 tests/tck/features/bugfix/SyntaxErrorCrash.feature create mode 100644 tests/tck/features/bugfix/TestYieldConstantAfterPipe.feature create mode 100644 tests/tck/features/bugfix/TimeDefaultValue.feature create mode 100644 tests/tck/features/bugfix/TruncatedStringIndex.feature create mode 100644 tests/tck/features/bugfix/TypoError.feature create mode 100644 tests/tck/features/bugfix/UseSpaceAndMatch.feature create mode 100644 tests/tck/features/bugfix/VariableExpression.feature create mode 100644 tests/tck/features/bugfix/VidInvalidLength.feature create mode 100644 tests/tck/features/ddl/Ddl.feature create mode 100644 tests/tck/features/delete/DeleteEdge.IntVid.feature create mode 100644 tests/tck/features/delete/DeleteEdge.feature create mode 100644 tests/tck/features/delete/DeleteTag.IntVid.feature create mode 100644 tests/tck/features/delete/DeleteTag.feature create mode 100644 tests/tck/features/delete/DeleteVertex.IntVid.feature create mode 100644 tests/tck/features/delete/DeleteVertex.feature create mode 100644 tests/tck/features/delete/DeleteVertexWithoutEdge.feature create mode 100644 tests/tck/features/explain/ExplainAndProfile.feature create mode 100644 tests/tck/features/expression/Attribute.feature create mode 100644 tests/tck/features/expression/Attribute1.feature create mode 100644 tests/tck/features/expression/BugFixWithngdata.feature create mode 100644 tests/tck/features/expression/Case.feature create mode 100644 tests/tck/features/expression/Depth.feature create mode 100644 tests/tck/features/expression/EndsWith.feature create mode 100644 tests/tck/features/expression/FixedString.feature create mode 100644 tests/tck/features/expression/FunctionCall.feature create mode 100644 tests/tck/features/expression/In.feature create mode 100644 tests/tck/features/expression/LabelExpr.feature create mode 100644 tests/tck/features/expression/ListComprehension.feature create mode 100644 tests/tck/features/expression/ListRangeSubscript.feature create mode 100644 tests/tck/features/expression/LogicalExpression.feature create mode 100644 tests/tck/features/expression/NotIn.feature create mode 100644 tests/tck/features/expression/Null.feature create mode 100644 tests/tck/features/expression/Predicate.feature create mode 100644 tests/tck/features/expression/Reduce.feature create mode 100644 tests/tck/features/expression/Regex.feature create mode 100644 tests/tck/features/expression/RelationalExpr.feature create mode 100644 tests/tck/features/expression/StartsWith.feature create mode 100644 tests/tck/features/expression/TimeComparison.feature create mode 100644 tests/tck/features/expression/TimeComputation.feature create mode 100644 tests/tck/features/expression/UnaryExpr.feature create mode 100644 tests/tck/features/expression/function/Mathematical.feature create mode 100644 tests/tck/features/expression/function/TypeConversion.feature create mode 100644 tests/tck/features/fetch/FetchEdges.intVid.feature create mode 100644 tests/tck/features/fetch/FetchEdges.strVid.feature create mode 100644 tests/tck/features/fetch/FetchEmpty.feature create mode 100644 tests/tck/features/fetch/FetchVertices.intVid.feature create mode 100644 tests/tck/features/fetch/FetchVertices.strVid.feature create mode 100644 tests/tck/features/fulltext_index/FulltextIndexScan.feature create mode 100644 tests/tck/features/fulltext_index/FultextIndexDDL.feature create mode 100644 tests/tck/features/function/coalesce.feature create mode 100644 tests/tck/features/function/json_extract.feature create mode 100644 tests/tck/features/function/round.feature create mode 100644 tests/tck/features/geo/GeoBase.feature create mode 100644 tests/tck/features/go/GO.IntVid.feature create mode 100644 tests/tck/features/go/GO.feature create mode 100644 tests/tck/features/go/GoYieldVertexEdge.feature create mode 100644 tests/tck/features/go/GroupbyLimit.IntVid.feature create mode 100644 tests/tck/features/go/GroupbyLimit.feature create mode 100644 tests/tck/features/go/Orderby.feature create mode 100644 tests/tck/features/go/SampleLimit.feature create mode 100644 tests/tck/features/go/SampleLimit.intVid.feature create mode 100644 tests/tck/features/go/SimpleCase.feature create mode 100644 tests/tck/features/index/Index.IntVid.feature create mode 100644 tests/tck/features/index/Index.feature create mode 100644 tests/tck/features/index/TagEdgeIndex.feature create mode 100644 tests/tck/features/insert/BoundCheck.feature create mode 100644 tests/tck/features/insert/Insert.IntVid.feature create mode 100644 tests/tck/features/insert/Insert.feature create mode 100644 tests/tck/features/insert/InsertEdgeOnDiffParts.feature create mode 100644 tests/tck/features/insert/InsertIfNotExists.feature create mode 100644 tests/tck/features/insert/insertVertexOnly.feature create mode 100644 tests/tck/features/lookup/ByIndex.feature create mode 100644 tests/tck/features/lookup/ByIndex.intVid.feature create mode 100644 tests/tck/features/lookup/EdgeIndexFullScan.feature create mode 100644 tests/tck/features/lookup/LookUp.IntVid.feature create mode 100644 tests/tck/features/lookup/LookUp.feature create mode 100644 tests/tck/features/lookup/LookUpLimit.feature create mode 100644 tests/tck/features/lookup/LookUpTopN.feature create mode 100644 tests/tck/features/lookup/LookupEdge.feature create mode 100644 tests/tck/features/lookup/LookupEdge2.feature create mode 100644 tests/tck/features/lookup/LookupTag.feature create mode 100644 tests/tck/features/lookup/LookupTag2.feature create mode 100644 tests/tck/features/lookup/Output.feature create mode 100644 tests/tck/features/lookup/Output.intVid.feature create mode 100644 tests/tck/features/lookup/TagIndexFullScan.feature create mode 100644 tests/tck/features/lookup/WithYield.feature create mode 100644 tests/tck/features/lookup/WithYield.intVid.feature create mode 100644 tests/tck/features/match/AllShortestPaths.feature create mode 100644 tests/tck/features/match/Base.IntVid.feature create mode 100644 tests/tck/features/match/Base.feature create mode 100644 tests/tck/features/match/IndexSelecting.feature create mode 100644 tests/tck/features/match/MatchById.IntVid.feature create mode 100644 tests/tck/features/match/MatchById.feature create mode 100644 tests/tck/features/match/MatchByVariable.feature create mode 100644 tests/tck/features/match/MatchGroupBy.feature create mode 100644 tests/tck/features/match/MultiLineMultiQueryParts.feature create mode 100644 tests/tck/features/match/MultiQueryParts.feature create mode 100644 tests/tck/features/match/Path.feature create mode 100644 tests/tck/features/match/PathExpr.feature create mode 100644 tests/tck/features/match/PathExprRefLocalVariable.feature create mode 100644 tests/tck/features/match/PipeAndVariable.feature create mode 100644 tests/tck/features/match/RedefinedNode.feature create mode 100644 tests/tck/features/match/SameTagPropname.feature create mode 100644 tests/tck/features/match/Scan.feature create mode 100644 tests/tck/features/match/SeekByEdge.feature create mode 100644 tests/tck/features/match/SeekById.feature create mode 100644 tests/tck/features/match/SeekById.intVid.feature create mode 100644 tests/tck/features/match/SeekByTag.feature create mode 100644 tests/tck/features/match/SeekByTag.intVid.feature create mode 100644 tests/tck/features/match/SelfReflectiveEdges.feature create mode 100644 tests/tck/features/match/SingleShorestPath.feature create mode 100644 tests/tck/features/match/StartFromAnyNode.IntVid.feature create mode 100644 tests/tck/features/match/StartFromAnyNode.feature create mode 100644 tests/tck/features/match/Unwind.feature create mode 100644 tests/tck/features/match/VariableLengthPattern.feature create mode 100644 tests/tck/features/match/VariableLengthPattern.intVid.feature create mode 100644 tests/tck/features/match/With.feature create mode 100644 tests/tck/features/match/ZeroStep.feature create mode 100644 tests/tck/features/match/ZeroStep.intVid.feature create mode 100644 tests/tck/features/match/ZeroStep2.feature create mode 100644 tests/tck/features/mutate/ClearSpace.feature create mode 100644 tests/tck/features/mutate/InsertDurationType.feature create mode 100644 tests/tck/features/mutate/InsertWithTimeType.feature create mode 100644 tests/tck/features/optimizer/CasesUsingTestSpace.feature create mode 100644 tests/tck/features/optimizer/CollapseProjectRule.feature create mode 100644 tests/tck/features/optimizer/CombineFilterRule.feature create mode 100644 tests/tck/features/optimizer/EliminateAppendVerticesRule.feature create mode 100644 tests/tck/features/optimizer/ElimintateInvalidProp.feature create mode 100644 tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature create mode 100644 tests/tck/features/optimizer/IndexScanRule.feature create mode 100644 tests/tck/features/optimizer/MergeGetNbrsDedupProjectRule.feature create mode 100644 tests/tck/features/optimizer/MergeGetVerticesDedupProjectRule.feature create mode 100644 tests/tck/features/optimizer/PrunePropertiesRule.feature create mode 100644 tests/tck/features/optimizer/PushEFilterDownRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownAggregateRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownBugFixes.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownCrossJoinRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownExpandAllRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownHashInnerJoinRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownHashLeftJoinRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownNodeRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownProjectRule.feature create mode 100644 tests/tck/features/optimizer/PushFilterDownTraverseRule.feature create mode 100644 tests/tck/features/optimizer/PushLimitDownExpandAllRule.feature create mode 100644 tests/tck/features/optimizer/PushLimitDownProjectRule.feature create mode 100644 tests/tck/features/optimizer/PushLimitDownScanEdgesRule.feature create mode 100644 tests/tck/features/optimizer/PushLimitDownScanVerticesRule.feature create mode 100644 tests/tck/features/optimizer/PushSampleDownRule.feature create mode 100644 tests/tck/features/optimizer/RemoveAppendVerticesBelowJoinRule.feature create mode 100644 tests/tck/features/optimizer/RemoveUselessProjectRule.feature create mode 100644 tests/tck/features/optimizer/TopNRule.feature create mode 100644 tests/tck/features/parser/Example.feature create mode 100644 tests/tck/features/parser/nebula.feature create mode 100644 tests/tck/features/path/AllPath.IntVid.feature create mode 100644 tests/tck/features/path/AllPath.feature create mode 100644 tests/tck/features/path/NoLoop.IntVid.feature create mode 100644 tests/tck/features/path/NoLoop.feature create mode 100644 tests/tck/features/path/ShortestPath.IntVid.feature create mode 100644 tests/tck/features/path/ShortestPath.feature create mode 100644 tests/tck/features/path/singleShortestPath.feature create mode 100644 tests/tck/features/schema/Comment.feature create mode 100644 tests/tck/features/schema/CreateSpaceAs.feature create mode 100644 tests/tck/features/schema/Schema.feature create mode 100644 tests/tck/features/set/Set.IntVid.feature create mode 100644 tests/tck/features/set/Set.feature create mode 100644 tests/tck/features/subgraph/subgraph.IntVid.feature create mode 100644 tests/tck/features/subgraph/subgraph.feature create mode 100644 tests/tck/features/subgraph/subgraphWithFilter.feature create mode 100644 tests/tck/features/ttl/TTL.feature create mode 100644 tests/tck/features/ttl/TTL2.feature create mode 100644 tests/tck/features/update/Update.IntVid.feature create mode 100644 tests/tck/features/update/Update.feature create mode 100644 tests/tck/features/user/User.feature create mode 100644 tests/tck/features/verify_client_version/VerifyClientVersion.feature create mode 100644 tests/tck/features/yield/NoSpaceChosen.feature create mode 100644 tests/tck/features/yield/join.feature create mode 100644 tests/tck/features/yield/parameter.feature create mode 100644 tests/tck/features/yield/return.feature create mode 100644 tests/tck/features/yield/yield.IntVid.feature create mode 100644 tests/tck/features/yield/yield.feature create mode 100644 tests/tck/job/Job.feature create mode 100644 tests/tck/ldbc/business_intelligence_workload/Read.feature create mode 100644 tests/tck/ldbc/interactive_workload/ComplexReads.feature create mode 100644 tests/tck/ldbc/interactive_workload/ShortReads.feature create mode 100644 tests/tck/openCypher/features/expressions/list/List2.feature create mode 100644 tests/tck/openCypher/features/expressions/map/Map1.feature create mode 100644 tests/tck/openCypher/features/expressions/map/Map2.feature create mode 100644 tests/tck/openCypher/features/expressions/path/Path1.feature create mode 100644 tests/tck/openCypher/features/expressions/path/Path2.feature create mode 100644 tests/tck/openCypher/features/expressions/path/Path3.feature create mode 100644 tests/tck/slowquery/KillSlowQueryBaseTest.feature create mode 100644 tests/tck/slowquery/KillSlowQueryViaDiffrentService.feature create mode 100644 tests/tck/slowquery/KillSlowQueryViaSameService.feature create mode 100644 tests/tck/slowquery/PermissionViaDifferentService.feature create mode 100644 tests/tck/slowquery/PermissionViaSameService.feature create mode 100644 tests/tck/steps/__init__.py create mode 100644 tests/tck/steps/conftest.py create mode 100644 tests/tck/steps/test_cluster.py create mode 100644 tests/tck/steps/test_jobs.py create mode 100644 tests/tck/steps/test_kill_permission_via_different_service.py create mode 100644 tests/tck/steps/test_kill_permission_via_same_service.py create mode 100644 tests/tck/steps/test_kill_slow_query_base_test.py create mode 100644 tests/tck/steps/test_kill_slow_query_via_different_service.py create mode 100644 tests/tck/steps/test_kill_slow_query_via_same_service.py create mode 100644 tests/tck/steps/test_ldbc.py create mode 100644 tests/tck/steps/test_tck.py create mode 100644 tests/tck/utils/.gitignore create mode 100644 tests/tck/utils/__init__.py create mode 100644 tests/tck/utils/mmh2.py create mode 100644 tests/tck/utils/nbv.py create mode 100644 tests/tck/utils/table.py create mode 100644 third-party/cxx-compiler-abi-version.sh create mode 100644 third-party/install-cmake.sh create mode 100644 third-party/install-gcc.sh create mode 100644 third-party/install-third-party.sh create mode 100644 udf/.gitignore create mode 100644 udf/standard_deviation.cpp create mode 100644 udf/standard_deviation.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..01ad18c --- /dev/null +++ b/.clang-format @@ -0,0 +1,175 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# +# This is the output of clang-format-10.0.0 --style=google --dump-config, +# except for changes mentioned below. +# We have locked the version of clang-format in order to avoid inconsistencies +# in the format caused by developers using different clang-format versions. +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 0 + - Regex: '^<.*\.h>' + Priority: 1 + SortPriority: 0 + - Regex: '^<.*' + Priority: 2 + SortPriority: 0 + - Regex: '.*' + Priority: 3 + SortPriority: 0 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentGotoLabels: false +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: c++17 +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..5342022 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,141 @@ +Checks: '-*,clang-diagnostic-*,clang-analyzer-*,-misc-unused-parameters, + + bugprone-argument-comment, + bugprone-assert-side-effect, + bugprone-bad-signal-to-kill-thread, + bugprone-bool-pointer-implicit-conversion, + bugprone-branch-clone, + bugprone-copy-constructor-init, + bugprone-dangling-handle, + bugprone-dynamic-static-initializers, + bugprone-fold-init-type, + bugprone-forward-declaration-namespace, + bugprone-forwarding-reference-overload, + bugprone-inaccurate-erase, + bugprone-incorrect-roundings, + bugprone-infinite-loop, + bugprone-integer-division, + bugprone-lambda-function-name, + bugprone-macro-parentheses, + bugprone-macro-repeated-side-effects, + bugprone-misplaced-operator-in-strlen-in-alloc, + bugprone-misplaced-pointer-arithmetic-in-alloc, + bugprone-misplaced-widening-cast, + bugprone-move-forwarding-reference, + bugprone-multiple-statement-macro, + bugprone-not-null-terminated-result, + bugprone-parent-virtual-call, + bugprone-posix-return, + bugprone-reserved-identifier, + bugprone-signed-char-misuse, + bugprone-sizeof-container, + bugprone-sizeof-expression, + bugprone-spuriously-wake-up-functions, + bugprone-string-constructor, + bugprone-string-integer-assignment, + bugprone-string-literal-with-embedded-nul, + bugprone-suspicious-enum-usage, + bugprone-suspicious-include, + bugprone-suspicious-memset-usage, + bugprone-suspicious-missing-comma, + bugprone-suspicious-semicolon, + bugprone-suspicious-string-compare, + bugprone-swapped-arguments, + bugprone-terminating-continue, + bugprone-throw-keyword-missing, + bugprone-too-small-loop-variable, + bugprone-undefined-memory-manipulation, + bugprone-undelegated-constructor, + bugprone-unhandled-self-assignment, + bugprone-unused-raii, + bugprone-unused-return-value, + bugprone-use-after-move, + + google-build-explicit-make-pair, + google-build-namespaces, + google-build-using-namespace, + google-default-arguments, + google-explicit-constructor, + google-global-names-in-headers, + google-objc-avoid-nsobject-new, + google-objc-function-naming, + google-objc-global-variable-declaration, + google-readability-avoid-underscore-in-googletest-name, + google-readability-casting, + google-runtime-int, + google-runtime-operator, + google-upgrade-googletest-case, + + modernize-avoid-bind, + -modernize-concat-nested-namespaces, + modernize-deprecated-headers, + modernize-deprecated-ios-base-aliases, + modernize-loop-convert, + modernize-make-shared, + modernize-make-unique, + modernize-raw-string-literal, + modernize-redundant-void-arg, + modernize-replace-auto-ptr, + modernize-replace-random-shuffle, + modernize-return-braced-init-list, + modernize-shrink-to-fit, + modernize-unary-static-assert, + modernize-use-bool-literals, + modernize-use-default-member-init, + modernize-use-emplace, + modernize-use-equals-default, + modernize-use-equals-delete, + -modernize-use-nodiscard, + modernize-use-nullptr, + modernize-use-override, + -modernize-use-trailing-return-type, + modernize-use-transparent-functors, + modernize-use-using, + + performance-faster-string-find, + performance-for-range-copy, + performance-implicit-conversion-in-loop, + performance-inefficient-algorithm, + performance-inefficient-string-concatenation, + performance-inefficient-vector-operation, + performance-move-constructor-init, + performance-no-automatic-move, + performance-trivially-destructible, + performance-type-promotion-in-math-fn, + performance-unnecessary-copy-initialization, +' + +HeaderFilterRegex: '/(?!third-party)|(?!interface)/' +AnalyzeTemporaryDtors: false +FormatStyle: 'file' +WarningsAsErrors: '*' +CheckOptions: + - key: cert-dcl16-c.NewSuffixes + value: 'L;LL;LU;LLU' + - key: cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic + value: '1' + - key: google-readability-braces-around-statements.ShortStatementLines + value: '1' + - key: google-readability-function-size.StatementThreshold + value: '800' + - key: google-readability-namespace-comments.ShortNamespaceLines + value: '10' + - key: google-readability-namespace-comments.SpacesBeforeComments + value: '2' + - key: modernize-loop-convert.MaxCopySize + value: '16' + - key: modernize-loop-convert.MinConfidence + value: reasonable + - key: modernize-loop-convert.NamingStyle + value: CamelCase + - key: modernize-pass-by-value.IncludeStyle + value: llvm + - key: modernize-replace-auto-ptr.IncludeStyle + value: llvm + - key: modernize-use-nullptr.NullMacros + value: 'NULL' + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantPrefix + value: k + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e94e807 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,38 @@ +*.md +docs/ + +*.swp +*.so +*.DS_Store + +# build +pkg-build/ +build/ +_build/ +_build.log +_install/ +install/ +install_manifest.txt +cmake-build-* + +# ccls +.ccls +.ccls-cache +compile_commands.json + +.ccache + +src/common/base/Base.h.gch +gen-* + +CPackConfig.cmake +CPackSourceConfig.cmake +CMakeCache.txt +Makefile +cmake_install.cmake +CTestTestfile.cmake +CMakeFiles/ +Testing/ +target/ +cluster.id +pids/ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..cee82ec --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +#Require an approved review in PRs including files with a designated code owner. +/conf/ @vesoft-inc/tech-committee-reviewers +/src/kvstore/raftex/ @critical27 +/cmake/ @vesoft-inc/tech-committee-reviewers +*.thrift @vesoft-inc/tech-committee-reviewers +*.yy @dutor @codesigner diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..ae5f80c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,39 @@ +--- +name: Bug Report +about: I want to report a bug. +title: '' +labels: type/bug +assignees: '' + +--- + +**Please check the FAQ documentation before raising an issue** + + + +**Describe the bug (__required__)** + + + +**Your Environments (__required__)** + +* OS: `uname -a` +* Compiler: `g++ --version` or `clang++ --version` +* CPU: `lscpu` +* Commit id (e.g. `a3ffc7d8`) + +**How To Reproduce(__required__)** + +Steps to reproduce the behavior: + +1. Step 1 +2. Step 2 +3. Step 3 + +**Expected behavior** + + + +**Additional context** + + diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md new file mode 100644 index 0000000..f59770f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -0,0 +1,16 @@ +--- +name: Enhancement +about: Suggest an enhancement to make the code neat or more efficient. +title: '' +labels: type/enhancement +assignees: '' +--- + +**Introduction** + + +**Contents** + + +**Related work** + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..a019e6a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature Request +about: Suggest a feature for this project. +title: '' +labels: type/feature req +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** + + +**Describe the solution you'd like** + + +**Describe alternatives you've considered** + + +**Additional context** + diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..3017396 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,18 @@ +--- +name: Question +about: I want to ask a question. +labels: question +--- + +**General Question** + + diff --git a/.github/Security.md b/.github/Security.md new file mode 100644 index 0000000..67d9a31 --- /dev/null +++ b/.github/Security.md @@ -0,0 +1,30 @@ +# Vulnerability Assessment Reporting: Beginner's Guide + +NebulaGraph is dedicated to offering stable and secure data services. We recognize the importance of community contributions to our security posture and invite you to report any vulnerabilities you may discover. + +Should you identify a potential security vulnerability within NebulaGraph or encounter a security incident, we urge you to get in touch with our team. + +For efficient communication, please send your findings to (security@vesoft.com) with the following details: + +* Vulnerability name (*): Provide a clear, concise title. Include a CVE identifier if available. +* Description of the security issue (*): Elaborate on the security concern. +* Impacted Components (*): Specify affected modules and their version numbers. +* Reproduction Steps (*): Detail the process to verify the vulnerability. +* Suggested Remediation: Offer potential solutions if possible. +* Contact information (*): + * Name + * Email + * Organization + * Consent for Identity Disclosure + + > Fields marked with an asterisk `(*)` are mandatory. + +## Response Protocol + +The NebulaGraph security team pledges to acknowledge receipt of your report via email within one working day. + +Post-resolution, we will promptly notify you and extend our gratitude for your invaluable assistance in enhancing NebulaGraph's security. + +Disclosure of the vulnerability will be withheld until an official patch is released. We appreciate your discretion and cooperation. + +We thank you for your attention to these guidelines and look forward to your participation in securing NebulaGraph. diff --git a/.github/actions/tagname-action/README.md b/.github/actions/tagname-action/README.md new file mode 100644 index 0000000..cc691fd --- /dev/null +++ b/.github/actions/tagname-action/README.md @@ -0,0 +1,25 @@ +# Extract tag name + +Extract tag information from release branch + +## Outputs + +### `tag` + +tag name + +### `tagnum` + +tag number + +## Example usage + +```yaml +- uses: ./.github/actions/tagname-action + id: tag + +- name: Other step + run: | + echo ${{ steps.tag.outputs.tag }} + echo ${{ steps.tag.outputs.tagnum }} +``` diff --git a/.github/actions/tagname-action/action.yml b/.github/actions/tagname-action/action.yml new file mode 100644 index 0000000..489f915 --- /dev/null +++ b/.github/actions/tagname-action/action.yml @@ -0,0 +1,24 @@ +name: "Extract tag information" +description: "Extract tag information" +outputs: + tag: + description: "tag name" + value: ${{ steps.tag.outputs.tag }} + tagnum: + description: "tag number" + value: ${{ steps.tag.outputs.tagnum }} + majorver: + description: "major version" + value: ${{ steps.tag.outputs.majorver }} +runs: + using: "composite" + steps: + - id: tag + run: | + tag=$(echo ${{ github.ref }} | rev | cut -d/ -f1 | rev) + tagnum=$(echo $tag | sed "s/^v//") + majorver=$(echo $tag | cut -d "." -f 1) + echo "tag=$tag" >> $GITHUB_OUTPUT + echo "tagnum=$tagnum" >> $GITHUB_OUTPUT + echo "majorver=$majorver" >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/actions/upload-assets-action/README.md b/.github/actions/upload-assets-action/README.md new file mode 100644 index 0000000..62cc71d --- /dev/null +++ b/.github/actions/upload-assets-action/README.md @@ -0,0 +1,22 @@ +# Upload file to release assets + +Upload file to release assets + +## Inputs + +### `tag` + +**Required** tag name of release branch. Default `${{ github.ref }}`. + +### `asset-path` + +**Required** file path to be uploaded. Default `''`. + +## Example usage + +```yaml +uses: ./.github/actions/upload-assets-action +with: + asset-path: build/cpack_output + tag: ${{ steps.tag.outputs.tag }} +``` diff --git a/.github/actions/upload-assets-action/action.yml b/.github/actions/upload-assets-action/action.yml new file mode 100644 index 0000000..58f80e9 --- /dev/null +++ b/.github/actions/upload-assets-action/action.yml @@ -0,0 +1,40 @@ +name: 'Upload release assets' +description: 'Upload file to release assets' +inputs: + tag: + description: 'git tag' + required: true + default: ${{ github.ref }} + asset-path: + description: 'file path to be uploaded' + required: true + default: '' +runs: + using: "composite" + steps: + - run: | + GH_RELEASE="https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ inputs.tag }}" + upload_url=$(curl -s --request GET --url $GH_RELEASE | grep -oP '(?<="upload_url": ")[^"]*' | cut -d'{' -f1) + j=0 + if [ -d ${{ inputs.asset-path }} ]; then + for filename in `ls ${{ inputs.asset-path }}`; + do + folder_list[j]=${{ inputs.asset-path }}/$filename + j=`expr $j + 1` + done + else + folder_list[0]=${{ inputs.asset-path }} + fi + echo "Uploading asset... " + for filepath in ${folder_list[@]}; + do + filename=$(basename "${filepath}") + content_type=$(file -b --mime-type ${filepath}) + curl --silent \ + --request POST \ + --url "$upload_url?name=$filename" \ + --header "authorization: Bearer ${{ github.token }}" \ + --header "content-type: $content_type" \ + --data-binary @"${filepath}" + done + shell: bash diff --git a/.github/actions/upload-to-oss-action/README.md b/.github/actions/upload-to-oss-action/README.md new file mode 100644 index 0000000..9afc308 --- /dev/null +++ b/.github/actions/upload-to-oss-action/README.md @@ -0,0 +1,42 @@ +# Upload file to oss + +Upload file to oss + +## Inputs + +### `key-id` + +**Required** access key ID of oss. Default `''`. + +### `key-secret` + +**Required** access key secret of oss. Default `''`. + +### `endpoint` + +**Required** endpoint of oss. Default `''`. + +### `bucket` + +**Required** bucket of oss. Default `''`. + +### `asset-path` + +**Required** file path to be uploaded. Default `''`. + +### `target-path` + +**Required** path where the oss object is stored. Default `''`. + +## Example usage + +```yaml +uses: ./.github/actions/upload-to-oss-action +with: + key-id: ${{ secrets.OSS_ID }} + key-secret: ${{ secrets.OSS_SECRET }} + endpoint: ${{ secrets.OSS_ENDPOINT }} + bucket: nebula-graph + asset-path: build/cpack_output + target-path: package/v2-nightly/${{ steps.vars.outputs.subdir }} +``` diff --git a/.github/actions/upload-to-oss-action/action.yml b/.github/actions/upload-to-oss-action/action.yml new file mode 100644 index 0000000..467acc2 --- /dev/null +++ b/.github/actions/upload-to-oss-action/action.yml @@ -0,0 +1,50 @@ +name: 'Upload to oss' +description: 'Upload files to oss' +inputs: + key-id: + description: 'access key ID' + required: true + default: '' + key-secret: + description: 'access key secret' + required: true + default: '' + endpoint: + description: 'endpooint' + required: true + default: '' + bucket: + description: 'bucket' + required: true + default: '' + asset-path: + description: 'file path to be uploaded' + required: true + default: '' + target-path: + description: 'file path stored on the OSS' + required: true + default: '' +runs: + using: "composite" + steps: + - run: | + j=0 + if [ -d ${{ inputs.asset-path }} ]; then + for filename in `ls ${{ inputs.asset-path }}`; + do + folder_list[j]=${{ inputs.asset-path }}/$filename + j=`expr $j + 1` + done + else + folder_list[0]=${{ inputs.asset-path }} + fi + echo "Uploading to oss... " + for filepath in ${folder_list[@]}; + do + ossutil64 -e ${{ inputs.endpoint}} \ + -i ${{ inputs.key-id }} \ + -k ${{ inputs.key-secret }} \ + -f cp $filepath oss://${{ inputs.bucket }}/${{ inputs.target-path }}/$(basename ${filepath}) + done + shell: bash diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..bbd347b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,42 @@ + + +## What type of PR is this? +- [ ] bug +- [ ] feature +- [ ] enhancement + +## What problem(s) does this PR solve? +#### Issue(s) number: + +#### Description: + + +## How do you solve it? + + + +## Special notes for your reviewer, ex. impact of this fix, design document, etc: + + + +## Checklist: +Tests: +- [ ] Unit test(positive and negative cases) +- [ ] Function test +- [ ] Performance test +- [ ] N/A + +Affects: +- [ ] Documentation affected (Please add the label if documentation needs to be modified.) +- [ ] Incompatibility (If it breaks the compatibility, please describe it and add the label.） +- [ ] If it's needed to cherry-pick (If cherry-pick to some branches is required, please label the destination version(s).) +- [ ] Performance impacted: Consumes more CPU/Memory + + +## Release notes: + +Please confirm whether to be reflected in release notes and how to describe: +> ex. Fixed the bug ..... diff --git a/.github/workflows/auto_cherry_pick.yml b/.github/workflows/auto_cherry_pick.yml new file mode 100644 index 0000000..2b20f18 --- /dev/null +++ b/.github/workflows/auto_cherry_pick.yml @@ -0,0 +1,38 @@ +name: Auto Cherry Pick +on: + workflow_dispatch: + inputs: + pr_label: + description: "Cherry pick label such as 'cherry-pick-v3.2'" + required: true + +defaults: + run: + shell: bash + +jobs: + auto-cherry-pick: + #if: ${{ startsWith(github.event.pull_request.labels.*.name, 'cherry-pick-') && github.event.pull_request.merged == true }} + runs-on: [self-hosted, nebula-fast] + container: + image: reg.vesoft-inc.com/dashboard/dashboard-dev:centos7 + credentials: + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} + steps: + - name: keep workspace empty + run: | + rm -rf * + - name: git config set + env: + GH_BOT_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global url."https://${GH_BOT_PAT}:x-oauth-basic@github.com/".insteadOf "https://github.com/" + export GOPRIVATE="github.com/vesoft-inc" + - name: auto cherry pick + uses: xigongdaEricyang/cherry-pick-robot@with-python + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + pr_label: ${{ github.event.inputs.pr_label }} + #pr_num: ${{ github.event.pull_request.number }} + auto_merge: true diff --git a/.github/workflows/check_label.yml b/.github/workflows/check_label.yml new file mode 100644 index 0000000..e410727 --- /dev/null +++ b/.github/workflows/check_label.yml @@ -0,0 +1,22 @@ +name: Auto label + +on: + issues: + types: + - reopened + - opened + - labeled + - unlabeled + - closed + +env: + GH_PAT: ${{ secrets.GITHUB_TOKEN }} + EVENT: ${{ toJSON(github.event)}} + EVENT_NAME: ${{ github.event_name}} + +jobs: + sync: + name: auto label + runs-on: ubuntu-latest + steps: + - uses: HarrisChu/auto_label@v1 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..d85086f --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,69 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + schedule: + - cron: '20 21 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: + - 'cpp' + - 'python' + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..db0f7a4 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,401 @@ +name: nightly + +on: + schedule: + - cron: "0 18 * * *" + +concurrency: + group: nightly + cancel-in-progress: true + +defaults: + run: + shell: bash +env: + OSS_DIR: minio/nightly-build/nebula-graph + +jobs: + package: + name: build package + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - ubuntu1604 + - ubuntu1804 + - ubuntu2004 + # - centos6 + - centos7 + - centos8 + container: + image: vesoft/nebula-dev:${{ matrix.os }} + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: "^1.16.7" + - name: package + run: ./package/package.sh -t Release -r ON -p OFF -s FALSE -k OFF + - name: output some vars + id: vars + env: + SHA_EXT: sha256sum.txt + run: | + find pkg-build/cpack_output -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \; + subdir=$(date -u +%Y.%m.%d) + echo "subdir=$subdir" >> $GITHUB_OUTPUT + # - uses: actions/upload-artifact@v3 + # with: + # name: ${{ matrix.os }}-nightly + # path: pkg-build/cpack_output + - uses: vesoft-inc/.github/actions/setup-minio@master + with: + minio_url: ${{ secrets.MINIO_ENDPOINT }} + access_key: ${{ secrets.MINIO_KEY }} + secret_key: ${{ secrets.MINIO_SECRET }} + - name: Copy dir to MinIO + run: mc cp -r pkg-build/cpack_output/ ${{ env.OSS_DIR }}/${{ steps.vars.outputs.subdir }}/ + + + docker: + name: build docker image + needs: package + runs-on: [self-hosted, nebula] + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + - uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + vesoft/nebula-graphd:nightly + target: graphd + cache-to: type=local,dest=/tmp/buildx-cache,mode=max + push: true + - uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + vesoft/nebula-storaged:nightly + target: storaged + cache-from: type=local,src=/tmp/buildx-cache + push: true + - uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + vesoft/nebula-metad:nightly + target: metad + cache-from: type=local,src=/tmp/buildx-cache + push: true + - uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + vesoft/nebula-tools:nightly + target: tools + cache-from: type=local,src=/tmp/buildx-cache + push: true + - name: delete the cache + run: | + rm -rf /tmp/buildx-cache + + coverage: + name: coverage + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - centos7 + compiler: + - gcc-9.3 + container: + image: vesoft/nebula-dev:${{ matrix.os }} + env: + CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + CCACHE_MAXSIZE: 8G + volumes: + - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + options: --cap-add=SYS_PTRACE + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - name: Prepare environment + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + run: | + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ + -DCMAKE_BUILD_TYPE=Debug \ + -DENABLE_TESTING=on \ + -DENABLE_COVERAGE=on \ + -GNinja \ + -B build + - name: Make + run: | + ccache -z + ninja -j $(($(nproc)/2+1)) + ccache -s + working-directory: build/ + - name: CTest + env: + ASAN_OPTIONS: fast_unwind_on_malloc=1 + run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure + working-directory: build/ + timeout-minutes: 20 + - name: Setup cluster + run: | + make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up + working-directory: tests/ + timeout-minutes: 2 + - name: Pytest + run: | + make RM_DIR=false DEBUG=false J=8 test + working-directory: tests/ + timeout-minutes: 15 + - name: TCK + run: | + make RM_DIR=false DEBUG=false J=8 tck + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: coverage + run: | + ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include /usr/lib /opt/vesoft build/ tests/ /test /mock .lex .yy + - uses: codecov/codecov-action@v3 + with: + files: fastcov.info + fail_ci_if_error: false + - name: Upload logs + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs/ + + test: + name: Tck test + needs: package + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - ubuntu1804 + - ubuntu2004 + - centos7 + - centos8 + extra_config: + - "ENABLE_SSL=true CA_SIGNED=true QUERY_CONCURRENTLY=false" + - "ENABLE_SSL=false CA_SIGNED=false QUERY_CONCURRENTLY=true" + + container: + image: vesoft/nebula-dev:${{ matrix.os }} + services: + elasticsearch: + image: elasticsearch:7.17.7 + ports: + - 9200:9200 + - 9300:9300 + env: + discovery.type: single-node + options: >- + --health-cmd "curl elasticsearch:9200" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - name: output some vars + id: vars + env: + SHA_EXT: sha256sum.txt + run: | + subdir=$(date -u +%Y.%m.%d) + echo "subdir=$subdir" >> $GITHUB_OUTPUT + + - id: oss_package + run: | + case ${{ matrix.os }} in + ubuntu1804) + echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.ubuntu1804.amd64.tar.gz" >> $GITHUB_OUTPUT + ;; + ubuntu2004) + echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.ubuntu2004.amd64.tar.gz" >> $GITHUB_OUTPUT + ;; + centos7) + echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.el7.x86_64.tar.gz" >> $GITHUB_OUTPUT + ;; + centos8) + echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.el8.x86_64.tar.gz" >> $GITHUB_OUTPUT + ;; + esac + - name: Prepare environment + id: prepare + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + id: cmake + run: | + echo "j=8" >> $GITHUB_OUTPUT + - uses: vesoft-inc/.github/actions/setup-minio@master + with: + minio_url: ${{ secrets.MINIO_ENDPOINT }} + access_key: ${{ secrets.MINIO_KEY }} + secret_key: ${{ secrets.MINIO_SECRET }} + - name: Copy dir to MinIO + run: | + mc cp ${{ env.OSS_DIR }}/${{ steps.vars.outputs.subdir }}/${{ steps.oss_package.outputs.p }} build/ + tar zxvf build/${{ steps.oss_package.outputs.p }} -C build + d=`echo ${{ steps.oss_package.outputs.p }} | sed 's/.tar.gz//'` + mv build/${d}/* build/. + - name: Setup cluster + run: | + make CONTAINERIZED=true ${{ matrix.extra_config }} up + working-directory: tests/ + timeout-minutes: 2 + - name: Pytest + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test + working-directory: tests/ + timeout-minutes: 15 + - name: TCK + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: Upload logs + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs*/ + + standalone: + name: standalone-build-tck + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - centos7 + compiler: + - gcc-9.3 + env: + CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + CCACHE_MAXSIZE: 8G + container: + image: vesoft/nebula-dev:${{ matrix.os }} + volumes: + - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + options: --cap-add=SYS_PTRACE + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - name: Prepare environment + id: prepare + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + id: cmake + run: | + case ${{ matrix.compiler }} in + gcc-*) + case ${{ matrix.os }} in + centos7) + # build with Release type + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTING=on \ + -DENABLE_STANDALONE_VERSION=on \ + -GNinja \ + -B build + echo "::set-output name=j::8" + ;; + esac + ;; + esac + - name: Make + run: | + ccache -z + ninja -j $(nproc) + ccache -s + working-directory: build/ + - name: CTest + env: + ASAN_OPTIONS: fast_unwind_on_malloc=1 + run: | + ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -LE segment_id_test + ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -L segment_id_test + working-directory: build/ + timeout-minutes: 20 + - name: Setup Cluster + run: | + make standalone-up + working-directory: tests/ + timeout-minutes: 4 + - name: TCK + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} standalone-tck + working-directory: tests/ + timeout-minutes: 60 + - name: LDBC + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: Upload logs + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs/ diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..5fe4171 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,239 @@ +name: pull_request + +on: + pull_request: + types: [synchronize, reopened, labeled] + branches: + - master + - 'release-**' + +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + lint: + name: lint + if: ${{ contains(github.event.pull_request.labels.*.name, 'ready-for-testing') && github.event.pull_request.merged != true }} + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.base.sha }} + - uses: actions/checkout@v3 + with: + clean: false + - name: Check License Header + uses: apache/skywalking-eyes/header@main + - name: Ensure clang-format-10 is available + run: | + command -v clang-format-10 > /dev/null || (sudo apt update && sudo apt install -y clang-format-10) + - name: Cpplint + run: | + ln -snf $PWD/.linters/cpp/hooks/pre-commit.sh $PWD/.linters/cpp/pre-commit.sh + .linters/cpp/pre-commit.sh $(git --no-pager diff --diff-filter=d --name-only ${{ github.event.pull_request.base.sha }} HEAD) + - name: Format check + run: | + git diff -U0 --no-color ${{ github.event.pull_request.base.sha }} HEAD | /usr/share/clang/clang-format-10/clang-format-diff.py -p1 | tee /tmp/.clang-format-diff + [ -s /tmp/.clang-format-diff ] && exit 1 || true + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - name: Prepare Gherkin exec environ + run: make init-all -C tests + - name: Check Gherkin feature format + run: make check-and-diff -C tests + + build: + name: build + needs: lint + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - centos7 + - ubuntu2004 + compiler: + - gcc-9.3 + - clang-10 + exclude: + - os: centos7 + compiler: clang-10 + env: + CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + CCACHE_MAXSIZE: 8G + container: + image: vesoft/nebula-dev:${{ matrix.os }} + volumes: + - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }} + options: --cap-add=SYS_PTRACE + services: + elasticsearch: + image: elasticsearch:7.17.7 + ports: + - 9200:9200 + - 9300:9300 + env: + discovery.type: single-node + options: >- + --health-cmd "curl elasticsearch:9200" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - name: Prepare environment + id: prepare + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + id: cmake + run: | + case ${{ matrix.compiler }} in + gcc-*) + case ${{ matrix.os }} in + centos7) + # build with Release type + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ + -DNEBULA_USE_LINKER=mold \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTING=on \ + -GNinja \ + -B build + ;; + ubuntu2004) + # build with Debug type + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \ + -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \ + -DNEBULA_USE_LINKER=mold \ + -DCMAKE_BUILD_TYPE=Debug \ + -DENABLE_TESTING=on \ + -DENABLE_COVERAGE=on \ + -GNinja \ + -B build + ;; + esac + ;; + clang-*) + # build with Sanitizer + cmake \ + -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/clang++ \ + -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/clang \ + -DNEBULA_USE_LINKER=mold \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_MEMORY_TRACKER=off \ + -DENABLE_ASAN=on \ + -DENABLE_TESTING=on \ + -GNinja \ + -B build + ;; + esac + - name: Make + run: | + case ${{ matrix.compiler }} in + gcc-*) + case ${{ matrix.os }} in + centos7) + # build with Release type + ccache -z + ninja -j $(nproc) + ccache -s + ;; + ubuntu2004) + # build with Debug type + ccache -z + ninja -j $(nproc) + ccache -s + ;; + esac + ;; + clang-*) + # build with Sanitizer + ccache -z + ninja -j $(nproc) + ccache -s + ;; + esac + working-directory: build/ + - name: CTest + env: + ASAN_OPTIONS: fast_unwind_on_malloc=1 + run: | + ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -LE segment_id_test + ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -L segment_id_test + working-directory: build/ + timeout-minutes: 20 + - name: Setup cluster + run: | + case ${{ matrix.compiler }} in + gcc-*) + case ${{ matrix.os }} in + centos7) + # normal cluster + make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' up + ;; + ubuntu2004) + # ssl cluster + make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' ENABLE_SSL=true CA_SIGNED=true up + ;; + esac + ;; + clang-*) + # graph ssl only cluster + make CONTAINERIZED=true ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' ENABLE_SSL=false ENABLE_GRAPH_SSL=true up + ;; + esac + working-directory: tests/ + timeout-minutes: 4 + - name: Pytest + run: | + make RM_DIR=false DEBUG=false J=8 test + working-directory: tests/ + timeout-minutes: 15 + - name: TCK + run: | + make RM_DIR=false DEBUG=false ENABLE_FT_INDEX=true ES_ADDRESS='"elasticsearch":9200' J=8 tck + working-directory: tests/ + timeout-minutes: 60 + - name: LDBC + run: | + make RM_DIR=false DEBUG=false J=8 ldbc + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: coverage + if: ${{ matrix.compiler == 'gcc-9.3' && matrix.os == 'ubuntu2004' }} + run: | + ~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include /usr/lib /opt/vesoft build/ tests/ /test /mock .lex .yy + - uses: codecov/codecov-action@v3 + if: ${{ matrix.compiler == 'gcc-9.3' && matrix.os == 'ubuntu2004' }} + with: + files: fastcov.info + fail_ci_if_error: false + - name: Sanitizer + if: ${{ always() }} + run: | + exit $(grep -P "SUMMARY: AddressSanitizer: \d+ byte\(s\) leaked in \d+ allocation\(s\)\." build/server_*/logs/*stderr.log | wc -l) + - name: Upload logs + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs/ + diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml new file mode 100644 index 0000000..d52963c --- /dev/null +++ b/.github/workflows/rc.yml @@ -0,0 +1,259 @@ +name: rc + +on: + workflow_dispatch: + inputs: + version: + description: "version such as '3.1.0'" + required: true + +concurrency: + group: rc + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + OSS_DIR: minio/rc-build/nebula-graph + +jobs: + package: + name: build package + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - ubuntu1604 + - ubuntu1804 + - ubuntu2004 + - centos7 + - centos8 + container: + image: vesoft/nebula-dev:${{ matrix.os }} + env: + BUILD_DIR: ./pkg-build + CPACK_DIR: ./pkg-build/cpack_output + SYMS_DIR: ./pkg-build/symbols + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: 'stable' + cache-dependency-path: nebula-console/go.sum + - id: tag + run: echo tagnum=${{ github.event.inputs.version }} >> $GITHUB_OUTPUT + - name: package + run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} + - name: output some vars + run: | + tar zcf ${{ env.CPACK_DIR }}/nebula-graph-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./* + find ${{ env.CPACK_DIR }} -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \; + - uses: vesoft-inc/.github/actions/setup-minio@master + with: + minio_url: ${{ secrets.MINIO_ENDPOINT }} + access_key: ${{ secrets.MINIO_KEY }} + secret_key: ${{ secrets.MINIO_SECRET }} + - name: Copy dir to MinIO + run: | + mc cp -r ${{ env.CPACK_DIR }}/ ${{ env.OSS_DIR }}/${{ steps.tag.outputs.tagnum }}/ + mc cp -r ${{ env.SYMS_DIR }}/ ${{ env.OSS_DIR }}/${{ steps.tag.outputs.tagnum }}/symbols/ + + docker_build: + needs: package + name: docker-build + runs-on: [self-hosted, nebula] + + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - id: tagname + run: | + echo tag="v${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + echo tagnum="${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + echo majorver="v$(echo ${{ github.event.inputs.version }} | cut -f1 -d'.')" >> $GITHUB_OUTPUT + - id: docker + run: | + graphdTag="" + storagedTag="" + metadTag="" + toolsTag="" + majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d".") + if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then + graphdTag="${{ secrets.HARBOR_REGISTRY }}/rc/nebula-graphd:latest" + storagedTag="${{ secrets.HARBOR_REGISTRY }}/rc/nebula-storaged:latest" + metadTag="${{ secrets.HARBOR_REGISTRY }}/rc/nebula-metad:latest" + toolsTag="${{ secrets.HARBOR_REGISTRY }}/rc/nebula-tools:latest" + fi + echo "graphdTag=$graphdTag" >> $GITHUB_OUTPUT + echo "storagedTag=$storagedTag" >> $GITHUB_OUTPUT + echo "metadTag=$metadTag" >> $GITHUB_OUTPUT + echo "toolsTag=$toolsTag" >> $GITHUB_OUTPUT + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ${{ secrets.HARBOR_REGISTRY }} + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} + - name: docker-graph + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-graphd:${{ steps.tagname.outputs.tag }} + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-graphd:${{ steps.tagname.outputs.majorver }} + ${{ steps.docker.outputs.graphdTag }} + target: graphd + cache-to: type=local,dest=/tmp/buildx-cache,mode=max + push: true + build-args: | + BRANCH=${{ steps.tagname.outputs.tag }} + VERSION=${{ steps.tagname.outputs.tagnum }} + - name: docker-storage + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-storaged:${{ steps.tagname.outputs.tag }} + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-storaged:${{ steps.tagname.outputs.majorver }} + ${{ steps.docker.outputs.storagedTag }} + target: storaged + push: true + cache-from: type=local,src=/tmp/buildx-cache + build-args: | + BRANCH=${{ steps.tagname.outputs.tag }} + VERSION=${{ steps.tagname.outputs.tagnum }} + - name: docker-meta + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-metad:${{ steps.tagname.outputs.tag }} + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-metad:${{ steps.tagname.outputs.majorver }} + ${{ steps.docker.outputs.metadTag }} + target: metad + push: true + cache-from: type=local,src=/tmp/buildx-cache + build-args: | + BRANCH=${{ steps.tagname.outputs.tag }} + VERSION=${{ steps.tagname.outputs.tagnum }} + - name: docker-tool + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-tools:${{ steps.tagname.outputs.tag }} + ${{ secrets.HARBOR_REGISTRY }}/rc/nebula-tools:${{ steps.tagname.outputs.majorver }} + ${{ steps.docker.outputs.toolsTag }} + target: tools + push: true + cache-from: type=local,src=/tmp/buildx-cache + build-args: | + BRANCH=${{ steps.tagname.outputs.tag }} + VERSION=${{ steps.tagname.outputs.tagnum }} + - name: delete the cache + run: | + rm -rf /tmp/buildx-cache + test: + name: test + needs: package + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + os: + - ubuntu1804 + - ubuntu2004 + - centos7 + - centos8 + extra_config: + - "ENABLE_SSL=true CA_SIGNED=true QUERY_CONCURRENTLY=false" + - "ENABLE_SSL=false CA_SIGNED=false QUERY_CONCURRENTLY=true" + container: + image: vesoft/nebula-dev:${{ matrix.os }} + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - id: tag + run: echo tagnum=${{ github.event.inputs.version }} >> $GITHUB_OUTPUT + + - id: oss_package + run: | + case ${{ matrix.os }} in + ubuntu1804) + echo "p=nebula-graph-${{ steps.tag.outputs.tagnum }}.ubuntu1804.amd64.tar.gz" >> $GITHUB_OUTPUT + ;; + ubuntu2004) + echo "p=nebula-graph-${{ steps.tag.outputs.tagnum }}.ubuntu2004.amd64.tar.gz" >> $GITHUB_OUTPUT + ;; + centos7) + echo "p=nebula-graph-${{ steps.tag.outputs.tagnum }}.el7.x86_64.tar.gz" >> $GITHUB_OUTPUT + ;; + centos8) + echo "p=nebula-graph-${{ steps.tag.outputs.tagnum }}.el8.x86_64.tar.gz" >> $GITHUB_OUTPUT + ;; + esac + - name: Prepare environment + id: prepare + run: | + [ -d build/ ] && rm -rf build/* || mkdir -p build + make init -C tests + - name: CMake + id: cmake + run: | + echo "j=8" >> $GITHUB_OUTPUT + - uses: vesoft-inc/.github/actions/setup-minio@master + with: + minio_url: ${{ secrets.MINIO_ENDPOINT }} + access_key: ${{ secrets.MINIO_KEY }} + secret_key: ${{ secrets.MINIO_SECRET }} + - name: Copy dir from MinIO + run: | + mc cp ${{ env.OSS_DIR }}/${{ steps.tag.outputs.tagnum }}/${{ steps.oss_package.outputs.p }} build/ + tar zxvf build/${{ steps.oss_package.outputs.p }} -C build + d=`echo ${{ steps.oss_package.outputs.p }} | sed 's/.tar.gz//'` + mv build/${d}/* build/. + - name: Setup cluster + run: | + make CONTAINERIZED=true ${{ matrix.extra_config }} up + working-directory: tests/ + timeout-minutes: 2 + - name: Pytest + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test + working-directory: tests/ + timeout-minutes: 15 + - name: TCK + run: | + make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck + working-directory: tests/ + timeout-minutes: 60 + - name: Down cluster + run: | + make RM_DIR=false down + working-directory: tests/ + timeout-minutes: 2 + - name: Upload logs + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs + path: ./build/server_*/logs*/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..74821b7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: release + +on: + release: + types: + - published + +concurrency: + group: release + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + package: + name: package + runs-on: [self-hosted, medium] + container: + image: vesoft/nebula-dev:centos7 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/tagname-action + id: tag + - uses: vesoft-inc/.github/actions/setup-minio@master + with: + minio_url: ${{ secrets.MINIO_ENDPOINT }} + access_key: ${{ secrets.MINIO_KEY }} + secret_key: ${{ secrets.MINIO_SECRET }} + - name: Copy rc to release on MinIO + run: | + files=$(mc ls minio/rc-build/nebula-graph/${{ steps.tag.outputs.tagnum }}/ | awk '{print $6}' | grep -v '/$') + for file in $files; do + mc cp minio/rc-build/nebula-graph/${{ steps.tag.outputs.tagnum }}/$file \ + minio/release-build/nebula-graph/${{ steps.tag.outputs.tagnum }}/ + done + mc cp -r minio/rc-build/nebula-graph/${{ steps.tag.outputs.tagnum }}/symbols/ \ + minio/release-build/nebula-graph/${{ steps.tag.outputs.tagnum }}/symbols/ + + docker_build: + name: docker-build + runs-on: [self-hosted, nebula] + strategy: + fail-fast: false + matrix: + service: + - graphd + - metad + - storaged + - tools + steps: + - uses: webiny/action-post-run@3.0.0 + with: + run: sh -c "find . -mindepth 1 -delete" + - uses: actions/checkout@v3 + - uses: ./.github/actions/tagname-action + id: tagname + - id: docker + run: | + majorver=$(git tag -l --sort=v:refname | tail -n1 | cut -f1 -d".") + tag="" + if [[ $majorver == ${{ steps.tagname.outputs.majorver }} ]]; then + tag="latest" + fi + echo "tag=$tag" >> $GITHUB_OUTPUT + - name: Sync docker images + env: + FROM_IMAGE: docker://${{ secrets.HARBOR_REGISTRY }}/rc/nebula-${{ matrix.service }} + TO_IMAGE: docker://docker.io/vesoft/nebula-${{ matrix.service }} + CMD: docker run --rm -i quay.io/containers/skopeo:latest copy -a --src-creds ${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }} --dest-creds ${{ secrets.DOCKER_USERNAME }}:${{ secrets.DOCKER_PASSWORD }} + run: | + ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.tag }} + ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.majorver }} + if [[ ! -z "${{ steps.docker.outputs.tag }}" ]]; then + ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.docker.outputs.tag }} + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d9538a --- /dev/null +++ b/.gitignore @@ -0,0 +1,71 @@ +*.swp +*.so +*.DS_Store + +# build +pkg-build +build +_build +_build.log +_install +install +bin/ +install_manifest.txt +src/version/Version.cpp + +# ccls +.ccls +.ccls-cache +compile_commands.json +# clangd cache +.clangd/ + +gen-* + +# cmake +CPackConfig.cmake +CPackSourceConfig.cmake +CMakeCache.txt +Makefile +cmake_install.cmake +CTestTestfile.cmake +CMakeFiles/ +Testing/ +target/ +cluster.id +pids/ +modules/ + +# tests +!tests/Makefile +tests/secrets +reformat-gherkin +nebula-python + +# IDE +.idea/ +.project +.settings/ +.classpath +cmake-build*/ +.vscode/ +.cache/ +core.* +workspace.* +.metals/ +.cproject +.ycm_extra_conf.py + +#py +*.egg-info +*.pot +*.py[co] +__pycache__ +venv/ + +#lock +*.lock + +#ctags +.tags +/.vs diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 0000000..d155626 --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,46 @@ +# +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +header: + license: + spdx-id: Apache-2.0 + copyright-owner: vesoft inc + content: | + + Copyright (c) 2020 vesoft inc. All rights reserved. + + This source code is licensed under Apache 2.0 License. + + pattern: | + + Copyright (c) \d{4} vesoft inc. All rights reserved. + + This source code is licensed under Apache 2.0 License. + + paths-ignore: + - '.licenserc.yaml' + - '.clang-tidy' + - '.dockerignore' + - '**/.gitignore' + - 'cmake' + - 'conf' + - 'docker' + - 'package' + - 'resources/*.csv' + - 'resources/*.json' + - 'scripts/*.service' + - 'src/**/*.thrift' + - 'src/**/*.lex' + - 'src/**/*.yy' + - 'src/**/*.dict' + - 'src/**/*.in' + - 'tests' + - '.github' + - '.linters' + - '**/*.md' + - 'third-party' + - 'LICENSE' + - 'NOTICE' + + comment: on-failure diff --git a/.linters/cpp/checkKeyword.py b/.linters/cpp/checkKeyword.py new file mode 100644 index 0000000..34f9cdc --- /dev/null +++ b/.linters/cpp/checkKeyword.py @@ -0,0 +1,172 @@ +# signout--coding:utf-8-- + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +import os +import re +import sys + +PARSER_FILE_PATH = 'src/parser/parser.yy' +SCANNER_FILE_PATH = 'src/parser/scanner.lex' + +reserved_key_words = [ + 'KW_GO', + 'KW_AS', + 'KW_TO', + 'KW_OR', + 'KW_AND', + 'KW_XOR', + 'KW_USE', + 'KW_SET', + 'KW_LIST', + 'KW_MAP', + 'KW_FROM', + 'KW_WHERE', + 'KW_MATCH', + 'KW_INSERT', + 'KW_YIELD', + 'KW_RETURN', + 'KW_DESCRIBE', + 'KW_DESC', + 'KW_VERTEX', + 'KW_VERTICES', + 'KW_EDGE', + 'KW_EDGES', + 'KW_UPDATE', + 'KW_UPSERT', + 'KW_WHEN', + 'KW_DELETE', + 'KW_FIND', + 'KW_PATH', + 'KW_LOOKUP', + 'KW_ALTER', + 'KW_STEPS', + 'KW_OVER', + 'KW_UPTO', + 'KW_REVERSELY', + 'KW_INDEX', + 'KW_INDEXES', + 'KW_REBUILD', + 'KW_BOOL', + 'KW_INT8', + 'KW_INT16', + 'KW_INT32', + 'KW_INT64', + 'KW_INT', + 'KW_FLOAT', + 'KW_DOUBLE', + 'KW_STRING', + 'KW_FIXED_STRING', + 'KW_TIMESTAMP', + 'KW_DATE', + 'KW_TIME', + 'KW_DATETIME', + 'KW_VID_SIZE', + 'KW_TAG', + 'KW_TAGS', + 'KW_UNION', + 'KW_INTERSECT', + 'KW_MINUS', + 'KW_NO', + 'KW_OVERWRITE', + 'KW_SHOW', + 'KW_ADD', + 'KW_CREATE', + 'KW_DROP', + 'KW_REMOVE', + 'KW_IF', + 'KW_NOT', + 'KW_EXISTS', + 'KW_WITH', + 'KW_CHANGE', + 'KW_GRANT', + 'KW_REVOKE', + 'KW_ON', + 'KW_BY', + 'KW_IN', + 'KW_NOT_IN', + 'KW_DOWNLOAD', + 'KW_GET', + 'KW_OF', + 'KW_ORDER', + 'KW_INGEST', + 'KW_COMPACT', + 'KW_FLUSH', + 'KW_SUBMIT', + 'KW_ASC', + 'KW_ASCENDING', + 'KW_DESCENDING', + 'KW_DISTINCT', + 'KW_FETCH', + 'KW_PROP', + 'KW_BALANCE', + 'KW_STOP', + 'KW_GROUP', + 'KW_IS', + 'KW_NULL', + 'KW_FORCE', + 'KW_RECOVER', + 'KW_EXPLAIN', + 'KW_UNWIND', + 'KW_CASE', + 'KW_HOSTS', + 'KW_ZONE', + 'KW_ZONES', + 'KW_RENAME', + 'KW_IGNORE_EXISTED_INDEX', + 'KW_GEOGRAPHY', + 'KW_DURATION', + 'KW_ACROSS', + 'KW_JOIN', + 'KW_LEFT', + 'KW_RIGHT', + 'KW_INNER', + 'KW_OUTER', + 'KW_SEMI', + 'KW_ANTI', +] + + +def get_unreserved_keyword(file_path): + parser_file = open(file_path) + flag = 0 + unreserved_key_words = [] + for line in parser_file.readlines(): + if line.strip() == 'unreserved_keyword': + flag = 1 + continue + if flag == 1: + if line.strip() == ';': + break + unreserved_key_words.append( + re.sub('\\s+[:|]\\s+(\\w+)\\s+.*', '\\1', line).strip()) + continue + + parser_file.close() + return unreserved_key_words + + +if __name__ == '__main__': + cmd = 'git diff --diff-filter=ACMRTUXB HEAD -p ' + \ + SCANNER_FILE_PATH + '|grep "^+"|grep -v "^+++"|grep "KW_"' + content = os.popen(cmd) + keywords = [] + for line in content.readlines(): + keyword = re.sub('.*(KW_\\w+)\s*;.*', '\\1', line.strip()) + keywords.append(keyword) + + if len(keywords) == 0: + exit(0) + unreserved_key_words = get_unreserved_keyword(PARSER_FILE_PATH) + new_key_words = [ + word for word in keywords if word not in reserved_key_words] + if len(new_key_words) == 0: + exit(0) + result = [word for word in new_key_words if word not in unreserved_key_words] + if len(result) == 0: + exit(0) + print('Keywords \"{}\" in src/parser/scanner.lex are not in the unreserved keyword list.'.format(result)) + print('Please add them to the unreserved keyword in the src/parser/parser.yy.') + exit(1) diff --git a/.linters/cpp/cpplint.py b/.linters/cpp/cpplint.py new file mode 100644 index 0000000..5cc9b34 --- /dev/null +++ b/.linters/cpp/cpplint.py @@ -0,0 +1,6605 @@ +#!/usr/bin/env python +# +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does google-lint on c++ files. + +The goal of this script is to identify places in the code that *may* +be in non-compliance with google style. It does not attempt to fix +up these problems -- the point is to educate. It does also not +attempt to find all problems, or to ensure that everything it does +find is legitimately a problem. + +In particular, we can get very confused by /* and // inside strings! +We do a small hack, which is to ignore //'s with "'s after them on the +same line, but it is far from perfect (in either direction). +""" + +import codecs +import copy +import getopt +import glob +import itertools +import math # for log +import os +import re +import sre_compile +import string +import sys +import sysconfig +import unicodedata +import xml.etree.ElementTree + +# if empty, use defaults +_valid_extensions = set([]) + +__VERSION__ = '1.4.4' + +try: + xrange # Python 2 +except NameError: + # -- pylint: disable=redefined-builtin + xrange = range # Python 3 + + +_USAGE = """ +Syntax: cpplint.py [--verbose=#] [--output=emacs|eclipse|vs7|junit] + [--filter=-x,+y,...] + [--counting=total|toplevel|detailed] [--root=subdir] + [--repository=path] + [--linelength=digits] [--headers=x,y,...] + [--recursive] + [--exclude=path] + [--extensions=hpp,cpp,...] + [--quiet] + [--version] + [file] ... + + Style checker for C/C++ source files. + This is a fork of the Google style checker with minor extensions. + + The style guidelines this tries to follow are those in + https://google.github.io/styleguide/cppguide.html + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To suppress false-positive errors of a certain category, add a + 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) + suppresses errors of all categories on that line. + + The files passed in will be linted; at least one file must be provided. + Default linted extensions are %s. + Other file types will be ignored. + Change the extensions with the --extensions flag. + + Flags: + + output=emacs|eclipse|vs7|junit + By default, the output is formatted to ease emacs parsing. Visual Studio + compatible output (vs7) may also be used. Further support exists for + eclipse (eclipse), and JUnit (junit). XML parsers such as those used + in Jenkins and Bamboo may also be used. Other formats are unsupported. + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + Errors with lower verbosity levels have lower confidence and are more + likely to be false positives. + + quiet + Don't print anything if no errors are found. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in cpplint, pass no arg: + --filter= + + counting=total|toplevel|detailed + The total number of errors found is always printed. If + 'toplevel' is provided, then the count of errors in each of + the top-level categories like 'build' and 'whitespace' will + also be printed. If 'detailed' is provided, then a count + is provided for each category like 'build/class'. + + repository=path + The top level directory of the repository, used to derive the header + guard CPP variable. By default, this is determined by searching for a + path that contains .git, .hg, or .svn. When this flag is specified, the + given path is used instead. This option allows the header guard CPP + variable to remain consistent even if members of a team have different + repository root directories (such as when checking out a subdirectory + with SVN). In addition, users of non-mainstream version control systems + can use this flag to ensure readable header guard CPP variables. + + Examples: + Assuming that Alice checks out ProjectName and Bob checks out + ProjectName/trunk and trunk contains src/chrome/ui/browser.h, then + with no --repository flag, the header guard CPP variable will be: + + Alice => TRUNK_SRC_CHROME_BROWSER_UI_BROWSER_H_ + Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ + + If Alice uses the --repository=trunk flag and Bob omits the flag or + uses --repository=. then the header guard CPP variable will be: + + Alice => SRC_CHROME_BROWSER_UI_BROWSER_H_ + Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ + + root=subdir + The root directory used for deriving header guard CPP variable. + This directory is relative to the top level directory of the repository + which by default is determined by searching for a directory that contains + .git, .hg, or .svn but can also be controlled with the --repository flag. + If the specified directory does not exist, this flag is ignored. + + Examples: + Assuming that src is the top level directory of the repository (and + cwd=top/src), the header guard CPP variables for + src/chrome/browser/ui/browser.h are: + + No flag => CHROME_BROWSER_UI_BROWSER_H_ + --root=chrome => BROWSER_UI_BROWSER_H_ + --root=chrome/browser => UI_BROWSER_H_ + --root=.. => SRC_CHROME_BROWSER_UI_BROWSER_H_ + + linelength=digits + This is the allowed line length for the project. The default value is + 80 characters. + + Examples: + --linelength=120 + + recursive + Search for files to lint recursively. Each directory given in the list + of files to be linted is replaced by all files that descend from that + directory. Files with extensions not in the valid extensions list are + excluded. + + exclude=path + Exclude the given path from the list of files to be linted. Relative + paths are evaluated relative to the current directory and shell globbing + is performed. This flag can be provided multiple times to exclude + multiple files. + + Examples: + --exclude=one.cc + --exclude=src/*.cc + --exclude=src/*.cc --exclude=test/*.cc + + extensions=extension,extension,... + The allowed file extensions that cpplint will check + + Examples: + --extensions=%s + + headers=x,y,... + The header extensions that cpplint will treat as .h in checks. Values are + automatically added to --extensions list. + (by default, only files with extensions %s will be assumed to be headers) + + Examples: + --headers=%s + --headers=hpp,hxx + --headers=hpp + + cpplint.py supports per-directory configurations specified in CPPLINT.cfg + files. CPPLINT.cfg file can contain a number of key=value pairs. + Currently the following options are supported: + + set noparent + filter=+filter1,-filter2,... + exclude_files=regex + linelength=80 + root=subdir + headers=x,y,... + + "set noparent" option prevents cpplint from traversing directory tree + upwards looking for more .cfg files in parent directories. This option + is usually placed in the top-level project directory. + + The "filter" option is similar in function to --filter flag. It specifies + message filters in addition to the |_DEFAULT_FILTERS| and those specified + through --filter command-line flag. + + "exclude_files" allows to specify a regular expression to be matched against + a file name. If the expression matches, the file is skipped and not run + through the linter. + + "linelength" allows to specify the allowed line length for the project. + + The "root" option is similar in function to the --root flag (see example + above). Paths are relative to the directory of the CPPLINT.cfg. + + The "headers" option is similar in function to the --headers flag + (see example above). + + CPPLINT.cfg has an effect on files in the same directory and all + sub-directories, unless overridden by a nested configuration file. + + Example file: + filter=-build/include_order,+build/include_alpha + exclude_files=.*\\.cc + + The above example disables build/include_order warning and enables + build/include_alpha as well as excludes all .cc from being + processed by linter, in the current directory (where the .cfg + file is located) and all sub-directories. +""" + +# We categorize each error message we print. Here are the categories. +# We want an explicit list so we can list them all in cpplint --filter=. +# If you add a new error message with a new category, add it to the list +# here! cpplint_unittest.py should tell you if you forget to do this. +_ERROR_CATEGORIES = [ + 'build/class', + 'build/c++11', + 'build/c++14', + 'build/c++tr1', + 'build/deprecated', + 'build/endif_comment', + 'build/explicit_make_pair', + 'build/forward_decl', + 'build/header_guard', + 'build/include', + 'build/include_subdir', + 'build/include_alpha', + 'build/include_order', + 'build/include_what_you_use', + 'build/namespaces_literals', + 'build/namespaces', + 'build/printf_format', + 'build/storage_class', + 'legal/copyright', + 'readability/alt_tokens', + 'readability/braces', + 'readability/casting', + 'readability/check', + 'readability/constructors', + 'readability/fn_size', + 'readability/inheritance', + 'readability/multiline_comment', + 'readability/multiline_string', + 'readability/namespace', + 'readability/nolint', + 'readability/nul', + 'readability/strings', + 'readability/todo', + 'readability/utf8', + 'runtime/arrays', + 'runtime/casting', + 'runtime/explicit', + 'runtime/int', + 'runtime/init', + 'runtime/invalid_increment', + 'runtime/member_string_references', + 'runtime/memset', + 'runtime/indentation_namespace', + 'runtime/operator', + 'runtime/printf', + 'runtime/printf_format', + 'runtime/references', + 'runtime/string', + 'runtime/threadsafe_fn', + 'runtime/vlog', + 'whitespace/blank_line', + 'whitespace/braces', + 'whitespace/comma', + 'whitespace/comments', + 'whitespace/empty_conditional_body', + 'whitespace/empty_if_body', + 'whitespace/empty_loop_body', + 'whitespace/end_of_line', + 'whitespace/ending_newline', + 'whitespace/forcolon', + 'whitespace/indent', + 'whitespace/line_length', + 'whitespace/newline', + 'whitespace/operators', + 'whitespace/parens', + 'whitespace/semicolon', + 'whitespace/tab', + 'whitespace/todo', + ] + +# These error categories are no longer enforced by cpplint, but for backwards- +# compatibility they may still appear in NOLINT comments. +_LEGACY_ERROR_CATEGORIES = [ + 'readability/streams', + 'readability/function', + ] + +# The default state of the category filter. This is overridden by the --filter= +# flag. By default all errors are on, so only add here categories that should be +# off by default (i.e., categories that must be enabled by the --filter= flags). +# All entries here should start with a '-' or '+', as in the --filter= flag. +_DEFAULT_FILTERS = ['-build/include_alpha'] + +# The default list of categories suppressed for C (not C++) files. +_DEFAULT_C_SUPPRESSED_CATEGORIES = [ + 'readability/casting', + ] + +# The default list of categories suppressed for Linux Kernel files. +_DEFAULT_KERNEL_SUPPRESSED_CATEGORIES = [ + 'whitespace/tab', + ] + +# We used to check for high-bit characters, but after much discussion we +# decided those were OK, as long as they were in UTF-8 and didn't represent +# hard-coded international strings, which belong in a separate i18n file. + +# C++ headers +_CPP_HEADERS = frozenset([ + # Legacy + 'algobase.h', + 'algo.h', + 'alloc.h', + 'builtinbuf.h', + 'bvector.h', + 'complex.h', + 'defalloc.h', + 'deque.h', + 'editbuf.h', + 'fstream.h', + 'function.h', + 'hash_map', + 'hash_map.h', + 'hash_set', + 'hash_set.h', + 'hashtable.h', + 'heap.h', + 'indstream.h', + 'iomanip.h', + 'iostream.h', + 'istream.h', + 'iterator.h', + 'list.h', + 'map.h', + 'multimap.h', + 'multiset.h', + 'ostream.h', + 'pair.h', + 'parsestream.h', + 'pfstream.h', + 'procbuf.h', + 'pthread_alloc', + 'pthread_alloc.h', + 'rope', + 'rope.h', + 'ropeimpl.h', + 'set.h', + 'slist', + 'slist.h', + 'stack.h', + 'stdiostream.h', + 'stl_alloc.h', + 'stl_relops.h', + 'streambuf.h', + 'stream.h', + 'strfile.h', + 'strstream.h', + 'tempbuf.h', + 'tree.h', + 'type_traits.h', + 'vector.h', + # 17.6.1.2 C++ library headers + 'algorithm', + 'array', + 'atomic', + 'bitset', + 'chrono', + 'codecvt', + 'complex', + 'condition_variable', + 'deque', + 'exception', + 'forward_list', + 'fstream', + 'functional', + 'future', + 'initializer_list', + 'iomanip', + 'ios', + 'iosfwd', + 'iostream', + 'istream', + 'iterator', + 'limits', + 'list', + 'locale', + 'map', + 'memory', + 'mutex', + 'new', + 'numeric', + 'ostream', + 'queue', + 'random', + 'ratio', + 'regex', + 'scoped_allocator', + 'set', + 'sstream', + 'stack', + 'stdexcept', + 'streambuf', + 'string', + 'strstream', + 'system_error', + 'thread', + 'tuple', + 'typeindex', + 'typeinfo', + 'type_traits', + 'unordered_map', + 'unordered_set', + 'utility', + 'valarray', + 'vector', + # 17.6.1.2 C++14 headers + 'shared_mutex', + # 17.6.1.2 C++17 headers + 'any', + 'charconv', + 'codecvt', + 'execution', + 'filesystem', + 'memory_resource', + 'optional', + 'string_view', + 'variant', + # 17.6.1.2 C++ headers for C library facilities + 'cassert', + 'ccomplex', + 'cctype', + 'cerrno', + 'cfenv', + 'cfloat', + 'cinttypes', + 'ciso646', + 'climits', + 'clocale', + 'cmath', + 'csetjmp', + 'csignal', + 'cstdalign', + 'cstdarg', + 'cstdbool', + 'cstddef', + 'cstdint', + 'cstdio', + 'cstdlib', + 'cstring', + 'ctgmath', + 'ctime', + 'cuchar', + 'cwchar', + 'cwctype', + ]) + +# Type names +_TYPES = re.compile( + r'^(?:' + # [dcl.type.simple] + r'(char(16_t|32_t)?)|wchar_t|' + r'bool|short|int|long|signed|unsigned|float|double|' + # [support.types] + r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|' + # [cstdint.syn] + r'(u?int(_fast|_least)?(8|16|32|64)_t)|' + r'(u?int(max|ptr)_t)|' + r')$') + + +# These headers are excluded from [build/include] and [build/include_order] +# checks: +# - Anything not following google file name conventions (containing an +# uppercase character, such as Python.h or nsStringAPI.h, for example). +# - Lua headers. +_THIRD_PARTY_HEADERS_PATTERN = re.compile( + r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') + +# Pattern for matching FileInfo.BaseName() against test file name +_test_suffixes = ['_test', '_regtest', '_unittest'] +_TEST_FILE_SUFFIX = '(' + '|'.join(_test_suffixes) + r')$' + +# Pattern that matches only complete whitespace, possibly across multiple lines. +_EMPTY_CONDITIONAL_BODY_PATTERN = re.compile(r'^\s*$', re.DOTALL) + +# Assertion macros. These are defined in base/logging.h and +# testing/base/public/gunit.h. +_CHECK_MACROS = [ + 'DCHECK', 'CHECK', + 'EXPECT_TRUE', 'ASSERT_TRUE', + 'EXPECT_FALSE', 'ASSERT_FALSE', + ] + +# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE +_CHECK_REPLACEMENT = dict([(macro_var, {}) for macro_var in _CHECK_MACROS]) + +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), + ('>=', 'GE'), ('>', 'GT'), + ('<=', 'LE'), ('<', 'LT')]: + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), + ('>=', 'LT'), ('>', 'LE'), + ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + +# Alternative tokens and their replacements. For full list, see section 2.5 +# Alternative tokens [lex.digraph] in the C++ standard. +# +# Digraphs (such as '%:') are not included here since it's a mess to +# match those on a word boundary. +_ALT_TOKEN_REPLACEMENT = { + 'and': '&&', + 'bitor': '|', + 'or': '||', + 'xor': '^', + 'compl': '~', + 'bitand': '&', + 'and_eq': '&=', + 'or_eq': '|=', + 'xor_eq': '^=', + 'not': '!', + 'not_eq': '!=' + } + +# Compile regular expression that matches all the above keywords. The "[ =()]" +# bit is meant to avoid matching these keywords outside of boolean expressions. +# +# False positives include C-style multi-line comments and multi-line strings +# but those have always been troublesome for cpplint. +_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( + r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') + + +# These constants define types of headers for use with +# _IncludeState.CheckNextIncludeOrder(). +_C_SYS_HEADER = 1 +_CPP_SYS_HEADER = 2 +_LIKELY_MY_HEADER = 3 +_POSSIBLE_MY_HEADER = 4 +_OTHER_HEADER = 5 + +# These constants define the current inline assembly state +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block + +# Match start of assembly blocks +_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' + r'(?:\s+(volatile|__volatile__))?' + r'\s*[{(]') + +# Match strings that indicate we're working on a C (not C++) file. +_SEARCH_C_FILE = re.compile(r'\b(?:LINT_C_FILE|' + r'vim?:\s*.*(\s*|:)filetype=c(\s*|:|$))') + +# Match string that indicates we're working on a Linux Kernel file. +_SEARCH_KERNEL_FILE = re.compile(r'\b(?:LINT_KERNEL_FILE)') + +_regexp_compile_cache = {} + +# {str, set(int)}: a map from error categories to sets of linenumbers +# on which those errors are expected and should be suppressed. +_error_suppressions = {} + +# The root directory used for deriving header guard CPP variable. +# This is set by --root flag. +_root = None +_root_debug = False + +# The top level repository directory. If set, _root is calculated relative to +# this directory instead of the directory containing version control artifacts. +# This is set by the --repository flag. +_repository = None + +# Files to exclude from linting. This is set by the --exclude flag. +_excludes = None + +# Whether to suppress PrintInfo messages +_quiet = False + +# The allowed line length of files. +# This is set by --linelength flag. +_line_length = 80 + +try: + unicode +except NameError: + # -- pylint: disable=redefined-builtin + basestring = unicode = str + +try: + long +except NameError: + # -- pylint: disable=redefined-builtin + long = int + +if sys.version_info < (3,): + # -- pylint: disable=no-member + # BINARY_TYPE = str + itervalues = dict.itervalues + iteritems = dict.iteritems +else: + # BINARY_TYPE = bytes + itervalues = dict.values + iteritems = dict.items + +def unicode_escape_decode(x): + if sys.version_info < (3,): + return codecs.unicode_escape_decode(x)[0] + else: + return x + +# Treat all headers starting with 'h' equally: .h, .hpp, .hxx etc. +# This is set by --headers flag. +_hpp_headers = set([]) + +# {str, bool}: a map from error categories to booleans which indicate if the +# category should be suppressed for every line. +_global_error_suppressions = {} + +def ProcessHppHeadersOption(val): + global _hpp_headers + try: + _hpp_headers = {ext.strip() for ext in val.split(',')} + except ValueError: + PrintUsage('Header extensions must be comma separated list.') + +def IsHeaderExtension(file_extension): + return file_extension in GetHeaderExtensions() + +def GetHeaderExtensions(): + if _hpp_headers: + return _hpp_headers + if _valid_extensions: + return {h for h in _valid_extensions if 'h' in h} + return set(['h', 'hh', 'hpp', 'hxx', 'h++', 'cuh']) + +# The allowed extensions for file names +# This is set by --extensions flag +def GetAllExtensions(): + return GetHeaderExtensions().union(_valid_extensions or set( + ['c', 'cc', 'cpp', 'cxx', 'c++', 'cu'])) + +def ProcessExtensionsOption(val): + global _valid_extensions + try: + extensions = [ext.strip() for ext in val.split(',')] + _valid_extensions = set(extensions) + except ValueError: + PrintUsage('Extensions should be a comma-separated list of values;' + 'for example: extensions=hpp,cpp\n' + 'This could not be parsed: "%s"' % (val,)) + +def GetNonHeaderExtensions(): + return GetAllExtensions().difference(GetHeaderExtensions()) + +def ParseNolintSuppressions(filename, raw_line, linenum, error): + """Updates the global list of line error-suppressions. + + Parses any NOLINT comments on the current line, updating the global + error_suppressions store. Reports an error if the NOLINT comment + was malformed. + + Args: + filename: str, the name of the input file. + raw_line: str, the line of input text, with comments. + linenum: int, the number of the current line. + error: function, an error handler. + """ + matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) + if matched: + if matched.group(1): + suppressed_line = linenum + 1 + else: + suppressed_line = linenum + category = matched.group(2) + if category in (None, '(*)'): # => "suppress all" + _error_suppressions.setdefault(None, set()).add(suppressed_line) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + _error_suppressions.setdefault(category, set()).add(suppressed_line) + elif category not in _LEGACY_ERROR_CATEGORIES: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) + + +def ProcessGlobalSuppressions(lines): + """Updates the list of global error suppressions. + + Parses any lint directives in the file that have global effect. + + Args: + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is terminated with a newline. + """ + for line in lines: + if _SEARCH_C_FILE.search(line): + for category in _DEFAULT_C_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True + if _SEARCH_KERNEL_FILE.search(line): + for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True + + +def ResetNolintSuppressions(): + """Resets the set of NOLINT suppressions to empty.""" + _error_suppressions.clear() + _global_error_suppressions.clear() + + +def IsErrorSuppressedByNolint(category, linenum): + """Returns true if the specified error category is suppressed on this line. + + Consults the global error_suppressions map populated by + ParseNolintSuppressions/ProcessGlobalSuppressions/ResetNolintSuppressions. + + Args: + category: str, the category of the error. + linenum: int, the current line number. + Returns: + bool, True iff the error should be suppressed due to a NOLINT comment or + global suppression. + """ + return (_global_error_suppressions.get(category, False) or + linenum in _error_suppressions.get(category, set()) or + linenum in _error_suppressions.get(None, set())) + + +def Match(pattern, s): + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) + + +def ReplaceAll(pattern, rep, s): + """Replaces instances of pattern in a string with a replacement. + + The compiled regex is kept in a cache shared by Match and Search. + + Args: + pattern: regex pattern + rep: replacement text + s: search string + + Returns: + string with replacements made (or original string if no replacements) + """ + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) + + +def Search(pattern, s): + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) + + +def _IsSourceExtension(s): + """File extension (excluding dot) matches a source file extension.""" + return s in GetNonHeaderExtensions() + + +class _IncludeState(object): + """Tracks line numbers for includes, and the order in which includes appear. + + include_list contains list of lists of (header, line number) pairs. + It's a lists of lists rather than just one flat list to make it + easier to update across preprocessor boundaries. + + Call CheckNextIncludeOrder() once for each header in the file, passing + in the type constants defined above. Calls in an illegal order will + raise an _IncludeError with an appropriate error message. + + """ + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + self.include_list = [[]] + self._section = None + self._last_header = None + self.ResetSection('') + + def FindHeader(self, header): + """Check if a header has already been included. + + Args: + header: header to check. + Returns: + Line number of previous occurrence, or -1 if the header has not + been seen before. + """ + for section_list in self.include_list: + for f in section_list: + if f[0] == header: + return f[1] + return -1 + + def ResetSection(self, directive): + """Reset section checking for preprocessor directive. + + Args: + directive: preprocessor directive (e.g. "if", "else"). + """ + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' + + # Update list of includes. Note that we never pop from the + # include list. + if directive in ('if', 'ifdef', 'ifndef'): + self.include_list.append([]) + elif directive in ('else', 'elif'): + self.include_list[-1] = [] + + def SetLastHeader(self, header_path): + self._last_header = header_path + + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. + + - replaces "-" with "_" so they both cmp the same. + - removes '-inl' since we don't require them to be after the main header. + - lowercase everything, just in case. + + Args: + header_path: Path to be canonicalized. + + Returns: + Canonicalized path. + """ + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + header_path: Canonicalized header to be checked. + + Returns: + Returns true if the header is in alphabetical order. + """ + # If previous section is different from current section, _last_header will + # be reset to empty string, so it's always less than current header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and + Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): + return False + return True + + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. + + This function also updates the internal state to be ready to check + the next include. + + Args: + header_type: One of the _XXX_HEADER constants defined above. + + Returns: + The empty string if the header is in the right order, or an + error message describing what's wrong. + + """ + error_message = ('Found %s after %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION + + if last_section != self._section: + self._last_header = '' + + return '' + + +class _CppLintState(object): + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + # backup of filter list. Used to restore the state after each file. + self._filters_backup = self.filters[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + self.quiet = False # Suppress non-error messages? + + # output format: + # "emacs" - format that emacs can parse (default) + # "eclipse" - format that eclipse can parse + # "vs7" - format that Microsoft Visual Studio 7 can parse + # "junit" - format that Jenkins, Bamboo, etc can parse + self.output_format = 'emacs' + + # For JUnit output, save errors and failures until the end so that they + # can be written into the XML + self._junit_errors = [] + self._junit_failures = [] + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetQuiet(self, quiet): + """Sets the module's quiet settings, and returns the previous setting.""" + last_quiet = self.quiet + self.quiet = quiet + return last_quiet + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "+whitespace/indent"). + Each filter should start with + or -; else we die. + + Raises: + ValueError: The comma-separated filters did not all start with '+' or '-'. + E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" + """ + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + self.AddFilters(filters) + + def AddFilters(self, filters): + """ Adds more filters to the existing list of error-message filters. """ + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError('Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def BackupFilters(self): + """ Saves the current filter list to backup storage.""" + self._filters_backup = self.filters[:] + + def RestoreFilters(self): + """ Restores filters previously backed up.""" + self.filters = self._filters_backup[:] + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in sorted(iteritems(self.errors_by_category)): + self.PrintInfo('Category \'%s\' errors found: %d\n' % + (category, count)) + if self.error_count > 0: + self.PrintInfo('Total errors found: %d\n' % self.error_count) + + def PrintInfo(self, message): + if not _quiet and self.output_format != 'junit': + sys.stdout.write(message) + + def PrintError(self, message): + if self.output_format == 'junit': + self._junit_errors.append(message) + else: + sys.stderr.write(message) + + def AddJUnitFailure(self, filename, linenum, message, category, confidence): + self._junit_failures.append((filename, linenum, message, category, + confidence)) + + def FormatJUnitXML(self): + num_errors = len(self._junit_errors) + num_failures = len(self._junit_failures) + + testsuite = xml.etree.ElementTree.Element('testsuite') + testsuite.attrib['name'] = 'cpplint' + testsuite.attrib['errors'] = str(num_errors) + testsuite.attrib['failures'] = str(num_failures) + + if num_errors == 0 and num_failures == 0: + testsuite.attrib['tests'] = str(1) + xml.etree.ElementTree.SubElement(testsuite, 'testcase', name='passed') + + else: + testsuite.attrib['tests'] = str(num_errors + num_failures) + if num_errors > 0: + testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') + testcase.attrib['name'] = 'errors' + error = xml.etree.ElementTree.SubElement(testcase, 'error') + error.text = '\n'.join(self._junit_errors) + if num_failures > 0: + # Group failures by file + failed_file_order = [] + failures_by_file = {} + for failure in self._junit_failures: + failed_file = failure[0] + if failed_file not in failed_file_order: + failed_file_order.append(failed_file) + failures_by_file[failed_file] = [] + failures_by_file[failed_file].append(failure) + # Create a testcase for each file + for failed_file in failed_file_order: + failures = failures_by_file[failed_file] + testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') + testcase.attrib['name'] = failed_file + failure = xml.etree.ElementTree.SubElement(testcase, 'failure') + template = '{0}: {1} [{2}] [{3}]' + texts = [template.format(f[1], f[2], f[3], f[4]) for f in failures] + failure.text = '\n'.join(texts) + + xml_decl = '\n' + return xml_decl + xml.etree.ElementTree.tostring(testsuite, 'utf-8').decode('utf-8') + + +_cpplint_state = _CppLintState() + + +def _OutputFormat(): + """Gets the module's output format.""" + return _cpplint_state.output_format + + +def _SetOutputFormat(output_format): + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) + +def _Quiet(): + """Return's the module's quiet setting.""" + return _cpplint_state.quiet + +def _SetQuiet(quiet): + """Set the module's quiet status, and return previous setting.""" + return _cpplint_state.SetQuiet(quiet) + + +def _VerboseLevel(): + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level + + +def _SetVerboseLevel(level): + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) + + +def _SetCountingStyle(level): + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) + + +def _Filters(): + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters + + +def _SetFilters(filters): + """Sets the module's error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.SetFilters(filters) + +def _AddFilters(filters): + """Adds more filter overrides. + + Unlike _SetFilters, this function does not reset the current list of filters + available. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.AddFilters(filters) + +def _BackupFilters(): + """ Saves the current filter list to backup storage.""" + _cpplint_state.BackupFilters() + +def _RestoreFilters(): + """ Restores filters previously backed up.""" + _cpplint_state.RestoreFilters() + +class _FunctionState(object): + """Tracks current function name and the number of lines in its body.""" + + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' + + def Begin(self, function_name): + """Start analyzing function body. + + Args: + function_name: The name of the function being tracked. + """ + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name + + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 + + def Check(self, error, filename, linenum): + """Report if too many lines in function body. + + Args: + error: The function to call with any errors found. + filename: The name of the current file. + linenum: The number of the line to check. + """ + if not self.in_a_function: + return + + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False + + +class _IncludeError(Exception): + """Indicates a problem with the include order in a file.""" + pass + + +class FileInfo(object): + """Provides utility functions for filenames. + + FileInfo provides easy access to the components of a file's path + relative to the project root. + """ + + def __init__(self, filename): + self._filename = filename + + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') + + def RepositoryName(self): + r"""FullName after removing the local path to the repository. + + If we have a real absolute path name here we can try to do something smart: + detecting the root of the checkout and truncating /path/to/checkout from + the name so that we get header guards that don't include things like + "C:\\Documents and Settings\\..." or "/home/username/..." in them and thus + people on different computers who have checked the source out to different + locations won't see bogus errors. + """ + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + # If the user specified a repository path, it exists, and the file is + # contained in it, use the specified repository path + if _repository: + repo = FileInfo(_repository).FullName() + root_dir = project_dir + while os.path.exists(root_dir): + # allow case insensitive compare on Windows + if os.path.normcase(root_dir) == os.path.normcase(repo): + return os.path.relpath(fullname, root_dir).replace('\\', '/') + one_up_dir = os.path.dirname(root_dir) + if one_up_dir == root_dir: + break + root_dir = one_up_dir + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by + # searching up from the current path. + root_dir = current_dir = os.path.dirname(fullname) + while current_dir != os.path.dirname(current_dir): + if (os.path.exists(os.path.join(current_dir, ".git")) or + os.path.exists(os.path.join(current_dir, ".hg")) or + os.path.exists(os.path.join(current_dir, ".svn"))): + root_dir = current_dir + current_dir = os.path.dirname(current_dir) + + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg")) or + os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. + + For 'chrome/browser/browser.cc', Split() would + return ('chrome/browser', 'browser', '.cc') + + Returns: + A tuple of (directory, basename, extension). + """ + + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project,) + os.path.splitext(rest) + + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] + + def Extension(self): + """File extension - text following the final period, includes that period.""" + return self.Split()[2] + + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) + + def IsSource(self): + """File has a source file extension.""" + return _IsSourceExtension(self.Extension()[1:]) + + +def _ShouldPrintError(category, confidence, linenum): + """If confidence >= verbose, category passes filter and is not suppressed.""" + + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False + + if confidence < _cpplint_state.verbose_level: + return False + + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False + + return True + + +def Error(filename, linenum, category, confidence, message): + """Logs the fact we've found a lint error. + + We log where the error was found, and also our confidence in the error, + that is, how certain we are this is a legitimate style regression, and + not a misidentification or a use that's sometimes justified. + + False positives can be suppressed by the use of + "cpplint(category)" comments on the offending line. These are + parsed into _error_suppressions. + + Args: + filename: The name of the file containing the error. + linenum: The number of the line containing the error. + category: A string used to describe the "category" this bug + falls under: "whitespace", say, or "runtime". Categories + may have a hierarchy separated by slashes: "whitespace/indent". + confidence: A number from 1-5 representing a confidence score for + the error, with 5 meaning that we are certain of the problem, + and 1 meaning that it could be a legitimate construct. + message: The error message. + """ + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + _cpplint_state.PrintError('%s(%s): error cpplint: [%s] %s [%d]\n' % ( + filename, linenum, category, message, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'junit': + _cpplint_state.AddJUnitFailure(filename, linenum, message, category, + confidence) + else: + final_message = '%s:%s: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence) + sys.stderr.write(final_message) + +# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. +_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( + r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') +# Match a single C style comment on the same line. +_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' +# Matches multi-line C style comments. +# This RE is a little bit more complicated than one might expect, because we +# have to take care of space removals tools so we can handle comments inside +# statements better. +# The current rule is: We only clear spaces from both sides when we're at the +# end of the line. Otherwise, we try to remove spaces from the right side, +# if this doesn't work we try on left side but only if there's a non-character +# on the right. +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( + r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + + _RE_PATTERN_C_COMMENTS + r'\s+|' + + r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + + _RE_PATTERN_C_COMMENTS + r')') + + +def IsCppString(line): + """Does line terminate so, that the next symbol is in string constant. + + This function does not consider single-line nor multi-line comments. + + Args: + line: is a partial line of code starting from the 0..n. + + Returns: + True, if next character appended to 'line' is inside a + string constant. + """ + + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + + +def CleanseRawStrings(raw_lines): + """Removes C++11 raw strings from lines. + + Before: + static const char kData[] = R"( + multi-line string + )"; + + After: + static const char kData[] = "" + (replaced by blank line) + ""; + + Args: + raw_lines: list of raw lines. + + Returns: + list of lines with C++11 raw strings replaced by empty strings. + """ + + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + len(delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '""' + + # Look for beginning of a raw string, and replace them with + # empty strings. This is done in a loop to handle multiple raw + # strings on the same line. + while delimiter is None: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + # + # Once we have matched a raw string, we check the prefix of the + # line to make sure that the line is not part of a single line + # comment. It's done this way because we remove raw strings + # before removing comments as opposed to removing comments + # before removing raw strings. This is because there are some + # cpplint checks that requires the comments to be preserved, but + # we don't want to check comments that are inside raw strings. + matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) + if (matched and + not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', + matched.group(1))): + delimiter = ')' + matched.group(2) + '"' + + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + else: + break + + lines_without_raw_strings.append(line) + + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings + + +def FindNextMultiLineCommentStart(lines, lineix): + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) + + +def FindNextMultiLineCommentEnd(lines, lineix): + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) + + +def RemoveMultiLineCommentsFromRange(lines, begin, end): + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '/**/' + + +def RemoveMultiLineComments(filename, lines, error): + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, + 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 + + +def CleanseComments(line): + """Removes //-comments and single-line C-style /* */ comments. + + Args: + line: A line of C++ source. + + Returns: + The line with single-line comments removed. + """ + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + + +class CleansedLines(object): + """Holds 4 copies of all lines with different preprocessing applied to them. + + 1) elided member contains lines without strings and comments. + 2) lines member contains lines without comments. + 3) raw_lines member contains all the lines without processing. + 4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw + strings removed. + All these members are of , and of the same length. + """ + + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append(CleanseComments( + self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. + + We nix strings first so we're not fooled by text like '"http://"' + + Args: + elided: The line being processed. + + Returns: + The line with collapsed strings. + """ + if _RE_PATTERN_INCLUDE.match(elided): + return elided + + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + + # Replace quoted strings and digit separators. Both single quotes + # and double quotes are processed in the same loop, otherwise + # nested quotes wouldn't work. + collapsed = '' + while True: + # Find the first quote character + match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) + if not match: + collapsed += elided + break + head, quote, tail = match.groups() + + if quote == '"': + # Collapse double quoted strings + second_quote = tail.find('"') + if second_quote >= 0: + collapsed += head + '""' + elided = tail[second_quote + 1:] + else: + # Unmatched double quote, don't bother processing the rest + # of the line since this is probably a multiline string. + collapsed += elided + break + else: + # Found single quote, check nearby text to eliminate digit separators. + # + # There is no special handling for floating point here, because + # the integer/fractional/exponent parts would all be parsed + # correctly as long as there are digits on both sides of the + # separator. So we are fine as long as we don't see something + # like "0.'3" (gcc 4.9.0 will not allow this literal). + if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): + match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) + collapsed += head + match_literal.group(1).replace("'", '') + elided = match_literal.group(2) + else: + second_quote = tail.find('\'') + if second_quote >= 0: + collapsed += head + "''" + elided = tail[second_quote + 1:] + else: + # Unmatched single quote + collapsed += elided + break + + return collapsed + + +def FindEndOfExpressionInLine(line, startpos, stack): + """Find the position just after the end of current parenthesized expression. + + Args: + line: a CleansedLines line. + startpos: start searching at this position. + stack: nesting stack at startpos. + + Returns: + On finding matching end: (index just after matching end, None) + On finding an unclosed expression: (-1, None) + Otherwise: (-1, new stack at end of this line) + """ + for i in xrange(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack + stack.append(char) + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + if stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + elif i > 0 and Search(r'\boperator\s*$', line[0:i]): + # operator<, don't add to stack + continue + else: + # Tentative start of template argument list + stack.append('<') + elif char in ')]}': + # Found end of parenthesized expression. + # + # If we are currently expecting a matching '>', the pending '<' + # must have been an operator. Remove them from expression stack. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + if ((stack[-1] == '(' and char == ')') or + (stack[-1] == '[' and char == ']') or + (stack[-1] == '{' and char == '}')): + stack.pop() + if not stack: + return (i + 1, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == '>': + # Found potential end of template argument list. + + # Ignore "->" and operator functions + if (i > 0 and + (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): + continue + + # Pop the stack if there is a matching '<'. Otherwise, ignore + # this '>' since it must be an operator. + if stack: + if stack[-1] == '<': + stack.pop() + if not stack: + return (i + 1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '>', the matching '<' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + + # Did not find end of expression or unbalanced parentheses on this line + return (-1, stack) + + +def CloseExpression(clean_lines, linenum, pos): + """If input points to ( or { or [ or <, finds the position that closes it. + + If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the + linenum/pos that correspond to the closing of the expression. + + TODO(unknown): cpplint spends a fair bit of time matching parentheses. + Ideally we would want to index all opening and closing parentheses once + and have CloseExpression be just a simple lookup, but due to preprocessor + tricks, this is not so easy. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *past* the closing brace, or + (line, len(lines), -1) if we never find a close. Note we ignore + strings and comments when matching; and the line we return is the + 'cleansed' line at linenum. + """ + + line = clean_lines.elided[linenum] + if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): + return (line, clean_lines.NumLines(), -1) + + # Check first line + (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) + if end_pos > -1: + return (line, linenum, end_pos) + + # Continue scanning forward + while stack and linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if end_pos > -1: + return (line, linenum, end_pos) + + # Did not find end of expression before end of file, give up + return (line, clean_lines.NumLines(), -1) + + +def FindStartOfExpressionInLine(line, endpos, stack): + """Find position at the matching start of current expression. + + This is almost the reverse of FindEndOfExpressionInLine, but note + that the input position and returned position differs by 1. + + Args: + line: a CleansedLines line. + endpos: start searching at this position. + stack: nesting stack at endpos. + + Returns: + On finding matching start: (index at matching start, None) + On finding an unclosed expression: (-1, None) + Otherwise: (-1, new stack at beginning of this line) + """ + i = endpos + while i >= 0: + char = line[i] + if char in ')]}': + # Found end of expression, push to expression stack + stack.append(char) + elif char == '>': + # Found potential end of template argument list. + # + # Ignore it if it's a "->" or ">=" or "operator>" + if (i > 0 and + (line[i - 1] == '-' or + Match(r'\s>=\s', line[i - 1:]) or + Search(r'\boperator\s*$', line[0:i]))): + i -= 1 + else: + stack.append('>') + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + i -= 1 + else: + # If there is a matching '>', we can pop the expression stack. + # Otherwise, ignore this '<' since it must be an operator. + if stack and stack[-1] == '>': + stack.pop() + if not stack: + return (i, None) + elif char in '([{': + # Found start of expression. + # + # If there are any unmatched '>' on the stack, they must be + # operators. Remove those. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + if ((char == '(' and stack[-1] == ')') or + (char == '[' and stack[-1] == ']') or + (char == '{' and stack[-1] == '}')): + stack.pop() + if not stack: + return (i, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '<', the matching '>' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + + i -= 1 + + return (-1, stack) + + +def ReverseCloseExpression(clean_lines, linenum, pos): + """If input points to ) or } or ] or >, finds the position that opens it. + + If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the + linenum/pos that correspond to the opening of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *at* the opening brace, or + (line, 0, -1) if we never find the matching opening brace. Note + we ignore strings and comments when matching; and the line we + return is the 'cleansed' line at linenum. + """ + line = clean_lines.elided[linenum] + if line[pos] not in ')}]>': + return (line, 0, -1) + + # Check last line + (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while stack and linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find start of expression before beginning of file, give up + return (line, 0, -1) + + +def CheckForCopyright(filename, lines, error): + """Logs an error if no Copyright message appears at the top of the file.""" + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, + 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') + + +def GetIndentLevel(line): + """Return the number of leading spaces in line. + + Args: + line: A string to check. + + Returns: + An integer count of leading spaces, possibly zero. + """ + indent = Match(r'^( *)\S', line) + if indent: + return len(indent.group(1)) + else: + return 0 + +def PathSplitToList(path): + """Returns the path split into a list by the separator. + + Args: + path: An absolute or relative path (e.g. '/a/b/c/' or '../a') + + Returns: + A list of path components (e.g. ['a', 'b', 'c]). + """ + lst = [] + while True: + (head, tail) = os.path.split(path) + if head == path: # absolute paths end + lst.append(head) + break + if tail == path: # relative paths end + lst.append(tail) + break + + path = head + lst.append(tail) + + lst.reverse() + return lst + +def GetHeaderGuardCPPVariable(filename): + """Returns the CPP variable that should be used as a header guard. + + Args: + filename: The name of a C++ header file. + + Returns: + The CPP variable that should be used as a header guard in the + named file. + + """ + + # Restores original filename in case that cpplint is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) + # Replace 'c++' with 'cpp'. + filename = filename.replace('C++', 'cpp').replace('c++', 'cpp') + + fileinfo = FileInfo(filename) + file_path_from_root = fileinfo.RepositoryName() + + def FixupPathFromRoot(): + if _root_debug: + sys.stderr.write("\n_root fixup, _root = '%s', repository name = '%s'\n" + % (_root, fileinfo.RepositoryName())) + + # Process the file path with the --root flag if it was set. + if not _root: + if _root_debug: + sys.stderr.write("_root unspecified\n") + return file_path_from_root + + def StripListPrefix(lst, prefix): + # f(['x', 'y'], ['w, z']) -> None (not a valid prefix) + if lst[:len(prefix)] != prefix: + return None + # f(['a, 'b', 'c', 'd'], ['a', 'b']) -> ['c', 'd'] + return lst[(len(prefix)):] + + # root behavior: + # --root=subdir , lstrips subdir from the header guard + maybe_path = StripListPrefix(PathSplitToList(file_path_from_root), + PathSplitToList(_root)) + + if _root_debug: + sys.stderr.write(("_root lstrip (maybe_path=%s, file_path_from_root=%s," + + " _root=%s)\n") % (maybe_path, file_path_from_root, _root)) + + if maybe_path: + return os.path.join(*maybe_path) + + # --root=.. , will prepend the outer directory to the header guard + full_path = fileinfo.FullName() + root_abspath = os.path.abspath(_root) + + maybe_path = StripListPrefix(PathSplitToList(full_path), + PathSplitToList(root_abspath)) + + if _root_debug: + sys.stderr.write(("_root prepend (maybe_path=%s, full_path=%s, " + + "root_abspath=%s)\n") % (maybe_path, full_path, root_abspath)) + + if maybe_path: + return os.path.join(*maybe_path) + + if _root_debug: + sys.stderr.write("_root ignore, returning %s\n" % (file_path_from_root)) + + # --root=FAKE_DIR is ignored + return file_path_from_root + + file_path_from_root = FixupPathFromRoot() + return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_' + + +def CheckForHeaderGuard(filename, clean_lines, error): + """Checks that the file contains a header guard. + + Logs an error if no #ifndef header guard is present. For other + headers, checks that the full pathname is used. + + Args: + filename: The name of the C++ header file. + clean_lines: A CleansedLines instance containing the file. + error: The function to call with any errors found. + """ + + # Don't check for header guards if there are error suppression + # comments somewhere in this file. + # + # Because this is silencing a warning for a nonexistent line, we + # only support the very specific NOLINT(build/header_guard) syntax, + # and not the general NOLINT or NOLINT(*) syntax. + raw_lines = clean_lines.lines_without_raw_strings + for i in raw_lines: + if Search(r'//\s*NOLINT\(build/header_guard\)', i): + return + + # notallow pragma once + for i in raw_lines: + if Search(r'^\s*#pragma\s+once', i): + error(filename, 0, 'build/header_guard', 5, + 'Pragma once not allowed, please use ifndef Instead') + return + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = '' + ifndef_linenum = 0 + define = '' + endif = '' + endif_linenum = 0 + for linenum, line in enumerate(raw_lines): + linesplit = line.split() + if len(linesplit) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + + if not ifndef or not define or ifndef != define: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, + error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + # Check for "//" comments on endif line. + ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, + error) + match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) + if match: + if match.group(1) == '_': + # Issue low severity warning for deprecated double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif // %s"' % cppvar) + return + + # Didn't find the corresponding "//" comment. If this file does not + # contain any "//" comments at all, it could be that the compiler + # only wants "/**/" comments, look for those instead. + no_single_line_comments = True + for i in xrange(1, len(raw_lines) - 1): + line = raw_lines[i] + if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): + no_single_line_comments = False + break + + if no_single_line_comments: + match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + if match: + if match.group(1) == '_': + # Low severity warning for double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif /* %s */"' % cppvar) + return + + # Didn't find anything + error(filename, endif_linenum, 'build/header_guard', 5, + '#endif line should be "#endif // %s"' % cppvar) + + +def CheckHeaderFileIncluded(filename, include_state, error): + """Logs an error if a source file does not include its header.""" + + # Do not check test files + fileinfo = FileInfo(filename) + if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()): + return + + for ext in GetHeaderExtensions(): + basefilename = filename[0:len(filename) - len(fileinfo.Extension())] + headerfile = basefilename + '.' + ext + if not os.path.exists(headerfile): + continue + headername = FileInfo(headerfile).RepositoryName() + first_include = None + for section_list in include_state.include_list: + for f in section_list: + if headername in f[0] or f[0] in headername: + return + if not first_include: + first_include = f[1] + + error(filename, first_include, 'build/include', 5, + '%s should include its header file %s' % (fileinfo.RepositoryName(), + headername)) + + +def CheckForBadCharacters(filename, lines, error): + """Logs an error for each line containing bad characters. + + Two kinds of bad characters: + + 1. Unicode replacement characters: These indicate that either the file + contained invalid UTF-8 (likely) or Unicode replacement characters (which + it shouldn't). Note that it's possible for this to throw off line + numbering if the invalid UTF-8 occurred adjacent to a newline. + + 2. NUL bytes. These are problematic for some tools. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + for linenum, line in enumerate(lines): + if unicode_escape_decode('\ufffd') in line: + error(filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + + +def CheckForNewlineAtEOF(filename, lines, error): + """Logs an error if there is no newline char at the end of the file. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') + + +def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): + """Logs an error if we see /* ... */ or "..." that extend past one line. + + /* ... */ comments are legit inside macros, for one line. + Otherwise, we prefer // comments, so it's ok to warn about the + other. Likewise, it's ok for strings to extend across multiple + lines, as long as a line continuation character (backslash) + terminates each line. Although not currently prohibited by the C++ + style guide, it's ugly and unnecessary. We don't do well with either + in this lint program, so we warn about both. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') + + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') + + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') + + +# (non-threadsafe name, thread-safe alternative, validation pattern) +# +# The validation pattern is used to eliminate false positives such as: +# _rand(); // false positive due to substring match. +# ->rand(); // some member function rand(). +# ACMRandom rand(seed); // some variable named rand. +# ISAACRandom rand(); // another variable named rand. +# +# Basically we require the return value of these functions to be used +# in some expression context on the same line by matching on some +# operator before the function name. This eliminates constructors and +# member function calls. +_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)' +_THREADING_LIST = ( + ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'), + ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'), + ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'), + ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'), + ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'), + ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'), + ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'), + ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), + ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), + ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), + ('strtok(', 'strtok_r(', + _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), + ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), + ) + + +def CheckPosixThreading(filename, clean_lines, linenum, error): + """Checks for calls to thread-unsafe functions. + + Much code has been originally written without consideration of + multi-threading. Also, engineers are relying on their old experience; + they have learned posix before threading extensions were added. These + tests guide the engineers to use thread-safe functions (when using + posix directly). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: + # Additional pattern matching check to confirm that this is the + # function we are looking for + if Search(pattern, line): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_func + + '...) instead of ' + single_thread_func + + '...) for improved thread safety.') + + +def CheckVlogArguments(filename, clean_lines, linenum, error): + """Checks that VLOG() is only used for defining a logging level. + + For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and + VLOG(FATAL) are not. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error(filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + +# Matches invalid increment: *count++, which moves pointer instead of +# incrementing a value. +_RE_PATTERN_INVALID_INCREMENT = re.compile( + r'^\s*\*\w+(\+\+|--);') + + +def CheckInvalidIncrement(filename, clean_lines, linenum, error): + """Checks for invalid increment *count++. + + For example following function: + void increment_counter(int* count) { + *count++; + } + is invalid, because it effectively does count++, moving pointer, and should + be replaced with ++*count, (*count)++ or *count += 1. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error(filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') + + +def IsMacroDefinition(clean_lines, linenum): + if Search(r'^#define', clean_lines[linenum]): + return True + + if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): + return True + + return False + + +def IsForwardClassDeclaration(clean_lines, linenum): + return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) + + +class _BlockInfo(object): + """Stores information about a generic block of code.""" + + def __init__(self, linenum, seen_open_brace): + self.starting_linenum = linenum + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + self.check_namespace_indentation = False + + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. + + This is mostly for checking the text after the class identifier + and the "{", usually where the base class is specified. For other + blocks, there isn't much to check, so we always pass. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. + + This is mostly used for checking end of namespace comments. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + def IsBlockInfo(self): + """Returns true if this block is a _BlockInfo. + + This is convenient for verifying that an object is an instance of + a _BlockInfo, but not an instance of any of the derived classes. + + Returns: + True for this class, False for derived classes. + """ + return self.__class__ == _BlockInfo + + +class _ExternCInfo(_BlockInfo): + """Stores information about an 'extern "C"' block.""" + + def __init__(self, linenum): + _BlockInfo.__init__(self, linenum, True) + + +class _ClassInfo(_BlockInfo): + """Stores information about a class.""" + + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name + self.is_derived = False + self.check_namespace_indentation = True + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False + + # Remember initial indentation level for this class. Using raw_lines here + # instead of elided to account for leading comments. + self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) + + # Try to find the end of the class. This will be confused by things like: + # class A { + # } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break + + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + # If there is a DISALLOW macro, it should appear near the end of + # the class. + seen_last_thing_in_class = False + for i in xrange(linenum - 1, self.starting_linenum, -1): + match = Search( + r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + + self.name + r'\)', + clean_lines.elided[i]) + if match: + if seen_last_thing_in_class: + error(filename, i, 'readability/constructors', 3, + match.group(1) + ' should be the last thing in the class') + break + + if not Match(r'^\s*$', clean_lines.elided[i]): + seen_last_thing_in_class = True + + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error(filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % parent) + + +class _NamespaceInfo(_BlockInfo): + """Stores information about a namespace.""" + + def __init__(self, name, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name or '' + self.check_namespace_indentation = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] + + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 + and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)): + return + + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' + + re.escape(self.name) + r'[\*/\.\\\s]*$'), + line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) + else: + # Anonymous namespace + if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + # If "// namespace anonymous" or "// anonymous namespace (more text)", + # mention "// anonymous namespace" as an acceptable form + if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line): + error(filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ' or "// anonymous namespace"') + else: + error(filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"') + + +class _PreprocessorInfo(object): + """Stores checkpoints of nesting stacks when #if/#else is seen.""" + + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if + + # The entire nesting stack up to #else + self.stack_before_else = [] + + # Whether we have already seen #else or #elif + self.seen_else = False + + +class NestingState(object): + """Holds states related to parsing braces.""" + + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] + + # Top of the previous stack before each Update(). + # + # Because the nesting_stack is updated at the end of each line, we + # had to do some convoluted checks to find out what is the current + # scope at the beginning of the line. This check is simplified by + # saving the previous top of nesting stack. + # + # We could save the full stack, but we only need the top. Copying + # the full nesting stack would slow down cpplint by ~10%. + self.previous_stack_top = [] + + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] + + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. + + Returns: + True if we have seen the opening brace, False if the innermost + block is still expecting an opening brace. + """ + return (not self.stack) or self.stack[-1].seen_open_brace + + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. + + Returns: + True if top of the stack is a namespace block, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + + def InExternC(self): + """Check if we are currently one level inside an 'extern "C"' block. + + Returns: + True if top of the stack is an extern block, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _ExternCInfo) + + def InClassDeclaration(self): + """Check if we are currently one level inside a class or struct declaration. + + Returns: + True if top of the stack is a class/struct, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _ClassInfo) + + def InAsmBlock(self): + """Check if we are currently one level inside an inline ASM block. + + Returns: + True if the top of the stack is a block containing inline ASM. + """ + return self.stack and self.stack[-1].inline_asm != _NO_ASM + + def InTemplateArgumentList(self, clean_lines, linenum, pos): + """Check if current position is inside template argument list. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: position just after the suspected template argument. + Returns: + True if (linenum, pos) is inside template arguments. + """ + while linenum < clean_lines.NumLines(): + # Find the earliest character that might indicate a template argument + line = clean_lines.elided[linenum] + match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) + if not match: + linenum += 1 + pos = 0 + continue + token = match.group(1) + pos += len(match.group(0)) + + # These things do not look like template argument list: + # class Suspect { + # class Suspect x; } + if token in ('{', '}', ';'): return False + + # These things look like template argument list: + # template + # template + # template + # template + if token in ('>', '=', '[', ']', '.'): return True + + # Check if token is an unmatched '<'. + # If not, move on to the next character. + if token != '<': + pos += 1 + if pos >= len(line): + linenum += 1 + pos = 0 + continue + + # We can't be sure if we just find a single '<', and need to + # find the matching '>'. + (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) + if end_pos < 0: + # Not sure if template argument list or syntax error in file + return False + linenum = end_line + pos = end_pos + return False + + def UpdatePreprocessor(self, line): + """Update preprocessor stack. + + We need to handle preprocessors due to classes like this: + #ifdef SWIG + struct ResultDetailsPageElementExtensionPoint { + #else + struct ResultDetailsPageElementExtensionPoint : public Extension { + #endif + + We make the following assumptions (good enough for most files): + - Preprocessor condition evaluates to true from #if up to first + #else/#elif/#endif. + + - Preprocessor condition evaluates to false from #else/#elif up + to #endif. We still perform lint checks on these lines, but + these do not affect nesting stack. + + Args: + line: current line to check. + """ + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) + + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass + + # TODO(unknown): Update() is too long, but we will refactor later. + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remember top of the previous nesting stack. + # + # The stack is always pushed/popped and not modified in place, so + # we can just do a shallow copy instead of copy.deepcopy. Using + # deepcopy would slow down cpplint by ~28%. + if self.stack: + self.previous_stack_top = self.stack[-1] + else: + self.previous_stack_top = None + + # Update pp_stack + self.UpdatePreprocessor(line) + + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and + inner_block.open_parentheses == 1 and + _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was _END_ASM, + # we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM and + inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + class_decl_match = Match( + r'^(\s*(?:template\s*<[\w\s<>,:=]*>\s*)?' + r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' + r'(.*)$', line) + if (class_decl_match and + (not self.stack or self.stack[-1].open_parentheses == 0)): + # We do not want to accept classes that are actually template arguments: + # template , + # template class Ignore3> + # void Function() {}; + # + # To avoid template argument cases, we scan forward and look for + # an unmatched '>'. If we see one, assume we are inside a + # template argument list. + end_declaration = len(class_decl_match.group(1)) + if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): + self.stack.append(_ClassInfo( + class_decl_match.group(3), class_decl_match.group(2), + clean_lines, linenum)) + line = class_decl_match.group(4) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', + line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 and + Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error(filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % ( + access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + elif Match(r'^extern\s*"[^"]*"\s*\{', line): + self.stack.append(_ExternCInfo(linenum)) + else: + self.stack.append(_BlockInfo(linenum, True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. + + Returns: + A _ClassInfo object if we are inside a class, or None otherwise. + """ + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None + + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. + + Call this when all lines in a file have been processed. + Args: + filename: The name of the current file. + error: The function to call with any errors found. + """ + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error(filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error(filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, + nesting_state, error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + + Complain about several constructs which gcc-2 accepts, but which are + not standard C++. Warning about these in lint is one way to ease the + transition to new compilers. + - put storage class first (e.g. "static const" instead of "const static"). + - "%lld" instead of %qd" in printf-type functions. + - "%1$d" is non-standard in printf-type functions. + - "\%" is an undefined character escape sequence. + - text after #endif is not allowed. + - invalid inner-style forward declaration. + - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error(filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + explicit_constructor_match = Match( + r'\s+(?:(?:inline|constexpr)\s+)*(explicit\s+)?' + r'(?:(?:inline|constexpr)\s+)*%s\s*' + r'\(((?:[^()]|\([^()]*\))*)\)' + % re.escape(base_classname), + line) + + if explicit_constructor_match: + is_marked_explicit = explicit_constructor_match.group(1) + + if not explicit_constructor_match.group(2): + constructor_args = [] + else: + constructor_args = explicit_constructor_match.group(2).split(',') + + # collapse arguments so that commas in template parameter lists and function + # argument parameter lists don't split arguments in two + i = 0 + while i < len(constructor_args): + constructor_arg = constructor_args[i] + while (constructor_arg.count('<') > constructor_arg.count('>') or + constructor_arg.count('(') > constructor_arg.count(')')): + constructor_arg += ',' + constructor_args[i + 1] + del constructor_args[i + 1] + constructor_args[i] = constructor_arg + i += 1 + + variadic_args = [arg for arg in constructor_args if '&&...' in arg] + defaulted_args = [arg for arg in constructor_args if '=' in arg] + noarg_constructor = (not constructor_args or # empty arg list + # 'void' arg specifier + (len(constructor_args) == 1 and + constructor_args[0].strip() == 'void')) + onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg + not noarg_constructor) or + # all but at most one arg defaulted + (len(constructor_args) >= 1 and + not noarg_constructor and + len(defaulted_args) >= len(constructor_args) - 1) or + # variadic arguments with zero or one argument + (len(constructor_args) <= 2 and + len(variadic_args) >= 1)) + initializer_list_constructor = bool( + onearg_constructor and + Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) + copy_constructor = bool( + onearg_constructor and + Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' + % re.escape(base_classname), constructor_args[0].strip())) + + if (not is_marked_explicit and + onearg_constructor and + not initializer_list_constructor and + not copy_constructor): + if defaulted_args or variadic_args: + error(filename, linenum, 'runtime/explicit', 5, + 'Constructors callable with one argument ' + 'should be marked explicit.') + else: + error(filename, linenum, 'runtime/explicit', 5, + 'Single-parameter constructors should be marked explicit.') + elif is_marked_explicit and not onearg_constructor: + if noarg_constructor: + error(filename, linenum, 'runtime/explicit', 5, + 'Zero-parameter constructors should not be marked explicit.') + + +def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): + """Checks for the correctness of various spacing around function calls. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', + r'\bfor\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', + r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break + + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and + not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and + not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and + not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and + not Search(r'\bcase\s+\(', fncall)): + # TODO(unknown): Space after an operator function seem to be a common + # error, silence those for now by restricting them to highest verbosity. + if Search(r'\boperator_*\b', line): + error(filename, linenum, 'whitespace/parens', 0, + 'Extra space before ( in function call') + else: + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') + + +def IsBlankLine(line): + """Returns true if the given line is blank. + + We consider a line to be blank if the line is empty or consists of + only white spaces. + + Args: + line: A line of a string. + + Returns: + True, if the given line is blank. + """ + return not line or line.isspace() + + +def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error): + is_namespace_indent_item = ( + len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and + nesting_state.previous_stack_top == nesting_state.stack[-2]) + + if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + clean_lines.elided, line): + CheckItemIndentationInNamespace(filename, clean_lines.elided, + line, error) + + +def CheckForFunctionLengths(filename, clean_lines, linenum, + function_state, error): + """Reports for long function bodies. + + For an overview why this is done, see: + https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions + + Uses a simplistic algorithm assuming other style guidelines + (especially spacing) are followed. + Only checks unindented functions, so class members are unchecked. + Trivial bodies are unchecked, so constructors with huge initializer lists + may be missed. + Blank/comment lines are not counted so as to avoid encouraging the removal + of vertical space and comments just to get through a lint check. + NOLINT *on the last line of a function* disables this check. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + function_state: Current function name and lines in body so far. + error: The function to call with any errors found. + """ + lines = clean_lines.lines + line = lines[linenum] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or ( + not Match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_linenum in xrange(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + if Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. + + +_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') + + +def CheckComment(line, filename, linenum, next_line_start, error): + """Checks for common mistakes in comments. + + Args: + line: The line in question. + filename: The name of the current file. + linenum: The number of the line to check. + next_line_start: The first non-whitespace column of the next line. + error: The function to call with any errors found. + """ + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: + # Allow one space for new scopes, two spaces otherwise: + if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and + ((commentpos >= 1 and + line[commentpos-1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos-2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + + # Checks for common mistakes in TODO comments. + comment = line[commentpos:] + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + + # If the comment contains an alphanumeric character, there + # should be a space somewhere between it and the // unless + # it's a /// or //! Doxygen comment. + if (Match(r'//[^ ]*\w', comment) and + not Match(r'(///|//\!)(\s+|$)', comment)): + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') + + +def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): + """Checks for the correctness of various spacing issues in the code. + + Things we check for: spaces around operators, spaces after + if/for/while/switch, no spaces around parens in function calls, two + spaces between code and comment, don't start a block with a blank + line, don't end a function with a blank line, don't add a blank line + after public/protected/private, don't have too many blank lines in a row. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { + # + # } + # + # A warning about missing end of namespace comments will be issued instead. + # + # Also skip blank line checks for 'extern "C"' blocks, which are formatted + # like namespaces. + if (IsBlankLine(line) and + not nesting_state.InNamespaceBody() and + not nesting_state.InExternC()): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum-2 + while (search_position >= 0 + and Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) + or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line + and Match(r'\s*}', next_line) + and next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, check comments + next_line_start = 0 + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + next_line_start = len(next_line) - len(next_line.lstrip()) + CheckComment(line, filename, linenum, next_line_start, error) + + # get rid of comments and strings + line = clean_lines.elided[linenum] + + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []', 'return []() {};', or 'auto [abc, ...] = ...;'. + if Search(r'\w\s+\[', line) and not Search(r'(?:auto&?|delete|return)\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Extra space before [') + + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search(r'for *\(.*[^:]:[^: ]', line) or + Search(r'for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') + + +def CheckOperatorSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing around operators. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Don't try to do spacing checks for operator methods. Do this by + # replacing the troublesome characters with something else, + # preserving column position for all other characters. + # + # The replacement is done repeatedly to avoid false positives from + # operators that call operators. + while True: + match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + if match: + line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) + else: + break + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if ((Search(r'[\w.]=', line) or + Search(r'=[\w.]', line)) + and not Search(r'\b(if|while|for) ', line) + # Operators taken from [lex.operators] in C++11 standard. + and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) + and not Search(r'operator=', line)): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + # + # If the operator is followed by a comma, assume it's be used in a + # macro context and don't do any checks. This avoids false + # positives. + # + # Note that && is not included here. This is because there are too + # many false positives due to RValue references. + match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around %s' % match.group(1)) + elif not Match(r'#.*include', line): + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + if match: + (_, _, end_pos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + if end_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') + + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + if match: + (_, _, start_pos) = ReverseCloseExpression( + clean_lines, linenum, len(match.group(1))) + if start_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # + # We also allow operators following an opening parenthesis, since + # those tend to be macros that deal with operators. + match = Search(r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line) + if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and + not (match.group(1) == 'operator' and match.group(2) == ';')): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <<') + + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >>') + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) + + +def CheckParenthesisSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing around parentheses. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # No spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: + error(filename, linenum, 'whitespace/parens', 5, + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', + line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) + + +def CheckCommaSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing near commas and semicolons. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + raw = clean_lines.lines_without_raw_strings + line = clean_lines.elided[linenum] + + # You should always have a space after a comma (either as fn arg or operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and + Search(r',[^,\s]', raw[linenum])): + error(filename, linenum, 'whitespace/comma', 3, + 'Missing space after ,') + + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') + + +def _IsType(clean_lines, nesting_state, expr): + """Check if expression looks like a type name, returns true if so. + + Args: + clean_lines: A CleansedLines instance containing the file. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + expr: The expression to check. + Returns: + True, if token looks like a type. + """ + # Keep only the last token in the expression + last_word = Match(r'^.*(\b\S+)$', expr) + if last_word: + token = last_word.group(1) + else: + token = expr + + # Match native types and stdint types + if _TYPES.match(token): + return True + + # Try a bit harder to match templated types. Walk up the nesting + # stack until we find something that resembles a typename + # declaration for what we are looking for. + typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) + + r'\b') + block_index = len(nesting_state.stack) - 1 + while block_index >= 0: + if isinstance(nesting_state.stack[block_index], _NamespaceInfo): + return False + + # Found where the opening brace is. We want to scan from this + # line up to the beginning of the function, minus a few lines. + # template + # class C + # : public ... { // start scanning here + last_line = nesting_state.stack[block_index].starting_linenum + + next_block_start = 0 + if block_index > 0: + next_block_start = nesting_state.stack[block_index - 1].starting_linenum + first_line = last_line + while first_line >= next_block_start: + if clean_lines.elided[first_line].find('template') >= 0: + break + first_line -= 1 + if first_line < next_block_start: + # Didn't find any "template" keyword before reaching the next block, + # there are probably no template things to check for this block + block_index -= 1 + continue + + # Look for typename in the specified range + for i in xrange(first_line, last_line + 1, 1): + if Search(typename_pattern, clean_lines.elided[i]): + return True + block_index -= 1 + + return False + + +def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error): + """Checks for horizontal spacing near commas. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces when they are delimiting blocks, classes, namespaces etc. + # And since you should never have braces at the beginning of a line, + # this is an easy test. Except that braces used for initialization don't + # follow the same rule; we often don't want spaces before those. + match = Match(r'^(.*[^ ({>]){', line) + + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # ternary = expr ? new type{} : nullptr; + # OuterTemplate{}> + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<>]:". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + leading_text = match.group(1) + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in xrange(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + # We also suppress warnings for `uint64_t{expression}` etc., as the style + # guide recommends brace initialization for integral types to avoid + # overflow/truncation. + if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text) + and not _IsType(clean_lines, nesting_state, leading_text)): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and + not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') + + +def IsDecltype(clean_lines, linenum, column): + """Check if the token ending on (linenum, column) is decltype(). + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: the number of the line to check. + column: end column of the token to check. + Returns: + True if this token is decltype() expression, False otherwise. + """ + (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) + if start_col < 0: + return False + if Search(r'\bdecltype\s*$', text[0:start_col]): + return True + return False + +def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): + """Checks for additional blank line issues related to sections. + + Currently the only thing checked here is blank line before protected/private. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + class_info: A _ClassInfo objects. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 or + linenum <= class_info.starting_linenum): + return + + matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as can be + # common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) and + not Search(r'\b(class|struct)\b', prev_line) and + not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error(filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % matched.group(1)) + + +def GetPreviousNonBlankLine(clean_lines, linenum): + """Return the most recent non-blank line and its line number. + + Args: + clean_lines: A CleansedLines instance containing the file contents. + linenum: The number of the line to check. + + Returns: + A tuple with two elements. The first element is the contents of the last + non-blank line before the current line, or the empty string if this is the + first non-blank line. The second is the line number of that line, or -1 + if this is the first non-blank line. + """ + + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) + + +def CheckBraces(filename, clean_lines, linenum, error): + """Looks for misplaced braces (e.g. at the end of line). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is using + # braces in a block to explicitly create a new scope, which is commonly used + # to control the lifetime of stack-allocated variables. Braces are also + # used for brace initializers inside function calls. We don't detect this + # perfectly: we just don't complain if the last non-whitespace character on + # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the + # previous line starts a preprocessor block. We also allow a brace on the + # following line if it is part of an array initialization and would not fit + # within the 80 character limit of the preceding line. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) and + not Match(r'\s*#', prevline) and + not (GetLineWidth(prevline) > _line_length - 2 and '[]' in prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'else if\s*\(', line): # could be multi-line if + brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + brace_on_right = endline[endpos:].find('{') != -1 + if brace_on_left != brace_on_right: # must be brace after if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Check single-line if/else bodies. The style guide says 'curly braces are not + # required for single-line statements'. We additionally allow multi-line, + # single statements, but we reject anything with more than one semicolon in + # it. This means that the first semicolon after the if should be at the end of + # its line, and the line after that should have an indent level equal to or + # lower than the if. We also check for ambiguous if/else nesting without + # braces. + if_else_match = Search(r'\b(if\s*\(|else\b)', line) + if if_else_match and not Match(r'\s*#', line): + if_indent = GetIndentLevel(line) + endline, endlinenum, endpos = line, linenum, if_else_match.end() + if_match = Search(r'\bif\s*\(', line) + if if_match: + # This could be a multiline if condition, so find the end first. + pos = if_match.end() - 1 + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) + # Check for an opening brace, either directly after the if or on the next + # line. If found, this isn't a single-statement conditional. + if (not Match(r'\s*{', endline[endpos:]) + and not (Match(r'\s*$', endline[endpos:]) + and endlinenum < (len(clean_lines.elided) - 1) + and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): + while (endlinenum < len(clean_lines.elided) + and ';' not in clean_lines.elided[endlinenum][endpos:]): + endlinenum += 1 + endpos = 0 + if endlinenum < len(clean_lines.elided): + endline = clean_lines.elided[endlinenum] + # We allow a mix of whitespace and closing braces (e.g. for one-liner + # methods) and a single \ after the semicolon (for macros) + endpos = endline.find(';') + if not Match(r';[\s}]*(\\?)$', endline[endpos:]): + # Semicolon isn't the last character, there's something trailing. + # Output a warning if the semicolon is not contained inside + # a lambda expression. + if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', + endline): + error(filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces') + elif endlinenum < len(clean_lines.elided) - 1: + # Make sure the next line is dedented + next_line = clean_lines.elided[endlinenum + 1] + next_indent = GetIndentLevel(next_line) + # With ambiguous nested if statements, this will error out on the + # if that *doesn't* match the else, regardless of whether it's the + # inner one or outer one. + if (if_match and Match(r'\s*else\b', next_line) + and next_indent != if_indent): + error(filename, linenum, 'readability/braces', 4, + 'Else clause should be indented at the same level as if. ' + 'Ambiguous nested if/else chains require braces.') + elif next_indent > if_indent: + error(filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces') + + +def CheckTrailingSemicolon(filename, clean_lines, linenum, error): + """Looks for redundant trailing semicolon. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] + + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization, there are more places where semicolons are + # required than not, so we use a whitelist approach to check these + # rather than a blacklist. These are the places where "};" should + # be replaced by just "}": + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; + # + # 2. else block: + # if (...) else {}; + # + # 3. const member function: + # Function(...) const {}; + # + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement a whitelist of safe macros instead of a blacklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the whitelist wrong means some extra + # semicolons, while the downside for getting the blacklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on + # - Compound literals + # - Lambdas + # - alignas specifier with anonymous structs + # - decltype + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression( + clean_lines, linenum, closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix) + func = Match(r'^(.*\])\s*$', line_prefix) + if ((macro and + macro.group(1) not in ( + 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or + Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or + Search(r'\bdecltype$', line_prefix) or + Search(r'\s+=\s*$', line_prefix)): + match = None + if (match and + opening_parenthesis[1] > 1 and + Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): + # Multi-line lambda-expression + match = None + + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) + + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + + # We need to check the line forward for NOLINT + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[endlinenum-1], endlinenum-1, + error) + ParseNolintSuppressions(filename, raw_lines[endlinenum], endlinenum, + error) + + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") + + +def CheckEmptyBlockBody(filename, clean_lines, linenum, error): + """Look for empty loop/conditional body with only a single semicolon. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression. + (end_line, end_linenum, end_pos) = CloseExpression( + clean_lines, linenum, line.find('(')) + + # Output warning if what follows the condition expression is a semicolon. + # No warning for all other cases, including whitespace or newline, since we + # have a separate check for semicolons preceded by whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') + + # Check for if statements that have completely empty bodies (no comments) + # and no else clauses. + if end_pos >= 0 and matched.group(1) == 'if': + # Find the position of the opening { for the if statement. + # Return without logging an error if it has no brackets. + opening_linenum = end_linenum + opening_line_fragment = end_line[end_pos:] + # Loop until EOF or find anything that's not whitespace or opening {. + while not Search(r'^\s*\{', opening_line_fragment): + if Search(r'^(?!\s*$)', opening_line_fragment): + # Conditional has no brackets. + return + opening_linenum += 1 + if opening_linenum == len(clean_lines.elided): + # Couldn't find conditional's opening { or any code before EOF. + return + opening_line_fragment = clean_lines.elided[opening_linenum] + # Set opening_line (opening_line_fragment may not be entire opening line). + opening_line = clean_lines.elided[opening_linenum] + + # Find the position of the closing }. + opening_pos = opening_line_fragment.find('{') + if opening_linenum == end_linenum: + # We need to make opening_pos relative to the start of the entire line. + opening_pos += end_pos + (closing_line, closing_linenum, closing_pos) = CloseExpression( + clean_lines, opening_linenum, opening_pos) + if closing_pos < 0: + return + + # Now construct the body of the conditional. This consists of the portion + # of the opening line after the {, all lines until the closing line, + # and the portion of the closing line before the }. + if (clean_lines.raw_lines[opening_linenum] != + CleanseComments(clean_lines.raw_lines[opening_linenum])): + # Opening line ends with a comment, so conditional isn't empty. + return + if closing_linenum > opening_linenum: + # Opening line after the {. Ignore comments here since we checked above. + bodylist = list(opening_line[opening_pos+1:]) + # All lines until closing line, excluding closing line, with comments. + bodylist.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum]) + # Closing line before the }. Won't (and can't) have comments. + bodylist.append(clean_lines.elided[closing_linenum][:closing_pos-1]) + body = '\n'.join(bodylist) + else: + # If statement has brackets and fits on a single line. + body = opening_line[opening_pos+1:closing_pos-1] + + # Check if the body is empty + if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body): + return + # The body is empty. Now make sure there's not an else clause. + current_linenum = closing_linenum + current_line_fragment = closing_line[closing_pos:] + # Loop until EOF or find anything that's not whitespace or else clause. + while Search(r'^\s*$|^(?=\s*else)', current_line_fragment): + if Search(r'^(?=\s*else)', current_line_fragment): + # Found an else clause, so don't log an error. + return + current_linenum += 1 + if current_linenum == len(clean_lines.elided): + break + current_line_fragment = clean_lines.elided[current_linenum] + + # The body is empty and there's no else clause until EOF or other code. + error(filename, end_linenum, 'whitespace/empty_if_body', 4, + ('If statement had no body and no else clause')) + + +def FindCheckMacro(line): + """Find a replaceable CHECK-like macro. + + Args: + line: line to search on. + Returns: + (macro name, start position), or (None, -1) if no replaceable + macro is found. + """ + for macro in _CHECK_MACROS: + i = line.find(macro) + if i >= 0: + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) + if not matched: + continue + return (macro, len(matched.group(1))) + return (None, -1) + + +def CheckCheck(filename, clean_lines, linenum, error): + """Checks the use of CHECK and EXPECT macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + (check_macro, start_pos) = FindCheckMacro(lines[linenum]) + if not check_macro: + return + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression( + clean_lines, linenum, start_pos) + if end_pos < 0: + return + + # If the check macro is followed by something other than a + # semicolon, assume users will log their own custom error messages + # and don't suggest any replacements. + if not Match(r'\s*;', last_line[end_pos:]): + return + + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] + else: + expression = lines[linenum][start_pos + 1:] + for i in xrange(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one character + # at a time, we do another regular expression match to consume several + # characters at once if possible. Trivial benchmark shows that this + # is more efficient when the operands are longer than a single + # character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) + + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return + + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) + # + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % ( + _CHECK_REPLACEMENT[check_macro][operator], + check_macro, operator)) + + +def CheckAltTokens(filename, clean_lines, linenum, error): + """Check alternative keywords being used in boolean expressions. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return + + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return + + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error(filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % ( + _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + + +def GetLineWidth(line): + """Determines the width of the line in column positions. + + Args: + line: A string, which may be a Unicode string. + + Returns: + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ + if isinstance(line, unicode): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(uc): + # Issue 337 + # https://mail.python.org/pipermail/python-list/2012-August/628809.html + if (sys.version_info.major, sys.version_info.minor) <= (3, 2): + # https://github.com/python/cpython/blob/2.7/Include/unicodeobject.h#L81 + is_wide_build = sysconfig.get_config_var("Py_UNICODE_SIZE") >= 4 + # https://github.com/python/cpython/blob/2.7/Objects/unicodeobject.c#L564 + is_low_surrogate = 0xDC00 <= ord(uc) <= 0xDFFF + if not is_wide_build and is_low_surrogate: + width -= 1 + + width += 1 + return width + else: + return len(line) + + +def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, + error): + """Checks rules from the 'C++ style rules' section of cppguide.html. + + Most of these rules are hard to test (naming, comment style), but we + do what we can. In particular we check for 2-space indents, line lengths, + tab usage, spaces inside code, etc. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + prev = raw_lines[linenum - 1] if linenum > 0 else '' + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' + classinfo = nesting_state.InnermostClass() + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + # There are certain situations we allow one space, notably for + # section labels, and also lines containing multi-line raw strings. + # We also don't check for lines that look like continuation lines + # (of lines ending in double quotes, commas, equals, or angle brackets) + # because the rules for how to indent those are non-trivial. + if (not Search(r'[",=><] *$', prev) and + (initial_spaces == 1 or initial_spaces == 3) and + not Match(scope_or_label_pattern, cleansed_line) and + not (clean_lines.raw_lines[linenum] != line and + Match(r'^\s*""', line))): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + + # Check if the line is a header guard. + is_header_guard = False + if IsHeaderExtension(file_extension): + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + # + # Doxygen documentation copying can get pretty long when using an overloaded + # function declaration + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line) and + not Match(r'^\s*//\s*[^\s]*$', line) and + not Match(r'^// \$Id:.*#[0-9]+ \$$', line) and + not Match(r'^\s*/// [@\\](copydoc|copydetails|copybrief) .*$', line)): + line_width = GetLineWidth(line) + if line_width > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) + + if (cleansed_line.count(';') > 1 and + # allow simple single line lambdas + not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}\n\r]*\}', + line) and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckTrailingSemicolon(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckOperatorSpacing(filename, clean_lines, linenum, error) + CheckParenthesisSpacing(filename, clean_lines, linenum, error) + CheckCommaSpacing(filename, clean_lines, linenum, error) + CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + + +_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') +# Matches the first component of a filename delimited by -s and _s. That is: +# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' +_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') + + +def _DropCommonSuffixes(filename): + """Drops common suffixes like _test.cc or -inl.h from filename. + + For example: + >>> _DropCommonSuffixes('foo/foo-inl.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/bar/foo.cc') + 'foo/bar/foo' + >>> _DropCommonSuffixes('foo/foo_internal.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') + 'foo/foo_unusualinternal' + + Args: + filename: The input filename. + + Returns: + The filename with the common suffix removed. + """ + for suffix in itertools.chain( + ('%s.%s' % (test_suffix.lstrip('_'), ext) + for test_suffix, ext in itertools.product(_test_suffixes, GetNonHeaderExtensions())), + ('%s.%s' % (suffix, ext) + for suffix, ext in itertools.product(['inl', 'imp', 'internal'], GetHeaderExtensions()))): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] + + +def _ClassifyInclude(fileinfo, include, is_system): + """Figures out what kind of header 'include' is. + + Args: + fileinfo: The current file cpplint is running over. A FileInfo instance. + include: The path to a #included file. + is_system: True if the #include used <> rather than "". + + Returns: + One of the _XXX_HEADER constants. + + For example: + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) + _C_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) + _CPP_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) + _LIKELY_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), + ... 'bar/foo_other_ext.h', False) + _POSSIBLE_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) + _OTHER_HEADER + """ + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS + + # Headers with C++ extensions shouldn't be considered C system headers + if is_system and os.path.splitext(include)[1] in ['.hpp', '.hxx', '.h++']: + is_system = False + + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + target_dir_pub = os.path.normpath(target_dir + '/../public') + target_dir_pub = target_dir_pub.replace('\\', '/') + if target_base == include_base and ( + include_dir == target_dir or + include_dir == target_dir_pub): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER + + + +def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): + """Check rules that are applicable to #include lines. + + Strings on #include lines are NOT removed from elided line, to make + certain tasks easier. However, to prevent false positives, checks + applicable to #include lines in CheckLanguage must be put here. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + fileinfo = FileInfo(filename) + line = clean_lines.lines[linenum] + + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + # Only do this check if the included header follows google naming + # conventions. If not, assume that it's a 3rd party API that + # requires special include conventions. + # + # We also make an exception for Lua headers, which follow google + # naming convention but not the include convention. + match = Match(r'#include\s*"([^/]+\.h)"', line) + if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): + error(filename, linenum, 'build/include_subdir', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + duplicate_line = include_state.FindHeader(include) + if duplicate_line >= 0: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, duplicate_line)) + return + + for extension in GetNonHeaderExtensions(): + if (include.endswith('.' + extension) and + os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): + error(filename, linenum, 'build/include', 4, + 'Do not include .' + extension + ' files from other packages') + return + + # We DO want to include a 3rd party looking header if it matches the + # filename. Otherwise we get an erroneous error "...should include its + # header" error later. + third_src_header = False + for ext in GetHeaderExtensions(): + basefilename = filename[0:len(filename) - len(fileinfo.Extension())] + headerfile = basefilename + '.' + ext + headername = FileInfo(headerfile).RepositoryName() + if headername in include or include in headername: + third_src_header = True + break + + if third_src_header or not _THIRD_PARTY_HEADERS_PATTERN.match(include): + include_state.include_list[-1].append((include, linenum)) + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) + if not include_state.IsInAlphabeticalOrder( + clean_lines, linenum, canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) + + + +def _GetTextInside(text, start_pattern): + r"""Retrieves all the text between matching open and close parentheses. + + Given a string of lines and a regular expression string, retrieve all the text + following the expression and between opening punctuation symbols like + (, [, or {, and the matching close-punctuation symbol. This properly nested + occurrences of the punctuations, so for the text like + printf(a(), b(c())); + a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. + start_pattern must match string having an open punctuation symbol at the end. + + Args: + text: The lines to extract text. Its comments and strings must be elided. + It can be single line and can span multiple lines. + start_pattern: The regexp string indicating where to start extracting + the text. + Returns: + The extracted text. + None if either the opening string or ending punctuation could not be found. + """ + # TODO(unknown): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(itervalues(matching_punctuation)) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) + + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] + + +# Patterns for matching call-by-reference parameters. +# +# Supports nested templates up to 2 levels deep using this messy pattern: +# < (?: < (?: < [^<>]* +# > +# | [^<>] )* +# > +# | [^<>] )* +# > +_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* +_RE_PATTERN_TYPE = ( + r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' + r'(?:\w|' + r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' + r'::)+') +# A call-by-reference parameter ends with '& identifier'. +_RE_PATTERN_REF_PARAM = re.compile( + r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' + r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') +# A call-by-const-reference parameter either ends with 'const& identifier' +# or looks like 'const type& identifier' when 'type' is atomic. +_RE_PATTERN_CONST_REF_PARAM = ( + r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + + r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') +# Stream types. +_RE_PATTERN_REF_STREAM_PARAM = ( + r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + r')') + + +def CheckLanguage(filename, clean_lines, linenum, file_extension, + include_state, nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. + + Some of these rules are hard to test (function overloading, using + uint32 inappropriately), but we do the best we can. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + include_state: An _IncludeState instance in which the headers are inserted. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return + + match = _RE_PATTERN_INCLUDE.search(line) + if match: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) + if match: + include_state.ResetSection(match.group(1)) + + + # Perform other checks now that we are sure that this is not an include line + CheckCasts(filename, clean_lines, linenum, error) + CheckGlobalStatic(filename, clean_lines, linenum, error) + CheckPrintf(filename, clean_lines, linenum, error) + + if IsHeaderExtension(file_extension): + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes declare or disable copy/assign + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(unknown): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', + line, re.I).group(1) + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' + % (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' + % (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + if Search(r'\bliterals\b', line): + error(filename, linenum, 'build/namespaces_literals', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + else: + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True + skip_next = False + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token because we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error(filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size.") + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (IsHeaderExtension(file_extension) + and Search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error(filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') + + +def CheckGlobalStatic(filename, clean_lines, linenum, error): + """Check for unsafe global or static objects. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Match two lines at a time to support multiline declarations + if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): + line += clean_lines.elided[linenum + 1].strip() + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access, and + # also because globals can be destroyed when some threads are still running. + # TODO(unknown): Generalize this to also find static unique_ptr instances. + # TODO(unknown): File bugs for clang-tidy to find these. + match = Match( + r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +' + r'([a-zA-Z0-9_:]+)\b(.*)', + line) + + # Remove false positives: + # - String pointers (as opposed to values). + # string *pointer + # const string *pointer + # string const *pointer + # string *const pointer + # + # - Functions and template specializations. + # string Function(... + # string Class::Method(... + # + # - Operators. These are matched separately because operator names + # cross non-word boundaries, and trying to match both operators + # and functions at the same time would decrease accuracy of + # matching identifiers. + # string Class::operator*() + if (match and + not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) and + not Search(r'\boperator\W', line) and + not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))): + if Search(r'\bconst\b', line): + error(filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string ' + 'instead: "%schar%s %s[]".' % + (match.group(1), match.group(2) or '', match.group(3))) + else: + error(filename, linenum, 'runtime/string', 4, + 'Static/global string variables are not permitted.') + + if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) or + Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') + + +def CheckPrintf(filename, clean_lines, linenum, error): + """Check for printf related issues. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\s*\(', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\s*\(', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) + + +def IsDerivedFunction(clean_lines, linenum): + """Check if current line contains an inherited function. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line contains a function with "override" + virt-specifier. + """ + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis + line, _, closing_paren = CloseExpression( + clean_lines, i, len(match.group(1))) + return (closing_paren >= 0 and + Search(r'\boverride\b', line[closing_paren:])) + return False + + +def IsOutOfLineMethodDefinition(clean_lines, linenum): + """Check if current line contains an out-of-line method definition. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line contains an out-of-line method definition. + """ + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): + return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None + return False + + +def IsInitializerList(clean_lines, linenum): + """Check if current line is inside constructor initializer list. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line appears to be inside constructor initializer + list, False otherwise. + """ + for i in xrange(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) + if remove_function_body: + line = remove_function_body.group(1) + + if Search(r'\s:\s*\w+[({]', line): + # A lone colon tend to indicate the start of a constructor + # initializer list. It could also be a ternary operator, which + # also tend to appear in constructor initializer lists as + # opposed to parameter lists. + return True + if Search(r'\}\s*,\s*$', line): + # A closing brace followed by a comma is probably the end of a + # brace-initialized member in constructor initializer list. + return True + if Search(r'[{};]\s*$', line): + # Found one of the following: + # - A closing brace or semicolon, probably the end of the previous + # function. + # - An opening brace, probably the start of current class or namespace. + # + # Current line is probably not inside an initializer list since + # we saw one of those things without seeing the starting colon. + return False + + # Got to the beginning of the file without seeing the start of + # constructor initializer list. + return False + + +def CheckForNonConstReference(filename, clean_lines, linenum, + nesting_state, error): + """Check for non-const references. + + Separate from CheckLanguage since it scans backwards from current + line, instead of scanning forward. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: + return + + # If a function is inherited, current function doesn't have much of + # a choice, so any non-const references should not be blamed on + # derived function. + if IsDerivedFunction(clean_lines, linenum): + return + + # Don't warn on out-of-line method definitions, as we would warn on the + # in-line declaration, if it isn't marked with 'override'. + if IsOutOfLineMethodDefinition(clean_lines, linenum): + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, startpos) = ReverseCloseExpression( + clean_lines, linenum, endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in xrange(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + if (nesting_state.previous_stack_top and + not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or + isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): + # Not at toplevel, not within a class, and not within a namespace + return + + # Avoid initializer lists. We only need to scan back from the + # current line for something that starts with ':'. + # + # We don't need to check the current line, since the '&' would + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break + if Match(r'^\s*:\s+\S', previous_line): + return + + # Avoid preprocessors + if Search(r'\\\s*$', line): + return + + # Avoid constructor initializer lists + if IsInitializerList(clean_lines, linenum): + return + + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(whitelisted_functions, line): + return + elif not Search(r'\S+\([^)]*$', line): + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in xrange(2): + if (linenum > i and + Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + return + + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and + not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + + ReplaceAll(' *<', '<', parameter)) + + +def CheckCasts(filename, clean_lines, linenum, error): + """Various cast related checks. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search( + r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b' + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + expecting_function = ExpectingFunctionArgs(clean_lines, linenum) + if match and not expecting_function: + matched_type = match.group(2) + + # matched_new_or_template is used to silence two false positives: + # - New operators + # - Template arguments with function types + # + # For template arguments, we match on types immediately following + # an opening bracket without any spaces. This is a fast way to + # silence the common case where the function type is the first + # template argument. False negative with less-than comparison is + # avoided because those operators are usually followed by a space. + # + # function // bracket + no space = false positive + # value < double(42) // bracket + space = true positive + matched_new_or_template = match.group(1) + + # Avoid arrays by looking for brackets that come after the closing + # parenthesis. + if Match(r'\([^()]+\)\s*\[', match.group(3)): + return + + # Other things to ignore: + # - Function pointers + # - Casts to pointer types + # - Placement new + # - Alias declarations + matched_funcptr = match.group(3) + if (matched_new_or_template is None and + not (matched_funcptr and + (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', + matched_funcptr) or + matched_funcptr.startswith('(*)'))) and + not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and + not Search(r'new\(\S+\)\s*' + matched_type, line)): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % + matched_type) + + if not expecting_function: + CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) + + # This doesn't catch all cases. Consider (const char * const)"hello". + # + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', + r'\((char\s?\*+\s?)\)\s*"', error): + pass + else: + # Check pointer casts for other than string constants + CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', + r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + # + # Some non-identifier character is required before the '&' for the + # expression to be recognized as a cast. These are casts: + # expression = &static_cast(temporary()); + # function(&(int*)(temporary())); + # + # This is not a cast: + # reference_type&(int* function_param); + match = Search( + r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' + r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) + if match: + # Try a better error message when the & is bound to something + # dereferenced by the casted pointer, as opposed to the casted + # pointer itself. + parenthesis_error = False + match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) + if match: + _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) + if x1 >= 0 and clean_lines.elided[y1][x1] == '(': + _, y2, x2 = CloseExpression(clean_lines, y1, x1) + if x2 >= 0: + extended_line = clean_lines.elided[y2][x2:] + if y2 < clean_lines.NumLines() - 1: + extended_line += clean_lines.elided[y2 + 1] + if Match(r'\s*(?:->|\[)', extended_line): + parenthesis_error = True + + if parenthesis_error: + error(filename, linenum, 'readability/casting', 4, + ('Are you taking an address of something dereferenced ' + 'from a cast? Wrapping the dereferenced expression in ' + 'parentheses will make the binding more obvious')) + else: + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) + + +def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): + """Checks for a C-style cast by looking for the pattern. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + cast_type: The string for the C++ cast to recommend. This is either + reinterpret_cast, static_cast, or const_cast, depending. + pattern: The regular expression used to find C-style casts. + error: The function to call with any errors found. + + Returns: + True if an error was emitted. + False otherwise. + """ + line = clean_lines.elided[linenum] + match = Search(pattern, line) + if not match: + return False + + # Exclude lines with keywords that tend to look like casts + context = line[0:match.start(1) - 1] + if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): + return False + + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False + + # operator++(int) and operator--(int) + if context.endswith(' operator++') or context.endswith(' operator--'): + return False + + # A single unnamed argument for a function tends to look like old style cast. + # If we see those, don't issue warnings for deprecated casts. + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', + remainder): + return False + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + return True + + +def ExpectingFunctionArgs(clean_lines, linenum): + """Checks whether where function type arguments are expected. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + + Returns: + True if the line at 'linenum' is inside something that expects arguments + of function types. + """ + line = clean_lines.elided[linenum] + return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or + (linenum >= 2 and + (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) or + Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]) or + Search(r'\bstd::m?function\s*\<\s*$', + clean_lines.elided[linenum - 1])))) + + +_HEADERS_CONTAINING_TEMPLATES = ( + ('', ('deque',)), + ('', ('unary_function', 'binary_function', + 'plus', 'minus', 'multiplies', 'divides', 'modulus', + 'negate', + 'equal_to', 'not_equal_to', 'greater', 'less', + 'greater_equal', 'less_equal', + 'logical_and', 'logical_or', 'logical_not', + 'unary_negate', 'not1', 'binary_negate', 'not2', + 'bind1st', 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', 'const_mem_fun1_t', + 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('', ('numeric_limits',)), + ('', ('list',)), + ('', ('multimap',)), + ('', ('allocator', 'make_shared', 'make_unique', 'shared_ptr', + 'unique_ptr', 'weak_ptr')), + ('', ('queue', 'priority_queue',)), + ('', ('multiset',)), + ('', ('stack',)), + ('', ('char_traits', 'basic_string',)), + ('', ('tuple',)), + ('', ('unordered_map', 'unordered_multimap')), + ('', ('unordered_set', 'unordered_multiset')), + ('', ('pair',)), + ('', ('vector',)), + + # gcc extensions. + # Note: std::hash is their hash, ::hash is our hash + ('', ('hash_map', 'hash_multimap',)), + ('', ('hash_set', 'hash_multiset',)), + ('', ('slist',)), + ) + +_HEADERS_MAYBE_TEMPLATES = ( + ('', ('copy', 'max', 'min', 'min_element', 'sort', + 'transform', + )), + ('', ('forward', 'make_pair', 'move', 'swap')), + ) + +_RE_PATTERN_STRING = re.compile(r'\bstring\b') + +_re_pattern_headers_maybe_templates = [] +for _header, _templates in _HEADERS_MAYBE_TEMPLATES: + for _template in _templates: + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # 'type::max()'. + _re_pattern_headers_maybe_templates.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), + _template, + _header)) +# Match set, but not foo->set, foo.set +_re_pattern_headers_maybe_templates.append( + (re.compile(r'[^>.]\bset\s*\<'), + 'set<>', + '')) +# Match 'map var' and 'std::map(...)', but not 'map(...)'' +_re_pattern_headers_maybe_templates.append( + (re.compile(r'(std\b::\bmap\s*\<)|(^(std\b::\b)map\b\(\s*\<)'), + 'map<>', + '')) + +# Other scripts may reach in and modify this pattern. +_re_pattern_templates = [] +for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), + _template + '<>', + _header)) + + +def FilesBelongToSameModule(filename_cc, filename_h): + """Check if these two filenames belong to the same module. + + The concept of a 'module' here is a as follows: + foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the + same 'module' if they are in the same directory. + some/path/public/xyzzy and some/path/internal/xyzzy are also considered + to belong to the same module here. + + If the filename_cc contains a longer path than the filename_h, for example, + '/absolute/path/to/base/sysinfo.cc', and this file would include + 'base/sysinfo.h', this function also produces the prefix needed to open the + header. This is used by the caller of this function to more robustly open the + header file. We don't have access to the real include paths in this context, + so we need this guesswork here. + + Known bugs: tools/base/bar.cc and base/bar.h belong to the same module + according to this implementation. Because of this, this function gives + some false positives. This should be sufficiently rare in practice. + + Args: + filename_cc: is the path for the source (e.g. .cc) file + filename_h: is the path for the header path + + Returns: + Tuple with a bool and a string: + bool: True if filename_cc and filename_h belong to the same module. + string: the additional prefix needed to open the header file. + """ + fileinfo_cc = FileInfo(filename_cc) + if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions(): + return (False, '') + + fileinfo_h = FileInfo(filename_h) + if not IsHeaderExtension(fileinfo_h.Extension().lstrip('.')): + return (False, '') + + filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))] + matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName()) + if matched_test_suffix: + filename_cc = filename_cc[:-len(matched_test_suffix.group(1))] + + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + filename_h = filename_h[:-(len(fileinfo_h.Extension()))] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path + + +def UpdateIncludeState(filename, include_dict, io=codecs): + """Fill up the include_dict with new includes found from the file. + + Args: + filename: the name of the header to read. + include_dict: a dictionary in which the headers are inserted. + io: The io factory to use to read the file. Provided for testability. + + Returns: + True if a header was successfully added. False otherwise. + """ + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + include_dict.setdefault(include, linenum) + return True + + +def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, + io=codecs): + """Reports for missing stl includes. + + This function will output warnings to make sure you are including the headers + necessary for the stl containers and functions that you use. We only give one + reason to include a header. For example, if you use both equal_to<> and + less<> in a .h file, only one (the latter in the file) of these will be + reported as a reason to include the . + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + include_state: An _IncludeState instance. + error: The function to call with any errors found. + io: The IO factory to use to read the header file. Provided for unittest + injection. + """ + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + + for linenum in xrange(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue + + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_headers_maybe_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + matched = pattern.search(line) + if matched: + # Don't warn about IWYU in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's flatten the include_state include_list and copy it into a dictionary. + include_dict = dict([item for sublist in include_state.include_list + for item in sublist]) + + # Did we find the header for this file (if any) and successfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = list(include_dict.keys()) + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_dict, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if not header_found: + for extension in GetNonHeaderExtensions(): + if filename.endswith('.' + extension): + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in sorted(required, key=required.__getitem__): + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_dict: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, + 'Add #include ' + required_header_unstripped + ' for ' + template) + + +_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') + + +def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): + """Check that make_pair's template arguments are deduced. + + G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are + specified explicitly, and such use isn't intended in any case. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error(filename, linenum, 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly') + + +def CheckRedundantVirtual(filename, clean_lines, linenum, error): + """Check if line contains a redundant "virtual" function-specifier. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Look for "virtual" on current line. + line = clean_lines.elided[linenum] + virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) + if not virtual: return + + # Ignore "virtual" keywords that are near access-specifiers. These + # are only used in class base-specifier and do not apply to member + # functions. + if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or + Match(r'^\s+(public|protected|private)\b', virtual.group(3))): + return + + # Ignore the "virtual" keyword from virtual base classes. Usually + # there is a column on the same line in these cases (virtual base + # classes are rare in google3 because multiple inheritance is rare). + if Match(r'^.*[^:]:[^:].*$', line): return + + # Look for the next opening parenthesis. This is the start of the + # parameter list (possibly on the next line shortly after virtual). + # TODO(unknown): doesn't work if there are virtual functions with + # decltype() or other things that use parentheses, but csearch suggests + # that this is rare. + end_col = -1 + end_line = -1 + start_col = len(virtual.group(2)) + for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: + # Match parentheses to find the end of the parameter list + (_, end_line, end_col) = CloseExpression( + clean_lines, start_line, start_col + len(parameter_list.group(1))) + break + start_col = 0 + + if end_col < 0: + return # Couldn't find end of parameter list, give up + + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). + for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: + error(filename, linenum, 'readability/inheritance', 4, + ('"virtual" is redundant since function is ' + 'already declared as "%s"' % match.group(1))) + + # Set end_col to check whole lines after we are done with the + # first line. + end_col = 0 + if Search(r'[^\w]\s*$', line): + break + + +def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): + """Check if line contains a redundant "override" or "final" virt-specifier. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Look for closing parenthesis nearby. We need one to confirm where + # the declarator ends and where the virt-specifier starts to avoid + # false positives. + line = clean_lines.elided[linenum] + declarator_end = line.rfind(')') + if declarator_end >= 0: + fragment = line[declarator_end:] + else: + if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: + fragment = line + else: + return + + # Check that at most one of "override" or "final" is present, not both + if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): + error(filename, linenum, 'readability/inheritance', 4, + ('"override" is redundant since function is ' + 'already declared as "final"')) + + + + +# Returns true if we are at a new block, and it is directly +# inside of a namespace. +def IsBlockInNameSpace(nesting_state, is_forward_declaration): + """Checks that the new block is directly in a namespace. + + Args: + nesting_state: The _NestingState object that contains info about our state. + is_forward_declaration: If the class is a forward declared class. + Returns: + Whether or not the new block is directly in a namespace. + """ + if is_forward_declaration: + return len(nesting_state.stack) >= 1 and ( + isinstance(nesting_state.stack[-1], _NamespaceInfo)) + + + return (len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.stack[-2], _NamespaceInfo)) + + +def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + raw_lines_no_comments, linenum): + """This method determines if we should apply our namespace indentation check. + + Args: + nesting_state: The current nesting state. + is_namespace_indent_item: If we just put a new class on the stack, True. + If the top of the stack is not a class, or we did not recently + add the class, False. + raw_lines_no_comments: The lines without the comments. + linenum: The current line number we are processing. + + Returns: + True if we should apply our namespace indentation check. Currently, it + only works for classes and namespaces inside of a namespace. + """ + + is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, + linenum) + + if not (is_namespace_indent_item or is_forward_declaration): + return False + + # If we are in a macro, we do not want to check the namespace indentation. + if IsMacroDefinition(raw_lines_no_comments, linenum): + return False + + return IsBlockInNameSpace(nesting_state, is_forward_declaration) + + +# Call this method if the line is directly inside of a namespace. +# If the line above is blank (excluding comments) or the start of +# an inner namespace, it cannot be indented. +def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, + error): + line = raw_lines_no_comments[linenum] + if Match(r'^\s+', line): + error(filename, linenum, 'runtime/indentation_namespace', 4, + 'Do not indent within a namespace') + + +def ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions=None): + """Processes a single line in the file. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + clean_lines: An array of strings, each representing a line of the file, + with comments stripped. + line: Number of line being processed. + include_state: An _IncludeState instance in which the headers are inserted. + function_state: A _FunctionState instance which counts function lines, etc. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error) + if nesting_state.InAsmBlock(): return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, + nesting_state, error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + CheckRedundantVirtual(filename, clean_lines, line, error) + CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) + if extra_check_functions: + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + +def FlagCxx11Features(filename, clean_lines, linenum, error): + """Flag those c++11 features that we only allow in certain places. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + + # Flag unapproved C++ TR1 headers. + if include and include.group(1).startswith('tr1/'): + error(filename, linenum, 'build/c++tr1', 5, + ('C++ TR1 headers such as <%s> are unapproved.') % include.group(1)) + + # Flag unapproved C++11 headers. + if include and include.group(1) in ('cfenv', + 'condition_variable', + 'fenv.h', + 'future', + 'mutex', + 'thread', + 'chrono', + 'ratio', + 'regex', + 'system_error', + ): + error(filename, linenum, 'build/c++11', 5, + ('<%s> is an unapproved C++11 header.') % include.group(1)) + + # The only place where we need to worry about C++11 keywords and library + # features in preprocessor directives is in macro definitions. + if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return + + # These are classes and free functions. The classes are always + # mentioned as std::*, but we only catch the free functions if + # they're not found by ADL. They're alphabetical by header. + for top_name in ( + # type_traits + 'alignment_of', + 'aligned_union', + ): + if Search(r'\bstd::%s\b' % top_name, line): + error(filename, linenum, 'build/c++11', 5, + ('std::%s is an unapproved C++11 class or function. Send c-style ' + 'an example of where it would make your code more readable, and ' + 'they may let you use it.') % top_name) + + +def FlagCxx14Features(filename, clean_lines, linenum, error): + """Flag those C++14 features that we restrict. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + + # Flag unapproved C++14 headers. + if include and include.group(1) in ('scoped_allocator', 'shared_mutex'): + error(filename, linenum, 'build/c++14', 5, + ('<%s> is an unapproved C++14 header.') % include.group(1)) + + +def ProcessFileData(filename, file_extension, lines, error, + extra_check_functions=None): + """Performs lint checks and reports any errors to the given error function. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is terminated with a newline. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) + + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = NestingState() + + ResetNolintSuppressions() + + CheckForCopyright(filename, lines, error) + ProcessGlobalSuppressions(lines) + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) + + if IsHeaderExtension(file_extension): + CheckForHeaderGuard(filename, clean_lines, error) + + for line in xrange(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions) + FlagCxx11Features(filename, clean_lines, line, error) + nesting_state.CheckCompletedBlocks(filename, error) + + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) + + # Check that the .cc file has included its header if it exists. + if _IsSourceExtension(file_extension): + CheckHeaderFileIncluded(filename, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) + +def ProcessConfigOverrides(filename): + """ Loads the configuration files and processes the config overrides. + + Args: + filename: The name of the file being processed by the linter. + + Returns: + False if the current |filename| should not be processed further. + """ + + abs_filename = os.path.abspath(filename) + cfg_filters = [] + keep_looking = True + while keep_looking: + abs_path, base_name = os.path.split(abs_filename) + if not base_name: + break # Reached the root directory. + + cfg_file = os.path.join(abs_path, "CPPLINT.cfg") + abs_filename = abs_path + if not os.path.isfile(cfg_file): + continue + + try: + with open(cfg_file) as file_handle: + for line in file_handle: + line, _, _ = line.partition('#') # Remove comments. + if not line.strip(): + continue + + name, _, val = line.partition('=') + name = name.strip() + val = val.strip() + if name == 'set noparent': + keep_looking = False + elif name == 'filter': + cfg_filters.append(val) + elif name == 'exclude_files': + # When matching exclude_files pattern, use the base_name of + # the current file name or the directory name we are processing. + # For example, if we are checking for lint errors in /foo/bar/baz.cc + # and we found the .cfg file at /foo/CPPLINT.cfg, then the config + # file's "exclude_files" filter is meant to be checked against "bar" + # and not "baz" nor "bar/baz.cc". + if base_name: + pattern = re.compile(val) + if pattern.match(base_name): + if _cpplint_state.quiet: + # Suppress "Ignoring file" warning when using --quiet. + return False + _cpplint_state.PrintInfo('Ignoring "%s": file excluded by "%s". ' + 'File path component "%s" matches ' + 'pattern "%s"\n' % + (filename, cfg_file, base_name, val)) + return False + elif name == 'linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + _cpplint_state.PrintError('Line length must be numeric.') + elif name == 'extensions': + ProcessExtensionsOption(val) + elif name == 'root': + global _root + # root directories are specified relative to CPPLINT.cfg dir. + _root = os.path.join(os.path.dirname(cfg_file), val) + elif name == 'headers': + ProcessHppHeadersOption(val) + else: + _cpplint_state.PrintError( + 'Invalid configuration option (%s) in file %s\n' % + (name, cfg_file)) + + except IOError: + _cpplint_state.PrintError( + "Skipping config file '%s': Can't open for reading\n" % cfg_file) + keep_looking = False + + # Apply all the accumulated filters in reverse order (top-level directory + # config options having the least priority). + for cfg_filter in reversed(cfg_filters): + _AddFilters(cfg_filter) + + return True + + +def ProcessFile(filename, vlevel, extra_check_functions=None): + """Does google-lint on a single file. + + Args: + filename: The name of the file to parse. + + vlevel: The level of errors to report. Every error of confidence + >= verbose_level will be reported. 0 is a good default. + + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + + _SetVerboseLevel(vlevel) + _BackupFilters() + old_errors = _cpplint_state.error_count + + if not ProcessConfigOverrides(filename): + _RestoreFilters() + return + + lf_lines = [] + crlf_lines = [] + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') + + # Remove trailing '\r'. + # The -1 accounts for the extra trailing blank line we get from split() + for linenum in range(len(lines) - 1): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + crlf_lines.append(linenum + 1) + else: + lf_lines.append(linenum + 1) + + except IOError: + _cpplint_state.PrintError( + "Skipping input '%s': Can't open for reading\n" % filename) + _RestoreFilters() + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in GetAllExtensions(): + _cpplint_state.PrintError('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(GetAllExtensions()))) + else: + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) + + # If end-of-line sequences are a mix of LF and CR-LF, issue + # warnings on the lines with CR. + # + # Don't issue any warnings if all lines are uniformly LF or CR-LF, + # since critique can handle these just fine, and the style guide + # doesn't dictate a particular end of line sequence. + # + # We can't depend on os.linesep to determine what the desired + # end-of-line sequence should be, since that will return the + # server-side end-of-line sequence. + if lf_lines and crlf_lines: + # Warn on every line with CR. An alternative approach might be to + # check whether the file is mostly CRLF or just LF, and warn on the + # minority, we bias toward LF here since most tools prefer LF. + for linenum in crlf_lines: + Error(filename, linenum, 'whitespace/newline', 1, + 'Unexpected \\r (^M) found; better to use only \\n') + + # Suppress printing anything if --quiet was passed unless the error + # count has increased after processing this file. + if not _cpplint_state.quiet or old_errors != _cpplint_state.error_count: + _cpplint_state.PrintInfo('Done processing %s\n' % filename) + _RestoreFilters() + + +def PrintUsage(message): + """Prints a brief usage string and exits, optionally with an error message. + + Args: + message: The optional error message. + """ + sys.stderr.write(_USAGE % (list(GetAllExtensions()), + ','.join(list(GetAllExtensions())), + GetHeaderExtensions(), + ','.join(GetHeaderExtensions()))) + + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(0) + +def PrintVersion(): + sys.stdout.write('Cpplint fork (https://github.com/cpplint/cpplint)\n') + sys.stdout.write('cpplint ' + __VERSION__ + '\n') + sys.stdout.write('Python ' + sys.version + '\n') + sys.exit(0) + +def PrintCategories(): + """Prints a list of all the error-categories used by error messages. + + These are the categories used to filter messages via --filter. + """ + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) + + +def ParseArguments(args): + """Parses the command line arguments. + + This may set the output format and verbosity level as side-effects. + + Args: + args: The command line arguments: + + Returns: + The list of filenames to lint. + """ + try: + (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', + 'v=', + 'version', + 'counting=', + 'filter=', + 'root=', + 'repository=', + 'linelength=', + 'extensions=', + 'exclude=', + 'recursive', + 'headers=', + 'quiet']) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + quiet = _Quiet() + counting_style = '' + recursive = False + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + if opt == '--version': + PrintVersion() + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse', 'junit'): + PrintUsage('The only allowed output formats are emacs, vs7, eclipse ' + 'and junit.') + output_format = val + elif opt == '--quiet': + quiet = True + elif opt == '--verbose' or opt == '--v': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage('Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--repository': + global _repository + _repository = val + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--exclude': + global _excludes + if not _excludes: + _excludes = set() + _excludes.update(glob.glob(val)) + elif opt == '--extensions': + ProcessExtensionsOption(val) + elif opt == '--headers': + ProcessHppHeadersOption(val) + elif opt == '--recursive': + recursive = True + + if not filenames: + PrintUsage('No files were specified.') + + if recursive: + filenames = _ExpandDirectories(filenames) + + if _excludes: + filenames = _FilterExcludedFiles(filenames) + + _SetOutputFormat(output_format) + _SetQuiet(quiet) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames + +def _ExpandDirectories(filenames): + """Searches a list of filenames and replaces directories in the list with + all files descending from those directories. Files with extensions not in + the valid extensions list are excluded. + + Args: + filenames: A list of files or directories + + Returns: + A list of all files that are members of filenames or descended from a + directory in filenames + """ + expanded = set() + for filename in filenames: + if not os.path.isdir(filename): + expanded.add(filename) + continue + + for root, _, files in os.walk(filename): + for loopfile in files: + fullname = os.path.join(root, loopfile) + if fullname.startswith('.' + os.path.sep): + fullname = fullname[len('.' + os.path.sep):] + expanded.add(fullname) + + filtered = [] + for filename in expanded: + if os.path.splitext(filename)[1][1:] in GetAllExtensions(): + filtered.append(filename) + + return filtered + +def _FilterExcludedFiles(filenames): + """Filters out files listed in the --exclude command line switch. File paths + in the switch are evaluated relative to the current working directory + """ + exclude_paths = [os.path.abspath(f) for f in _excludes] + return [f for f in filenames if os.path.abspath(f) not in exclude_paths] + +def main(): + filenames = ParseArguments(sys.argv[1:]) + backup_err = sys.stderr + try: + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReader(sys.stderr, 'replace') + + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + # If --quiet is passed, suppress printing error count unless there are errors. + if not _cpplint_state.quiet or _cpplint_state.error_count > 0: + _cpplint_state.PrintErrorCounts() + + if _cpplint_state.output_format == 'junit': + sys.stderr.write(_cpplint_state.FormatJUnitXML()) + + finally: + sys.stderr = backup_err + + sys.exit(_cpplint_state.error_count > 0) + + +if __name__ == '__main__': + main() diff --git a/.linters/cpp/hooks/pre-commit.sh b/.linters/cpp/hooks/pre-commit.sh new file mode 100644 index 0000000..fe289cf --- /dev/null +++ b/.linters/cpp/hooks/pre-commit.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Copyright (c) 2018 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +CPPLINT=`dirname $0`/../../.linters/cpp/cpplint.py +CHECKKEYWORD=`dirname $0`/../../.linters/cpp/checkKeyword.py + +echo "Performing checkout keyword..." +python3 $CHECKKEYWORD + +if [ $? -ne 0 ]; then + echo "Checkout keyword failed" + exit 1 +fi + +if [ $# -eq 0 ];then + # Since cpplint.py could only apply on our working tree, + # so we forbid committing with unstaged changes present. + # Otherwise, lints can be bypassed via changing without commit. + if ! git diff-files --exit-code --quiet + then + echo "You have unstaged changes, please stage or stash them first." + exit 1 + fi + CHECK_FILES=$(git diff --name-only --diff-filter=ACMRTUXB HEAD | egrep '.*\.cpp$|.*\.h$' | grep -v 'com_vesoft_client_NativeClient.h' | grep -v 'com_vesoft_nebula_NebulaCodec.h') +else + CHECK_FILES=$(find $@ -not \( -path src/CMakeFiles -prune \) \ + -not \( -path src/interface/gen-cpp2 -prune \) \ + -name "*.[h]" -o -name "*.cpp" \ + | grep -v 'GraphScanner.*' | grep -v 'GraphParser.*' \ + | grep -v 'com_vesoft_client_NativeClient.h' \ + | grep -v 'com_vesoft_nebula_NebulaCodec.h') +fi + +# No changes on interested files +if [[ -z $CHECK_FILES ]]; then + echo "There's no source files to perform C++ linters..." + exit 0 +fi + +echo "Performing C++ linters..." + +CPPLINT_EXTENS=cpp,h +CPPLINT_FILTER=-whitespace/indent,-build/include_what_you_use,-readability/todo,-build/include,-build/header_guard,-runtime/references,-build/c++11 + +python3 $CPPLINT --quiet --extensions=$CPPLINT_EXTENS \ + --filter=$CPPLINT_FILTER --linelength=100 $CHECK_FILES 2>&1 + +result=$? +if [ ! $result -eq 0 ]; then + echo "cpplint code style check failed, please fix and recommit." + exit 1 +fi + +echo "Performing C++ code format check..." + +CLANG_FALLBACK=/opt/vesoft/toolset/clang/10.0.0/ +if [ -z "$CLANG_HOME" ] && [ -d "$CLANG_FALLBACK" ]; then + CLANG_HOME=$CLANG_FALLBACK +fi + +CLANG_FORMAT=$(command -v clang-format-10) +if [ -z "$CLANG_FORMAT" ]; then + CLANG_FORMAT=$CLANG_HOME/bin/clang-format +fi + +CLANG_FORMAT_DIFF=$(command -v clang-format-diff-10) +if [ -z "$CLANG_FORMAT_DIFF" ]; then + CLANG_FORMAT_DIFF=$CLANG_HOME/share/clang/clang-format-diff.py +fi + +if [ -z "$CLANG_FORMAT" ] || [ -z "$CLANG_FORMAT_DIFF" ]; then + if [ ! -d "$CLANG_HOME" ]; then + echo "The $CLANG_HOME directory was not found." + fi + if [ -z "$CLANG_FORMAT" ]; then + echo "Could not find clang-format" + fi + if [ -z "$CLANG_FORMAT_DIFF" ]; then + echo "Could not find clang-format-diff" + fi + echo "source changes cannot be automatically formatted." + exit 0 +fi + +git diff -U0 --no-color --staged | "$CLANG_FORMAT_DIFF" -i -p1 -binary "$CLANG_FORMAT" +git add $CHECK_FILES diff --git a/.linters/java/nebula_java_style_checks.xml b/.linters/java/nebula_java_style_checks.xml new file mode 100644 index 0000000..eb606c0 --- /dev/null +++ b/.linters/java/nebula_java_style_checks.xml @@ -0,0 +1,236 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6fdb7b8 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,121 @@ +# Copyright (c) 2018 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +# The build can be controlled by defining following variables on the +# command line +# +# CMAKE_C_COMPILER -- Specify the compiler for C language +# CMAKE_CXX_COMPILER -- Specify the compiler for C++ language +# +# NEBULA_THIRDPARTY_ROOT -- Specify the root directory for third-party +# NEBULA_OTHER_ROOT -- Specify the root directory for user build +# -- Split with ":", exp: DIR:DIR +# +# ENABLE_JEMALLOC -- Link jemalloc into all executables +# ENABLE_NATIVE -- Build native client +# ENABLE_TESTING -- Build unit test +# ENABLE_PACK_ONE -- Package to one or multi packages + +# CMake version check +cmake_minimum_required(VERSION 3.9.0) + +# Set the project name +project("Nebula Graph" C CXX) + +option(ENABLE_PACK_ONE "Whether to package into one" ON) +option(ENABLE_VERBOSE_BISON "Enable Bison to report state" OFF) +option(ENABLE_CONSOLE_COMPILATION "Enable nebula-console compilation" OFF) +option(ENABLE_PACKAGE_TAR "Enable package artifacts to tar." OFF) +option(ENABLE_CREATE_GIT_HOOKS "Enable create git hooks." ON) +option(ENABLE_INCLUDE_WHAT_YOU_USE "Enable include-what-you-use find nouse include files" OFF) + +add_definitions(-DNEBULA_HOME=${CMAKE_SOURCE_DIR}) + +if(ENABLE_STANDALONE_VERSION) + add_definitions(-DBUILD_STANDALONE) +endif() + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/nebula) + +include(PlatformCheck) +include(NebulaCMakeMacros) +include(GeneralCMakeOptions) +include(GeneralCMakeConfig) +include(GeneralCompilerConfig) +include(LinkerConfig) +include(CompilerLauncher) +include(ThirdPartyConfig) +include(SanitizerConfig) +include(GitHooksConfig) +include(GitInfoConfig) +include(NebulaCustomTargets) +include(IncludeWhatYouUse) +include(TimeTrace) + +add_custom_target( + clang-format + COMMAND "find" "src/" "-type" "f" "\\(" "-iname" "\\*.h" "-o" "-iname" "\\*.cpp" "\\)" "|" "xargs" "clang-format" "-i" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +if (ENABLE_NATIVE) + message(STATUS "ENABLE_NATIVE is ${ENABLE_NATIVE}") + add_compile_options(-fPIC) +endif() + +if(ENABLE_MEMORY_TRACKER) + if(ENABLE_JEMALLOC) + if(NOT ENABLE_ASAN) + add_definitions(-DENABLE_MEMORY_TRACKER) + message(STATUS "MemoryTracker is ENABLED") + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_ASAN=off") + endif() + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_JEMALLOC=on") + endif() +else() + message(WARNING "MemoryTracker is DISABLED") +endif() + +include_directories(AFTER ${CMAKE_SOURCE_DIR}/src) +include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src) + +# For simplicity, we make all ordinary libraries depend on the compile-time generated files, +# including the precompiled header, a.k.a Base.h.gch, and thrift headers. +macro(nebula_add_library name type) + add_library(${name} ${type} ${ARGN}) + if (PCHSupport_FOUND) + add_dependencies( + ${name} + base_obj_gch + ) + endif() + add_dependencies( + ${name} + common_thrift_generator + graph_thrift_generator + storage_thrift_generator + meta_thrift_generator + raftex_thrift_generator + # hbase_thrift_generator + parser_target + wkt_parser_target + datetime_parser_target + ) +endmacro() + +nebula_add_subdirectory(src) +nebula_add_subdirectory(conf) +nebula_add_subdirectory(resources) +nebula_add_subdirectory(scripts) + +include(CPackage) +package( + ${ENABLE_PACK_ONE} + "nebula-graph" + "https://github.com/vesoft-inc/nebula/releases" + ${CMAKE_SOURCE_DIR}/package +) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e673e6d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +See [How to contribute](https://github.com/vesoft-inc/nebula-community/blob/master/Contributors/how-to-contribute.md) for details about how to contribute to **NebulaGraph**. diff --git a/Coding_Style_Guide.md b/Coding_Style_Guide.md new file mode 100644 index 0000000..bb2b48b --- /dev/null +++ b/Coding_Style_Guide.md @@ -0,0 +1,10 @@ +# Contributor Covenant Coding Style Guide + +Currently we use clang-format to format the code. We recommend you to configure the clang-format according to the IDE / editor you use. The following link is how to configure vim / emacs / vscode to use clang-format. + +- Vim + https://github.com/rhysd/vim-clang-format +- Emacs + https://github.com/llvm-mirror/clang/blob/master/tools/clang-format/clang-format.el +- VS Code + https://code.visualstudio.com/docs/cpp/cpp-ide#_code-formatting diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README-CN.md b/README-CN.md new file mode 100644 index 0000000..7a2fcd7 --- /dev/null +++ b/README-CN.md @@ -0,0 +1,122 @@ +

+ +
中文 | English +
世界上唯一能够容纳千亿个顶点和万亿条边，并提供毫秒级查询延时的图数据库解决方案
+

+ +

+ + WeiXin + + + Sina Weibo + + + GitHub stars + + + GitHub forks + + +

+ +# NebulaGraph 是什么？ + + +**NebulaGraph** 是一款开源的图数据库，擅长处理千亿个顶点和万亿条边的超大规模数据集。 + +NebulaGraph 社区已成长为一个荟聚了众多用户、融合了各类图技术场景实践知识的活跃开源社区。你可以在其中与大家共同交流 NebulaGraph [周边生态项目](https://docs.nebula-graph.com.cn/master/20.appendix/6.eco-tool-version/)的应用心得，或者社交媒体、实时推荐、网络安全、金融风控、知识图谱、人工智能等[大规模生产场景](https://nebula-graph.com.cn/cases)的实践经验。 + +**NebulaGraph** 特点如下： + +* 全对称分布式架构 +* 存储与计算分离 +* 水平可扩展性 +* RAFT 协议下的数据强一致 +* 支持 openCypher +* 用户鉴权 +* 支持多种类型的图计算算法 + +**NebulaGraph** 内核架构图如下： + +![image](https://docs-cdn.nebula-graph.com.cn/figures/nebula-graph-architecture_3.png) + +点击 [NebulaGraph 官网](https://www.nebula-graph.com.cn/) 了解更多信息。 + + + +## 快速使用 + +您可以在[云上](https://docs.nebula-graph.com.cn/3.3.0/2.quick-start/1.quick-start-overview/#_2)或[本地](https://docs.nebula-graph.com.cn/3.3.0/2.quick-start/1.quick-start-overview/#_6)快速体验 **NebulaGraph**。 + + + +## 安装方式 + +您可以通过[下载](https://www.nebula-graph.com.cn/download)安装包或者[源码编译](https://docs.nebula-graph.com.cn/3.3.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/1.install-nebula-graph-by-compiling-the-source-code/)安装 **NebulaGraph**： + +## 获取帮助 + +在使用 **NebulaGraph** 过程中遇到任何问题，都可以通过下面的方式寻求帮助： + +* [FAQ](https://docs.nebula-graph.com.cn/3.3.0/20.appendix/0.FAQ/) +* [访问论坛](https://discuss.nebula-graph.com.cn/) +* [查看文档](https://docs.nebula-graph.com.cn/) + +## 如何贡献 + +**NebulaGraph** 是一个完全开源的项目，欢迎开源爱好者通过以下方式参与到 **NebulaGraph** 社区： + +* 在 GitHub 上提 [Issue](https://github.com/vesoft-inc/nebula/issues)。 +* 贡献代码，详情请参见[如何贡献](https://docs.nebula-graph.com.cn/master/15.contribution/how-to-contribute/)。 + +## 许可证 + +**NebulaGraph** 使用 [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) 许可证，您可以免费下载，修改以及部署源代码。
+您还可以将 **NebulaGraph** 作为后端服务部署以支持您的 SaaS 部署。 + +## 联系方式 + +* 访问[官网](http://nebula-graph.com.cn/) +* [![WeiXin](https://img.shields.io/badge/WeChat-%E5%BE%AE%E4%BF%A1-brightgreen)](https://user-images.githubusercontent.com/38887077/67449282-4362b300-f64c-11e9-878f-7efc373e5e55.jpg) +* [![Sina Weibo](https://img.shields.io/badge/Weibo-%E5%BE%AE%E5%8D%9A-red)](https://weibo.com/p/1006067122684542/home?from=page_100606&mod=TAB#place) +* Email: info@vesoft.com + +## 加入 NebulaGraph 社区 + + + +| 加入 NebulaGraph 社区 | 加入方式 | +| ----------------------- | ------------------------------------------------------------ | +| 微信群 | [![WeChat Group](https://img.shields.io/badge/微信群-000000?style=for-the-badge&logo=wechat)](https://wj.qq.com/s2/8321168/8e2f/) | +| 提问 | [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) [![Stack Overflow](https://img.shields.io/badge/Stack%20Overflow-nebula--graph-orange?style=for-the-badge&logo=stack-overflow&logoColor=white)](https://stackoverflow.com/questions/tagged/nebula-graph) [![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) | +| 聊天 | [![Chat History](https://img.shields.io/badge/Community%20Chat-000000?style=for-the-badge&logo=discord&logoColor=white)](https://community-chat.nebula-graph.io/) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) | +| NebulaGraph Meetup 活动 | [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Zoom](https://img.shields.io/badge/Zoom-2D8CFF?style=for-the-badge&logo=zoom&logoColor=white)](https://us02web.zoom.us/meeting/register/tZ0rcuypqDMvGdLuIm4VprTlx96wrEf062SH) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/) [![Meeting Archive](https://img.shields.io/badge/Meeting_Archive-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) | + +
+ +#### 如果你喜欢这个项目，或者对你有用，请[点击](https://github.com/vesoft-inc/nebula)右上角 ⭐️ Star 收藏吧~ + diff --git a/README.md b/README.md index f9be055..b1d778f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,135 @@ -# knowlegegraph +

+ +
English | 中文 +
A distributed, scalable, lightning-fast graph database
+

+

+ + Stack Overflow + + + code coverage + + + nightly build + + + GitHub stars + + + GitHub forks + + + GitHub TODOs + +
+

+ + +# NebulaGraph + +## Introduction + +**NebulaGraph** is a popular open-source graph database that can handle large volumes of data with milliseconds of latency, scale up quickly, and have the ability to perform fast graph analytics. NebulaGraph has been widely used for social media, recommendation systems, knowledge graphs, security, capital flows, AI, etc. See [our users](https://nebula-graph.io/cases). + +The following lists some of **NebulaGraph** features: + +* Symmetrically distributed +* Storage and computing separation +* Horizontal scalability +* Strong data consistency by RAFT protocol +* OpenCypher-compatible query language +* Role-based access control for higher-level security +* Different types of graph analytics algorithms + +The following figure shows the architecture of the **NebulaGraph** core. +![NebulaGraph Architecture](https://docs-cdn.nebula-graph.com.cn/figures/nebula-graph-architecture_3.png) + +Learn more on [NebulaGraph website](https://nebula-graph.io/). + + +## Quick start + +Read the [getting started](https://docs.nebula-graph.io/3.6.0/2.quick-start/1.quick-start-workflow/) docs for a quick start. + +## Using NebulaGraph + +NebulaGraph is a distributed graph database with multiple components. You can [download](https://www.nebula-graph.io/download) or try in following ways: + +- [Build from source](https://docs.nebula-graph.io/3.3.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/1.install-nebula-graph-by-compiling-the-source-code/) + +- In the cloud: [AWS](https://docs.nebula-graph.io/3.1.3/nebula-cloud/nebula-cloud-on-aws/1.aws-overview/) and [Azure](https://docs.nebula-graph.io/3.1.3/nebula-cloud/nebula-cloud-on-azure/azure-self-managed/1.azure-overview/) + +## Getting help + +In case you encounter any problems playing around **NebulaGraph**, please reach out for help: +* [FAQ](https://docs.nebula-graph.io/3.6.0/20.appendix/0.FAQ/) +* [Discussions](https://github.com/vesoft-inc/nebula/discussions) +* [Documentation](https://docs.nebula-graph.io/) + +## DevTools + +NebulaGraph comes with a set of tools to help you manage and monitor your graph database. See [Ecosystem](https://docs.nebula-graph.io/3.6.0/20.appendix/6.eco-tool-version/). + +## Contributing + +Contributions are warmly welcomed and greatly appreciated. And here are a few ways you can contribute: + +* Start by some [issues](https://github.com/vesoft-inc/nebula/issues) +* Submit Pull Requests to us. See [how-to-contribute](https://docs.nebula-graph.io/master/15.contribution/how-to-contribute/). + +## Landscape + +

+ +
+ +[NebulaGraph](https://landscape.cncf.io/?item=app-definition-and-development--database--nebulagraph) enriches the [CNCF Database Landscape](https://landscape.cncf.io/?group=projects-and-products&view-mode=card#app-definition-and-development--database). +

+ +## Licensing + +**NebulaGraph** is under [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license, so you can freely download, modify, and deploy the source code to meet your needs.
+You can also freely deploy **NebulaGraph** as a back-end service to support your SaaS deployment. + +## Contact + +* [Community Chat](https://community-chat.nebula-graph.io/) +* [Slack Channel](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) +* [Stack Overflow](https://stackoverflow.com/questions/tagged/nebula-graph) +* X(Twitter): [@NebulaGraph](https://twitter.com/NebulaGraph) +* [LinkedIn Page](https://www.linkedin.com/company/nebula-graph/) +* Email: info@vesoft.com + +## Community + +| Join NebulaGraph Community | Where to Find us | +| ----------------------------------- | ------------------------------------------------------------ | +| Asking Questions | [![Stack Overflow](https://img.shields.io/badge/Stack%20Overflow-nebula--graph-orange?style=for-the-badge&logo=stack-overflow&logoColor=white)](https://stackoverflow.com/questions/tagged/nebula-graph) [![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) | +| Chat with Community Members | [![Chat History](https://img.shields.io/badge/Community%20Chat-000000?style=for-the-badge&logo=discord&logoColor=white)](https://community-chat.nebula-graph.io/) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) | +| NebulaGraph Meetup | [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Zoom](https://img.shields.io/badge/Zoom-2D8CFF?style=for-the-badge&logo=zoom&logoColor=white)](https://us02web.zoom.us/meeting/register/tZ0rcuypqDMvGdLuIm4VprTlx96wrEf062SH) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/) [![Meeting Archive](https://img.shields.io/badge/Meeting_Archive-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) | +| Chat, Asking, or Meeting in Chinese | [![WeChat Group](https://img.shields.io/badge/WeChat_Group-000000?style=for-the-badge&logo=wechat)](https://wj.qq.com/s2/8321168/8e2f/) [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) | + +
+ +#### If you find NebulaGraph interesting, please ⭐️ [Star](https://github.com/vesoft-inc/nebula) it at the top of the GitHub page. diff --git a/cmake/CPackage.cmake b/cmake/CPackage.cmake new file mode 100644 index 0000000..0034a0e --- /dev/null +++ b/cmake/CPackage.cmake @@ -0,0 +1,172 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# + + +# Used to package into deb or RPM files +macro(package to_one name home_page scripts_dir) + set(CPACK_PACKAGE_DESCRIPTION ${name}) + set(CPACK_PACKAGE_CONTACT ${name}) + set(CPACK_PACKAGE_VERSION ${NEBULA_BUILD_VERSION}) + set(CPACK_RPM_PACKAGE_LICENSE "Apache 2.0") + set(CPACK_PACKAGE_NAME ${name}) + # set(CPACK_SET_DESTDIR TRUE) + set(CPACK_PACKAGE_RELOCATABLE ON) + set(CPACK_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) + set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) + if (EXISTS "/etc/redhat-release") + if(ENABLE_PACKAGE_TAR) + set(CPACK_GENERATOR "TGZ") + else() + set(CPACK_GENERATOR "RPM") + endif() + file (STRINGS "/etc/redhat-release" SYSTEM_NAME) + if (${SYSTEM_NAME} MATCHES "CentOS") + set(HOST_SYSTEM_NAME "el") + execute_process( + COMMAND echo ${SYSTEM_NAME} + COMMAND tr -dc "0-9." + COMMAND cut -d "." -f1 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER}) + elseif (${SYSTEM_NAME} MATCHES "Fedora") + set(HOST_SYSTEM_NAME "fc") + execute_process( + COMMAND echo ${SYSTEM_NAME} + COMMAND cut -d " " -f3 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER}) + else() + set(HOST_SYSTEM_VER "Unknown") + endif() + elseif (EXISTS "/etc/lsb-release") + if(ENABLE_PACKAGE_TAR) + set(CPACK_GENERATOR "TGZ") + else() + set(CPACK_GENERATOR "DEB") + endif() + file (STRINGS "/etc/lsb-release" SYSTEM_NAME) + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND cut -d ";" -f 1 + COMMAND cut -d "=" -f 2 + OUTPUT_VARIABLE HOST_SYSTEM_NAME + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND cut -d ";" -f 2 + COMMAND cut -d "=" -f 2 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + string(REPLACE "." "" HOST_SYSTEM_VER ${HOST_SYSTEM_VER}) + if (${HOST_SYSTEM_NAME} MATCHES "Ubuntu") + string(CONCAT HOST_SYSTEM_VER "ubuntu" ${HOST_SYSTEM_VER}) + # the ubuntu need to modify the architecture name + if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64") + set(CMAKE_HOST_SYSTEM_PROCESSOR "amd64") + endif() + # Adapt the Kylin system + elseif (${HOST_SYSTEM_NAME} MATCHES "Kylin" AND ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64") + string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER}) + set(CMAKE_HOST_SYSTEM_PROCESSOR "arm64") + endif() + elseif (EXISTS "/etc/issue") + file (STRINGS "/etc/issue" SYSTEM_NAME) + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND sed -n "1p" + COMMAND cut -d " " -f 1 + OUTPUT_VARIABLE HOST_SYSTEM_NAME + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND sed -n "1p" + COMMAND cut -d " " -f 3 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if (${HOST_SYSTEM_NAME} MATCHES "Debian") + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND sed -n "1p" + COMMAND cut -d " " -f 3 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(ENABLE_PACKAGE_TAR) + set(CPACK_GENERATOR "TGZ") + else() + set(CPACK_GENERATOR "DEB") + endif() + # Adapt the NeoKylin system + elseif (${HOST_SYSTEM_NAME} MATCHES "NeoKylin" AND ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "mips64") + execute_process( + COMMAND echo "${SYSTEM_NAME}" + COMMAND sed -n "1p" + COMMAND cut -d " " -f 4 + OUTPUT_VARIABLE HOST_SYSTEM_VER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(CMAKE_HOST_SYSTEM_PROCESSOR "mips64el") + if(ENABLE_PACKAGE_TAR) + set(CPACK_GENERATOR "TGZ") + else() + set(CPACK_GENERATOR "RPM") + endif() + endif() + string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER}) + set(CMAKE_HOST_SYSTEM_PROCESSOR "mips64el") + else() + set(HOST_SYSTEM_VER "Unknown") + endif() + + print_config(NEBULA_BUILD_VERSION) + print_config(HOST_SYSTEM_NAME) + print_config(HOST_SYSTEM_VER) + print_config(CPACK_GENERATOR) + print_config(CMAKE_HOST_SYSTEM_PROCESSOR) + + set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.${HOST_SYSTEM_VER}.${CMAKE_HOST_SYSTEM_PROCESSOR}) + if (${to_one}) + set(CPACK_COMPONENTS_GROUPING ALL_COMPONENTS_IN_ONE) + else () + set(CPACK_COMPONENTS_GROUPING ONE_PER_GROUP) + endif() + + set(CPACK_DEB_COMPONENT_INSTALL YES) + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE ${CMAKE_HOST_SYSTEM_PROCESSOR}) + set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${home_page}) + set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA ${scripts_dir}/postinst) + + set(CPACK_RPM_SPEC_MORE_DEFINE "%define debug_package %{nil} + %define __os_install_post %{nil}") + set(CPACK_RPM_COMPONENT_INSTALL YES) + set(CPACK_RPM_PACKAGE_ARCHITECTURE ${CMAKE_HOST_SYSTEM_PROCESSOR}) + set(CPACK_RPM_PACKAGE_URL ${home_page}) + set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE ${scripts_dir}/rpm_postinst) + set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /usr/local) + set(CPACK_RPM_PACKAGE_RELOCATABLE ON) + + include(CPack) + + cpack_add_component(common GROUP common) + cpack_add_component(graph GROUP graph DEPENDS common) + cpack_add_component(storage GROUP storage DEPENDS common) + cpack_add_component(meta GROUP meta DEPENDS common) + cpack_add_component(tool GROUP tool) + cpack_add_component(nebula-console GROUP nebula-console) + cpack_add_component_group(common) + cpack_add_component_group(graph) + cpack_add_component_group(storage) + cpack_add_component_group(meta) + cpack_add_component_group(tool) + cpack_add_component_group(nebula-console) +endmacro() diff --git a/cmake/ClangTidy.cmake b/cmake/ClangTidy.cmake new file mode 100644 index 0000000..57df975 --- /dev/null +++ b/cmake/ClangTidy.cmake @@ -0,0 +1,18 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# + +if (ENABLE_CLANG_TIDY) + if (${CMAKE_VERSION} VERSION_LESS "3.6.0") + message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") + endif() + + find_program (CLANG_TIDY_PATH NAMES "clang-tidy") + if (CLANG_TIDY_PATH) + message(STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}.") + set (CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_PATH} --config=) + else () + message(STATUS "clang-tidy is not found.") + endif () +endif () diff --git a/cmake/FetchModule.cmake b/cmake/FetchModule.cmake new file mode 100644 index 0000000..b81ebc1 --- /dev/null +++ b/cmake/FetchModule.cmake @@ -0,0 +1,75 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# + +find_package(Git) +macro(nebula_fetch_module) + cmake_parse_arguments( + module # + "" # + "URL;TAG;UPDATE;NAME;CHECKOUT" # + "" # + ${ARGN} + ) + set(module_dir ${CMAKE_SOURCE_DIR}/modules/${module_NAME}) + if(NOT EXISTS ${module_dir}/.git) + message(STATUS "Cloning from ${module_URL}:${module_TAG}") + execute_process( + COMMAND + ${GIT_EXECUTABLE} clone + --single-branch + --branch ${module_TAG} + ${module_URL} ${module_dir} + RESULT_VARIABLE fetch_status + ERROR_VARIABLE ERROR_MESSAGE + ) + if(NOT ${fetch_status} EQUAL 0) + message(FATAL_ERROR "Cloning failed") + endif() + else() + if (${module_CHECKOUT}) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD + WORKING_DIRECTORY ${module_dir} + OUTPUT_VARIABLE branch_name + ) + string(REPLACE "\n" "" branch_name "${branch_name}") + if(NOT ${branch_name} STREQUAL ${module_TAG}) + message(STATUS "The branch of ${module_NAME} is ${branch_name}, need to change to ${module_TAG}") + execute_process( + COMMAND ${GIT_EXECUTABLE} remote set-branches origin --add ${module_TAG} + WORKING_DIRECTORY ${module_dir} + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} config remote.origin.fetch + WORKING_DIRECTORY ${module_dir} + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} fetch + WORKING_DIRECTORY ${module_dir} + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} checkout ${module_TAG} + WORKING_DIRECTORY ${module_dir} + RESULT_VARIABLE checkout_status + ERROR_VARIABLE error_msg + ) + if(NOT ${checkout_status} EQUAL 0) + message(FATAL_ERROR "Checkout to branch ${module_TAG} failed: ${error_msg}, error_code: ${checkout_status}") + endif() + endif() + endif() + if(${module_UPDATE}) + message(STATUS "Updating from ${module_URL}") + execute_process( + COMMAND ${GIT_EXECUTABLE} pull + WORKING_DIRECTORY ${module_dir} + RESULT_VARIABLE fetch_status + ) + if(NOT ${fetch_status} EQUAL 0) + message(FATAL_ERROR "Updating failed") + endif() + endif() + endif() +endmacro() diff --git a/cmake/FindBreakpad.cmake b/cmake/FindBreakpad.cmake new file mode 100644 index 0000000..6513638 --- /dev/null +++ b/cmake/FindBreakpad.cmake @@ -0,0 +1,33 @@ +# - Try to find Breakpad includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Breakpad) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Breakpad_FOUND System has breakpad, include and lib dirs found +# Breakpad_INCLUDE_DIR The breakpad includes directories. +# Breakpad_LIBRARY The breakpad library. + +find_path(Breakpad_INCLUDE_DIR NAMES breakpad) +find_library(Breakpad_LIBRARY NAMES libbreakpad_client.a) + +message(STATUS "BREAKPAD: ${Breakpad_INCLUDE_DIR}") + +if(Breakpad_INCLUDE_DIR AND Breakpad_LIBRARY) + set(Breakpad_FOUND TRUE) + mark_as_advanced( + Breakpad_INCLUDE_DIR + Breakpad_LIBRARY + ) +endif() + +if(NOT Breakpad_FOUND) + message(FATAL_ERROR "Breakpad doesn't exist") +endif() + + diff --git a/cmake/FindBzip2.cmake b/cmake/FindBzip2.cmake new file mode 100644 index 0000000..70f944a --- /dev/null +++ b/cmake/FindBzip2.cmake @@ -0,0 +1,34 @@ +# - Try to find Bzip2 includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Bzip2) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Bzip2_FOUND System has bzip2, include and lib dirs found. +# Bzip2_INCLUDE_DIR The bzip2 includes directories. +# Bzip2_LIBRARY The bzip2 library. +# Bzip2_BIN The bzip2 binary. + +find_path(Bzip2_INCLUDE_DIR NAMES bzlib.h) +find_library(Bzip2_LIBRARY NAMES libbz2.a) +find_program(Bzip2_BIN NAMES bzip2) + +if(Bzip2_INCLUDE_DIR AND Bzip2_LIBRARY AND Bzip2_BIN) + set(Bzip2_FOUND TRUE) + mark_as_advanced( + Bzip2_INCLUDE_DIR + Bzip2_LIBRARY + Bzip2_BIN + ) +endif() + +if(NOT Bzip2_FOUND) + message(FATAL_ERROR "Bzip2 doesn't exist") +endif() + + diff --git a/cmake/FindDoubleConversion.cmake b/cmake/FindDoubleConversion.cmake new file mode 100644 index 0000000..cdfc87f --- /dev/null +++ b/cmake/FindDoubleConversion.cmake @@ -0,0 +1,30 @@ +# - Try to find libdouble-conversion includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(DoubleConversion) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# DoubleConversion_FOUND System has double-conversion, include and lib dirs found +# DoubleConversion_INCLUDE_DIR The double-conversion includes directories. +# DoubleConversion_LIBRARY The double-conversion library. + +find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion/double-conversion.h) +find_library(DoubleConversion_LIBRARY NAMES libdouble-conversion.a) + +if(DoubleConversion_INCLUDE_DIR AND DoubleConversion_LIBRARY) + set(DoubleConversion_FOUND TRUE) + mark_as_advanced( + DoubleConversion_INCLUDE_DIR + DoubleConversion_LIBRARY + ) +endif() + +if(NOT DoubleConversion_FOUND) + message(FATAL_ERROR "Double-conversion doesn't exist") +endif() + diff --git a/cmake/FindFatal.cmake b/cmake/FindFatal.cmake new file mode 100644 index 0000000..5b4e287 --- /dev/null +++ b/cmake/FindFatal.cmake @@ -0,0 +1,27 @@ +# - Try to find Fatal includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Fatal) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Fatal_FOUND System has readline, include and lib dirs found +# Fatal_INCLUDE_DIR The readline includes directories. + +find_path(Fatal_INCLUDE_DIR NAMES fatal) + +if(Fatal_INCLUDE_DIR) + set(Fatal_FOUND TRUE) + mark_as_advanced( + Fatal_INCLUDE_DIR + ) +endif() + +if(NOT Fatal_FOUND) + message(FATAL_ERROR "Fatal doesn't exist") +endif() + diff --git a/cmake/FindFbthrift.cmake b/cmake/FindFbthrift.cmake new file mode 100644 index 0000000..0efd4e6 --- /dev/null +++ b/cmake/FindFbthrift.cmake @@ -0,0 +1,33 @@ +# - Try to find Fbthrift includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Fbthrift) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Fbthrift_FOUND System has fbthrift, thrift1 and include and lib dirs found. +# Fbthrift_INCLUDE_DIR The fbthrift includes directories. +# Fbthrift_LIBRARY The fbthrift library. +# Fbthrift_BIN The fbthrift binary. + +find_path(Fbthrift_INCLUDE_DIR NAMES thrift) +find_library(Fbthrift_LIBRARY NAMES libthriftcpp2.a) +find_program(Fbthrift_BIN NAMES thrift1) + +if(Fbthrift_INCLUDE_DIR AND Fbthrift_LIBRARY AND Fbthrift_BIN) + set(Fbthrift_FOUND TRUE) + mark_as_advanced( + Fbthrift_INCLUDE_DIR + Fbthrift_LIBRARY + Fbthrift_BIN + ) +endif() + +if(NOT Fbthrift_FOUND) + message(FATAL_ERROR "Fbthrift doesn't exist") +endif() + diff --git a/cmake/FindFizz.cmake b/cmake/FindFizz.cmake new file mode 100644 index 0000000..d044c71 --- /dev/null +++ b/cmake/FindFizz.cmake @@ -0,0 +1,30 @@ +# - Try to find Fizz includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Fizz) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Fizz_FOUND System has fizz, include and lib dirs found +# Fizz_INCLUDE_DIR The fizz includes directories. +# Fizz_LIBRARY The fizz library. + +find_path(Fizz_INCLUDE_DIR NAMES fizz) +find_library(Fizz_LIBRARY NAMES libfizz.a) + +if(Fizz_INCLUDE_DIR AND Fizz_LIBRARY) + set(Fizz_FOUND TRUE) + mark_as_advanced( + Fizz_INCLUDE_DIR + Fizz_LIBRARY + ) +endif() + +if(NOT Fizz_FOUND) + message(FATAL_ERROR "Fizz doesn't exist") +endif() + diff --git a/cmake/FindFolly.cmake b/cmake/FindFolly.cmake new file mode 100644 index 0000000..ba4e052 --- /dev/null +++ b/cmake/FindFolly.cmake @@ -0,0 +1,30 @@ +# - Try to find Folly includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Folly) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Folly_FOUND System has folly, include and lib dirs found +# Folly_INCLUDE_DIR The folly includes directories. +# Folly_LIBRARY The folly library. + +find_path(Folly_INCLUDE_DIR NAMES folly) +find_library(Folly_LIBRARY NAMES libfolly.a libfollybenchmark.a) + +if(Folly_INCLUDE_DIR AND Folly_LIBRARY) + set(Folly_FOUND TRUE) + mark_as_advanced( + Folly_INCLUDE_DIR + Folly_LIBRARY + ) +endif() + +if(NOT Folly_FOUND) + message(FATAL_ERROR "Folly doesn't exist") +endif() + diff --git a/cmake/FindGflags.cmake b/cmake/FindGflags.cmake new file mode 100644 index 0000000..144e1c0 --- /dev/null +++ b/cmake/FindGflags.cmake @@ -0,0 +1,30 @@ +# - Try to find Gflags includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Gflags) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Gflags_FOUND System has Gflags, include and lib dirs found +# Gflags_INCLUDE_DIR The Gflags includes directories. +# Gflags_LIBRARY The Gflags library. + +find_path(Gflags_INCLUDE_DIR NAMES gflags) +find_library(Gflags_LIBRARY NAMES libgflags.a) + +if(Gflags_INCLUDE_DIR AND Gflags_LIBRARY) + set(Gflags_FOUND TRUE) + mark_as_advanced( + Gflags_INCLUDE_DIR + Gflags_LIBRARY + ) +endif() + +if(NOT Gflags_FOUND) + message(FATAL_ERROR "Gflags doesn't exist") +endif() + diff --git a/cmake/FindGlog.cmake b/cmake/FindGlog.cmake new file mode 100644 index 0000000..5394183 --- /dev/null +++ b/cmake/FindGlog.cmake @@ -0,0 +1,30 @@ +# - Try to find Glog includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Glog) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Glog_FOUND System has Glog, include and lib dirs found +# Glog_INCLUDE_DIR The Glog includes directories. +# Glog_LIBRARY The Glog library. + +find_path(Glog_INCLUDE_DIR NAMES glog) +find_library(Glog_LIBRARY NAMES libglog.a) + +if(Glog_INCLUDE_DIR AND Glog_LIBRARY) + set(Glog_FOUND TRUE) + mark_as_advanced( + Glog_INCLUDE_DIR + Glog_LIBRARY + ) +endif() + +if(NOT Glog_FOUND) + message(FATAL_ERROR "Glog doesn't exist") +endif() + diff --git a/cmake/FindGoogletest.cmake b/cmake/FindGoogletest.cmake new file mode 100644 index 0000000..cb22c8d --- /dev/null +++ b/cmake/FindGoogletest.cmake @@ -0,0 +1,30 @@ +# - Try to find Googletest includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Googletest) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Googletest_FOUND System has Googletest, include and lib dirs found +# Googletest_INCLUDE_DIR The Googletest includes directories. +# Googletest_LIBRARY The Googletest library. + +find_path(Googletest_INCLUDE_DIR NAMES gmock gtest) +find_library(Googletest_LIBRARY NAMES libgmock.a libgmock_main.a libgtest.a libgtest_main.a) + +if(Googletest_INCLUDE_DIR AND Googletest_LIBRARY) + set(Googletest_FOUND TRUE) + mark_as_advanced( + Googletest_INCLUDE_DIR + Googletest_LIBRARY + ) +endif() + +if(NOT Googletest_FOUND) + message(FATAL_ERROR "Googletest doesn't exist") +endif() + diff --git a/cmake/FindJemalloc.cmake b/cmake/FindJemalloc.cmake new file mode 100644 index 0000000..e85a842 --- /dev/null +++ b/cmake/FindJemalloc.cmake @@ -0,0 +1,29 @@ +# - Try to find Jemalloc includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Jemalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Jemalloc_FOUND System has Jemalloc, include and lib dirs found +# Jemalloc_INCLUDE_DIR The Jemalloc includes directories. +# Jemalloc_LIBRARY The Jemalloc library. + +find_path(Jemalloc_INCLUDE_DIR NAMES jemalloc) +find_library(Jemalloc_LIBRARY NAMES libjemalloc.a) + +if(Jemalloc_INCLUDE_DIR AND Jemalloc_LIBRARY) + set(Jemalloc_FOUND TRUE) + mark_as_advanced( + Jemalloc_INCLUDE_DIR + Jemalloc_LIBRARY + ) +endif() + +if(NOT Jemalloc_FOUND) + message(FATAL_ERROR "Jemalloc doesn't exist") +endif() diff --git a/cmake/FindLibevent.cmake b/cmake/FindLibevent.cmake new file mode 100644 index 0000000..4d314af --- /dev/null +++ b/cmake/FindLibevent.cmake @@ -0,0 +1,30 @@ +# - Try to find Libevent includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Libevent) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Libevent_FOUND System has Libevent, include and lib dirs found +# Libevent_INCLUDE_DIR The Libevent includes directories. +# Libevent_LIBRARY The Libevent library. + +find_path(Libevent_INCLUDE_DIR NAMES event.h) +find_library(Libevent_LIBRARY NAMES libevent.a) + +if(Libevent_INCLUDE_DIR AND Libevent_LIBRARY) + set(Libevent_FOUND TRUE) + mark_as_advanced( + Libevent_INCLUDE_DIR + Libevent_LIBRARY + ) +endif() + +if(NOT Libevent_FOUND) + message(FATAL_ERROR "Libevent doesn't exist") +endif() + diff --git a/cmake/FindLibunwind.cmake b/cmake/FindLibunwind.cmake new file mode 100644 index 0000000..3be74fa --- /dev/null +++ b/cmake/FindLibunwind.cmake @@ -0,0 +1,24 @@ +# FindLibunwind +# ------------- +# +# Find Libunwind +# +# Find LibUnwind library +# +# :: +# +# LIBUNWIND_LIBRARY_DIR - Can be provided to advise the search +# +# Libunwind_FOUND - True if libunwind is found. +# LIBUNWIND_LIBRARIES - libunwind libraries to link against. + +find_library(LIBUNWIND_LIBRARY NAMES unwind libunwind.so.8 PATHS ${LIBUNWIND_LIBRARY_DIR}) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Libunwind REQUIRED_VARS LIBUNWIND_LIBRARY) + +if (Libunwind_FOUND) + set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY}) +endif () + +mark_as_advanced( LIBUNWIND_LIBRARY ) diff --git a/cmake/FindMoldLinker.cmake b/cmake/FindMoldLinker.cmake new file mode 100644 index 0000000..fe63cfd --- /dev/null +++ b/cmake/FindMoldLinker.cmake @@ -0,0 +1,18 @@ +# Copyright (c) 2023 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# + +find_program (MOLD NAMES "mold") +if (NOT MOLD) + message(FATAL_ERROR "Could not find the mold linker") +endif() + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12.1.0) + get_filename_component(MOLD_PATH ${MOLD} DIRECTORY) + nebula_add_exe_linker_flag(-B${MOLD_PATH}/../libexec/mold) + nebula_add_shared_linker_flag(-B${MOLD_PATH}/../libexec/mold) +else() + nebula_add_exe_linker_flag(-fuse-ld=mold) + nebula_add_shared_linker_flag(-fuse-ld=mold) +endif() diff --git a/cmake/FindPCHSupport.cmake b/cmake/FindPCHSupport.cmake new file mode 100644 index 0000000..04a36b1 --- /dev/null +++ b/cmake/FindPCHSupport.cmake @@ -0,0 +1,95 @@ +# This code is based on +# http://rosegarden.svn.sourceforge.net/viewvc/rosegarden/trunk/rosegarden/cmake_admin/FindPCHSupport.cmake?revision=7699&view=markup +# +# - Try to find precompiled headers support for GCC 3.4 and 4.x +# Once done this will define: +# +# Variable: +# PCHSupport_FOUND +# +# Macro: +# ADD_PRECOMPILED_HEADER + +IF(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + EXEC_PROGRAM( + ${CMAKE_CXX_COMPILER} + ARGS --version + OUTPUT_VARIABLE _compiler_output) + STRING(REGEX REPLACE ".* ([0-9]\\.[0-9]\\.[0-9]) .*" "\\1" + gcc_compiler_version ${_compiler_output}) + IF(gcc_compiler_version MATCHES "[4-9]\\.[0-9]\\.[0-9]") + SET(PCHSupport_FOUND TRUE) + MESSAGE("-- Found PCH Support: gcc compiler version is " ${gcc_compiler_version}) + ELSE() + IF(gcc_compiler_version MATCHES "3\\.4\\.[0-9]") + SET(PCHSupport_FOUND TRUE) + MESSAGE("-- Found PCH Support: gcc compiler version is " ${gcc_compiler_version}) + ENDIF() + ENDIF() +ELSEIF(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + SET(PCHSupport_FOUND TRUE) +ENDIF() + + + +MACRO(ADD_PRECOMPILED_HEADER _targetName _input) + + GET_FILENAME_COMPONENT(_name ${_input} NAME) + SET(_source "${CMAKE_CURRENT_SOURCE_DIR}/${_input}") + SET(_output "${CMAKE_CURRENT_SOURCE_DIR}/${_name}.gch") + STRING(TOUPPER "CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}" _flags_var_name) + SET(_compiler_FLAGS ${${_flags_var_name}}) + GET_DIRECTORY_PROPERTY(_directory_compile_options COMPILE_OPTIONS) + LIST(APPEND _compiler_FLAGS ${_directory_compile_options}) + + GET_DIRECTORY_PROPERTY(_directory_flags INCLUDE_DIRECTORIES) + SET(_system_INCLUDE_DIRS "/usr/include" "/usr/local/include") + FOREACH(item ${_directory_flags}) + # Exclude standard paths + LIST(FIND _system_INCLUDE_DIRS ${item} _index) + IF(NOT ${_index} EQUAL -1) + continue() + ENDIF() + + IF(item MATCHES "^${PROJECT_SOURCE_DIR}.*") + # Directories in this project + IF(item MATCHES "^${PROJECT_SOURCE_DIR}/.*third-party.*") + # third-party + LIST(APPEND _compiler_FLAGS "-isystem ${item}") + ELSE() + # Our own directories + LIST(APPEND _compiler_FLAGS "-I ${item}") + ENDIF() + ELSE() + # All of others + LIST(APPEND _compiler_FLAGS "-isystem ${item}") + ENDIF() + ENDFOREACH(item) + + GET_DIRECTORY_PROPERTY(_directory_flags COMPILE_DEFINITIONS) + FOREACH(item ${_directory_flags}) + IF(NOT ${item} MATCHES "^GIT_INFO_SHA") + LIST(APPEND _compiler_FLAGS "-D${item}") + ENDIF() + ENDFOREACH(item) + + SEPARATE_ARGUMENTS(_compiler_FLAGS) + MESSAGE("Precompile header file " ${_source} " into " ${_output}) + IF(CMAKE_CXX_EXTENSIONS OR NOT DEFINED CMAKE_CXX_EXTENSIONS) + SET(_cxx_standard "gnu++${CMAKE_CXX_STANDARD}") + ELSE() + SET(_cxx_standard "c++${CMAKE_CXX_STANDARD}") + ENDIF() + ADD_CUSTOM_COMMAND( + OUTPUT ${_output} + COMMAND ${CMAKE_CXX_COMPILER} + ${_compiler_FLAGS} + -x c++-header + -std=${_cxx_standard} + -o ${_output} ${_source} + MAIN_DEPENDENCY ${_source} + DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt) + ADD_CUSTOM_TARGET(${_targetName}_gch DEPENDS ${_output}) + ADD_DEPENDENCIES(${_targetName} ${_targetName}_gch) + +ENDMACRO(ADD_PRECOMPILED_HEADER) diff --git a/cmake/FindProxygen.cmake b/cmake/FindProxygen.cmake new file mode 100644 index 0000000..88e888b --- /dev/null +++ b/cmake/FindProxygen.cmake @@ -0,0 +1,31 @@ +# - Try to find Proxygen includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Proxygen) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Proxygen_FOUND System has Proxygen, include and lib dirs found +# Proxygen_INCLUDE_DIR The Proxygen includes directories. +# Proxygen_LIBRARY The Proxygen library. + +find_path(Proxygen_INCLUDE_DIR NAMES proxygen) +find_library(Proxygen_LIBRARY NAMES libproxygen.a) + +if(Proxygen_INCLUDE_DIR AND Proxygen_LIBRARY) + set(Proxygen_FOUND TRUE) + mark_as_advanced( + Proxygen_INCLUDE_DIR + Proxygen_LIBRARY + ) +endif(Proxygen_INCLUDE_DIR AND Proxygen_LIBRARY) + +if(NOT Proxygen_FOUND) + message(FATAL_ERROR "Proxygen doesn't exist") +endif() + + diff --git a/cmake/FindRocksdb.cmake b/cmake/FindRocksdb.cmake new file mode 100644 index 0000000..7160e18 --- /dev/null +++ b/cmake/FindRocksdb.cmake @@ -0,0 +1,30 @@ +# - Try to find Rocksdb includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Rocksdb) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Rocksdb_FOUND System has Rocksdb, include and lib dirs found +# Rocksdb_INCLUDE_DIR The Rocksdb includes directories. +# Rocksdb_LIBRARY The Rocksdb library. + +find_path(Rocksdb_INCLUDE_DIR NAMES rocksdb) +find_library(Rocksdb_LIBRARY NAMES librocksdb.a) + +if(Rocksdb_INCLUDE_DIR AND Rocksdb_LIBRARY) + set(Rocksdb_FOUND TRUE) + mark_as_advanced( + Rocksdb_INCLUDE_DIR + Rocksdb_LIBRARY + ) +endif() + +if(NOT Rocksdb_FOUND) + message(FATAL_ERROR "Rocksdb doesn't exist") +endif() + diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake new file mode 100644 index 0000000..bb7b5d1 --- /dev/null +++ b/cmake/FindSnappy.cmake @@ -0,0 +1,30 @@ +# - Try to find Snappy includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Snappy) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Snappy_FOUND System has Snappy, include and lib dirs found +# Snappy_INCLUDE_DIR The Snappy includes directories. +# Snappy_LIBRARY The Snappy library. + +find_path(Snappy_INCLUDE_DIR NAMES snappy.h) +find_library(Snappy_LIBRARY NAMES libsnappy.a) + +if(Snappy_INCLUDE_DIR AND Snappy_LIBRARY) + set(Snappy_FOUND TRUE) + mark_as_advanced( + Snappy_INCLUDE_DIR + Snappy_LIBRARY + ) +endif() + +if(NOT Snappy_FOUND) + message(FATAL_ERROR "Snappy doesn't exist") +endif() + diff --git a/cmake/FindSodium.cmake b/cmake/FindSodium.cmake new file mode 100644 index 0000000..01143cd --- /dev/null +++ b/cmake/FindSodium.cmake @@ -0,0 +1,30 @@ +# - Try to find Sodium includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Sodium) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Sodium_FOUND System has Sodium, include and lib dirs found +# Sodium_INCLUDE_DIR The Sodium includes directories. +# Sodium_LIBRARY The Sodium library. + +find_path(Sodium_INCLUDE_DIR NAMES sodium) +find_library(Sodium_LIBRARY NAMES libsodium.a) + +if(Sodium_INCLUDE_DIR AND Sodium_LIBRARY) + set(Sodium_FOUND TRUE) + mark_as_advanced( + Sodium_INCLUDE_DIR + Sodium_LIBRARY + ) +endif() + +if(NOT Sodium_FOUND) + message(FATAL_ERROR "Sodium doesn't exist") +endif() + diff --git a/cmake/FindWangle.cmake b/cmake/FindWangle.cmake new file mode 100644 index 0000000..d3e55e0 --- /dev/null +++ b/cmake/FindWangle.cmake @@ -0,0 +1,30 @@ +# - Try to find Wangle includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Wangle) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Wangle_FOUND System has Wangle, include and lib dirs found +# Wangle_INCLUDE_DIR The Wangle includes directories. +# Wangle_LIBRARY The Wangle library. + +find_path(Wangle_INCLUDE_DIR NAMES wangle) +find_library(Wangle_LIBRARY NAMES libwangle.a) + +if(Wangle_INCLUDE_DIR AND Wangle_LIBRARY) + set(Wangle_FOUND TRUE) + mark_as_advanced( + Wangle_INCLUDE_DIR + Wangle_LIBRARY + ) +endif() + +if(NOT Wangle_FOUND) + message(FATAL_ERROR "Wangle doesn't exist") +endif() + diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake new file mode 100644 index 0000000..3501b1c --- /dev/null +++ b/cmake/FindZstd.cmake @@ -0,0 +1,32 @@ +# - Try to find Zstd includes dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Zstd) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Variables defined by this module: +# +# Zstd_FOUND System has Zstd, include and lib dirs found +# Zstd_INCLUDE_DIR The Zstd includes directories. +# Zstd_LIBRARY The Zstd library. +# Zstd_BIN The Zstd binary. + +find_path(Zstd_INCLUDE_DIR NAMES zstd.h) +find_library(Zstd_LIBRARY NAMES libzstd.a) + +if(Zstd_INCLUDE_DIR AND Zstd_LIBRARY) + set(Zstd_FOUND TRUE) + mark_as_advanced( + Zstd_INCLUDE_DIR + Zstd_LIBRARY + ) +endif() + +if(NOT Zstd_FOUND) + message(FATAL_ERROR "Zstd doesn't exist") +endif() + + diff --git a/cmake/ThriftGenerate.cmake b/cmake/ThriftGenerate.cmake new file mode 100644 index 0000000..353a33c --- /dev/null +++ b/cmake/ThriftGenerate.cmake @@ -0,0 +1,138 @@ +# +# Requirements: +# Please provide the following two variables before using these macros: +# ${THRIFT1} - path/to/bin/thrift1 +# ${THRIFT_TEMPLATES} - path/to/include/thrift/templates +# ${THRIFTCPP2} - path/to/lib/thriftcpp2 +# +# +# bypass_source_check +# This tells cmake to ignore if it doesn't see the following sources in +# the library that will be installed. Thrift files are generated at compile +# time so they do not exist at source check time +# +# Params: +# @sources - The list of files to ignore in source check +# + +macro(bypass_source_check sources) +set_source_files_properties( + ${sources} + PROPERTIES GENERATED TRUE + ) +endmacro() + + +# +# thrift_generate +# This is used to codegen thrift files using the thrift compiler +# Params: +# @file_name - The name of tge thrift file +# @services - A list of services that are declared in the thrift file +# @output_path - The directory where the thrift file lives +# +# Output: +# file-cpp2-target - A custom target to add a dependency +# ${file-cpp2-HEADERS} - The generated Header Files. +# ${file-cpp2-SOURCES} - The generated Source Files. +# +# Notes: +# If any of the fields is empty, it is still required to provide an empty string +# +# When using file_cpp2-SOURCES it should always call: +# bypass_source_check(${file_cpp2-SOURCES}) +# This will prevent cmake from complaining about missing source files +# + +macro(thrift_generate file_name services file_path output_path include_prefix) +set("${file_name}-cpp2-HEADERS" + ${output_path}/gen-cpp2/${file_name}_constants.h + ${output_path}/gen-cpp2/${file_name}_data.h + ${output_path}/gen-cpp2/${file_name}_types.h + ${output_path}/gen-cpp2/${file_name}_types_custom_protocol.h + ${output_path}/gen-cpp2/${file_name}_types.tcc +) + +set("${file_name}-cpp2-SOURCES" + ${output_path}/gen-cpp2/${file_name}_constants.cpp + ${output_path}/gen-cpp2/${file_name}_data.cpp + ${output_path}/gen-cpp2/${file_name}_metadata.cpp + ${output_path}/gen-cpp2/${file_name}_types.cpp +) + +foreach(service ${services}) + set("${file_name}-cpp2-HEADERS" + ${${file_name}-cpp2-HEADERS} + ${output_path}/gen-cpp2/${service}.h + ${output_path}/gen-cpp2/${service}.tcc + ${output_path}/gen-cpp2/${service}AsyncClient.h + ${output_path}/gen-cpp2/${service}_custom_protocol.h + ) + set("${file_name}-cpp2-SOURCES" + ${${file_name}-cpp2-SOURCES} + ${output_path}/gen-cpp2/${service}.cpp + ${output_path}/gen-cpp2/${service}AsyncClient.cpp + ${output_path}/gen-cpp2/${service}_processmap_binary.cpp + ${output_path}/gen-cpp2/${service}_processmap_compact.cpp + ) +endforeach() + +add_custom_command( + OUTPUT ${${file_name}-cpp2-HEADERS} ${${file_name}-cpp2-SOURCES} + COMMAND ${THRIFT1} + --strict "--allow-neg-enum-vals" + --gen "mstch_cpp2:include_prefix=${include_prefix},stack_arguments" + --gen "py" + --gen "js:node:" + --gen "csharp" + --gen "java:hashcode" + --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v3/,use_context" + -o "." "${file_path}/${file_name}.thrift" + COMMAND + mkdir -p "./gen-rust/${file_name}" + && ${THRIFT1} + --strict "--allow-neg-enum-vals" + --gen "mstch_rust" + -o "gen-rust/${file_name}" + "${file_path}/${file_name}.thrift" + && ( mv ./gen-rust/${file_name}/gen-rust/lib.rs ./gen-rust/${file_name} ) + && ( rm -r ./gen-rust/${file_name}/gen-rust/ ) + DEPENDS "${file_path}/${file_name}.thrift" + COMMENT "Generating thrift files for ${file_name}" +) + +bypass_source_check(${${file_name}-cpp2-SOURCES}) +add_custom_target( + ${file_name}_thrift_generator + DEPENDS ${${file_name}-cpp2-HEADERS} ${${file_name}-cpp2-SOURCES} +) + +add_library( + "${file_name}_thrift_obj" + OBJECT + ${${file_name}-cpp2-SOURCES} +) +add_dependencies(${file_name}_thrift_obj ${file_name}_thrift_generator) + +set_target_properties( + "${file_name}_thrift_obj" + PROPERTIES CXX_CLANG_TIDY "" +) + +target_compile_options(${file_name}_thrift_obj PRIVATE "-Wno-pedantic") +target_compile_options(${file_name}_thrift_obj PRIVATE "-Wno-extra") + +if(NOT "${file_name}" STREQUAL "common") + add_dependencies( + ${file_name}_thrift_obj + common_thrift_generator + ) +endif() + +if("${file_name}" STREQUAL "storage") + add_dependencies( + ${file_name}_thrift_obj + meta_thrift_generator + ) +endif() +endmacro() diff --git a/cmake/nebula/ABIConfig.cmake b/cmake/nebula/ABIConfig.cmake new file mode 100644 index 0000000..2ef15f0 --- /dev/null +++ b/cmake/nebula/ABIConfig.cmake @@ -0,0 +1,8 @@ +# Is abi 11 +if (${DISABLE_CXX11_ABI}) + message(STATUS "Set D_GLIBCXX_USE_CXX11_ABI to 0") + add_compile_options(-D_GLIBCXX_USE_CXX11_ABI=0) +else() + message(STATUS "Set D_GLIBCXX_USE_CXX11_ABI to 1") + add_compile_options(-D_GLIBCXX_USE_CXX11_ABI=1) +endif() diff --git a/cmake/nebula/CompilerLauncher.cmake b/cmake/nebula/CompilerLauncher.cmake new file mode 100644 index 0000000..1c8f12f --- /dev/null +++ b/cmake/nebula/CompilerLauncher.cmake @@ -0,0 +1,29 @@ +option(ENABLE_COMPILER_LAUNCHER "Using compiler launcher if available" ON) +if(NOT (ENABLE_COMPILER_LAUNCHER AND ENABLE_CCACHE)) + return() +endif() + +set(COMPILER_LAUNCHER_OPTION + "ccache" + CACHE STRING "Compiler cache to be used") +set(COMPILER_LAUNCHER_OPTION_VALUES "ccache" "sccache" "distcc") +set_property(CACHE COMPILER_LAUNCHER_OPTION PROPERTY STRINGS ${COMPILER_LAUNCHER_OPTION_VALUES}) +list( + FIND + COMPILER_LAUNCHER_OPTION_VALUES + ${COMPILER_LAUNCHER_OPTION} + COMPILER_LAUNCHER_OPTION_INDEX) + +if(${COMPILER_LAUNCHER_OPTION_INDEX} EQUAL -1) + message( + STATUS + "Using custom compiler launcher system: '${COMPILER_LAUNCHER_OPTION}', explicitly supported entries are ${COMPILER_LAUNCHER_OPTION_VALUES}") +endif() + +find_program(COMPILER_LAUNCHER_BINARY ${COMPILER_LAUNCHER_OPTION}) +if(COMPILER_LAUNCHER_BINARY) + message(STATUS "${COMPILER_LAUNCHER_OPTION} found and enabled") + set(CMAKE_CXX_COMPILER_LAUNCHER ${COMPILER_LAUNCHER_BINARY}) +else() + message(WARNING "${COMPILER_LAUNCHER_OPTION} is enabled but was not found. Not using it") +endif() diff --git a/cmake/nebula/GeneralCMakeConfig.cmake b/cmake/nebula/GeneralCMakeConfig.cmake new file mode 100644 index 0000000..8c71828 --- /dev/null +++ b/cmake/nebula/GeneralCMakeConfig.cmake @@ -0,0 +1,25 @@ +# By default, all dynamic and static libraries will be placed at ${CMAKE_BINARY_DIR}/lib, +# while all executables at ${CMAKE_BINARY_DIR}/bin. +# But for the sake of cleanliness, all executables ending with `_test' will be placed +# at ${CMAKE_BINARY_DIR}/bin/test, while those ending with `_bm' at ${CMAKE_BINARY_DIR}/bin/bench. +# Please see `nebula_add_executable' for this rule. +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + +if(ENABLE_TESTING) + enable_testing() +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() +print_config(CMAKE_BUILD_TYPE) + +if (${CMAKE_INSTALL_PREFIX} STREQUAL "/usr/local") + set(CMAKE_INSTALL_PREFIX "/usr/local/nebula") +endif() +print_config(CMAKE_INSTALL_PREFIX) + + +set(CMAKE_SKIP_RPATH TRUE) diff --git a/cmake/nebula/GeneralCMakeOptions.cmake b/cmake/nebula/GeneralCMakeOptions.cmake new file mode 100644 index 0000000..0e3c855 --- /dev/null +++ b/cmake/nebula/GeneralCMakeOptions.cmake @@ -0,0 +1,31 @@ +############ Nebula defined options immutable during configure stage ############# +message(">>>> Options of ${PROJECT_NAME} <<<<") +option(ENABLE_TESTING "Build unit tests" ON) +option(ENABLE_CCACHE "Use ccache to speed up compiling" ON) +option(ENABLE_WERROR "Regard warnings as errors" ON) +option(ENABLE_JEMALLOC "Use jemalloc as memory allocator" ON) +option(ENABLE_ASAN "Build with AddressSanitizer" OFF) +option(ENABLE_UBSAN "Build with UndefinedBehaviourSanitizer" OFF) +option(ENABLE_TSAN "Build with ThreadSanitizer" OFF) +option(ENABLE_STATIC_ASAN "Statically link against libasan" OFF) +option(ENABLE_STATIC_UBSAN "Statically link against libubsan" OFF) +option(ENABLE_FUZZY_TESTING "Enable Fuzzy tests" OFF) +option(ENABLE_FRAME_POINTER "Build with frame pointer" OFF) +option(ENABLE_STRICT_ALIASING "Build with -fstrict-aliasing" OFF) +option(ENABLE_COVERAGE "Build with coverage report" OFF) +option(ENABLE_PIC "Build with -fPIC" OFF) +option(ENABLE_COMPRESSED_DEBUG_INFO "Compress debug info to reduce binary size" ON) +option(ENABLE_CLANG_TIDY "Enable clang-tidy if present" OFF) +option(ENABLE_GDB_SCRIPT_SECTION "Add .debug_gdb_scripts section" OFF) +option(DISABLE_CXX11_ABI "Whether to disable cxx11 abi" OFF) +option(ENABLE_BREAKPAD "Whether to enable breakpad" OFF) +option(ENABLE_STANDALONE_VERSION "Enable standalone version build" OFF) +option(ENABLE_MEMORY_TRACKER "Enable memory tracker" ON) + +get_cmake_property(variable_list VARIABLES) +foreach(_varname ${variable_list}) + string(REGEX MATCH "^ENABLE" matched ${_varname}) + if(matched) + print_option(${_varname}) + endif() +endforeach() diff --git a/cmake/nebula/GeneralCompilerConfig.cmake b/cmake/nebula/GeneralCompilerConfig.cmake new file mode 100644 index 0000000..7106d0a --- /dev/null +++ b/cmake/nebula/GeneralCompilerConfig.cmake @@ -0,0 +1,93 @@ +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +print_config(CMAKE_CXX_STANDARD) +print_config(CMAKE_CXX_COMPILER) +print_config(CMAKE_CXX_COMPILER_ID) + +if (!CMAKE_CXX_COMPILER) + message(FATAL_ERROR "No C++ compiler found") +endif() + +include(CheckCXXCompilerFlag) + +add_compile_options(-Wall) +add_compile_options(-Wextra) +add_compile_options(-Wpedantic) +add_compile_options(-Wunused-parameter) +add_compile_options(-Wshadow) +add_compile_options(-Wnon-virtual-dtor) +add_compile_options(-Woverloaded-virtual) +add_compile_options(-Wignored-qualifiers) + +# For s2 +add_definitions(-DS2_USE_GLOG) +add_definitions(-DS2_USE_GFLAGS) +# For breakpad +add_definitions(-D__STDC_FORMAT_MACROS) + +include_directories(AFTER ${CMAKE_SOURCE_DIR}/src) +include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src) + + +if(ENABLE_WERROR) + add_compile_options(-Werror) + add_compile_options(-Wno-attributes) +endif() + +if(NOT ENABLE_STRICT_ALIASING) + add_compile_options(-fno-strict-aliasing) +else() + add_compile_options(-fstrict-aliasing) +endif() + +if(ENABLE_FRAME_POINTER) + add_compile_options(-fno-omit-frame-pointer) +else() + add_compile_options(-fomit-frame-pointer) +endif() + +if(ENABLE_PIC) + add_compile_options(-fPIC) +endif() + +if(ENABLE_TESTING AND ENABLE_COVERAGE) + add_compile_options(--coverage) + add_compile_options(-g) + add_compile_options(-O0) + nebula_add_exe_linker_flag(-coverage) + nebula_add_exe_linker_flag(-lgcov) +endif() + +# TODO(doodle) Add option suggest-override for gnu +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # Here we need to specify the path of libstdc++ used by Clang + if(NOT ${NEBULA_CLANG_USE_GCC_TOOLCHAIN} STREQUAL "") + print_config(NEBULA_CLANG_USE_GCC_TOOLCHAIN) + execute_process( + COMMAND ${NEBULA_CLANG_USE_GCC_TOOLCHAIN}/bin/gcc -dumpmachine + OUTPUT_VARIABLE gcc_target_triplet + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + add_compile_options("--gcc-toolchain=${NEBULA_CLANG_USE_GCC_TOOLCHAIN}") + nebula_add_exe_linker_flag(--gcc-toolchain=${NEBULA_CLANG_USE_GCC_TOOLCHAIN}) + nebula_add_shared_linker_flag(--gcc-toolchain=${NEBULA_CLANG_USE_GCC_TOOLCHAIN}) + if(NOT "${gcc_target_triplet}" STREQUAL "") + add_compile_options(--target=${gcc_target_triplet}) + nebula_add_exe_linker_flag(--target=${gcc_target_triplet}) + nebula_add_shared_linker_flag(--target=${gcc_target_triplet}) + endif() + endif() +#add_compile_options(-Wno-mismatched-tags) + add_compile_options(-Wno-self-assign-overloaded) + add_compile_options(-Wno-self-move) + add_compile_options(-Wno-format-pedantic) + add_compile_options(-Wno-gnu-zero-variadic-macro-arguments) + set(libatomic_link_flags "-Wl,-Bstatic -latomic -Wl,-Bdynamic") + set(CMAKE_REQUIRED_FLAGS "${libatomic_link_flags}") + check_cxx_compiler_flag("${libatomic_link_flags}" has_static_libatomic) + if(NOT has_static_libatomic) + set(libatomic_link_flags "-latomic") + endif() +endif() diff --git a/cmake/nebula/GitHooksConfig.cmake b/cmake/nebula/GitHooksConfig.cmake new file mode 100644 index 0000000..6573efa --- /dev/null +++ b/cmake/nebula/GitHooksConfig.cmake @@ -0,0 +1,19 @@ +if(ENABLE_CREATE_GIT_HOOKS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git/") + # Create the pre-commit hook every time we run cmake + message(STATUS "Create the pre-commit hook") + set(PRE_COMMIT_HOOK ${CMAKE_CURRENT_SOURCE_DIR}/.git/hooks/pre-commit) + execute_process( + COMMAND + "rm" "-f" ${PRE_COMMIT_HOOK} + ) + execute_process( + COMMAND + "ln" "-s" ${CMAKE_CURRENT_SOURCE_DIR}/.linters/cpp/hooks/pre-commit.sh ${PRE_COMMIT_HOOK} + RESULT_VARIABLE retcode + ) + if(${retcode} EQUAL 0) + MESSAGE(STATUS "Creating pre-commit hook done") + else() + MESSAGE(FATAL_ERROR "Creating pre-commit hook failed: ${retcode}") + endif() +endif() diff --git a/cmake/nebula/GitInfoConfig.cmake b/cmake/nebula/GitInfoConfig.cmake new file mode 100644 index 0000000..c5e6703 --- /dev/null +++ b/cmake/nebula/GitInfoConfig.cmake @@ -0,0 +1,12 @@ +find_package(Git) +if (GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_INFO_SHA + ) +endif() + +if (GIT_INFO_SHA) + string(REGEX REPLACE "[^0-9a-f]+" "" GIT_INFO_SHA "${GIT_INFO_SHA}") +endif() diff --git a/cmake/nebula/IncludeWhatYouUse.cmake b/cmake/nebula/IncludeWhatYouUse.cmake new file mode 100644 index 0000000..07e86c0 --- /dev/null +++ b/cmake/nebula/IncludeWhatYouUse.cmake @@ -0,0 +1,10 @@ +if(ENABLE_INCLUDE_WHAT_YOU_USE) + find_program(INCLUDE_WHAT_YOU_USE include-what-you-use) + if(INCLUDE_WHAT_YOU_USE) + message("use include-what-you-use") + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "include-what-you-use;-Xiwyu;--transitive_includes_only") + else() + message(STATUS "iwyu requested but executable not found") + endif() +endif() + diff --git a/cmake/nebula/InstallThirdParty.cmake b/cmake/nebula/InstallThirdParty.cmake new file mode 100644 index 0000000..f0a5a38 --- /dev/null +++ b/cmake/nebula/InstallThirdParty.cmake @@ -0,0 +1,28 @@ +set(third_party_install_prefix ${CMAKE_BINARY_DIR}/third-party/install) +message(STATUS "Downloading prebuilt third party automatically...") +if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + if(NOT ${NEBULA_CLANG_USED_GCC_TOOLCHAIN} STREQUAL "") + set(cxx_cmd ${NEBULA_CLANG_USED_GCC_TOOLCHAIN}/bin/g++) + else() + set(cxx_cmd ${CMAKE_CXX_COMPILER}) + endif() +else() + set(cxx_cmd ${CMAKE_CXX_COMPILER}) +endif() +if(${DISABLE_CXX11_ABI}) + set(cxx_cmd "${cxx_cmd} -D_GLIBCXX_USE_CXX11_ABI=0") +else() + set(cxx_cmd "${cxx_cmd} -D_GLIBCXX_USE_CXX11_ABI=1") +endif() +message(STATUS "cxx_cmd: ${cxx_cmd}") +execute_process( + COMMAND + env CXX=${cxx_cmd} version=${NEBULA_THIRDPARTY_VERSION} ${CMAKE_SOURCE_DIR}/third-party/install-third-party.sh --prefix=${third_party_install_prefix} + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} +) + +if(EXISTS ${third_party_install_prefix}) + set(NEBULA_THIRDPARTY_ROOT ${third_party_install_prefix}) +endif() +unset(third_party_install_prefix) diff --git a/cmake/nebula/LinkerConfig.cmake b/cmake/nebula/LinkerConfig.cmake new file mode 100644 index 0000000..434555b --- /dev/null +++ b/cmake/nebula/LinkerConfig.cmake @@ -0,0 +1,61 @@ +set(NEBULA_USE_LINKER + "bfd" + CACHE STRING "Linker to be used") +set(USER_LINKER_OPTION_VALUES "lld" "gold" "bfd" "mold") +set_property(CACHE NEBULA_USE_LINKER PROPERTY STRINGS ${USER_LINKER_OPTION_VALUES}) +list( + FIND + USER_LINKER_OPTION_VALUES + ${NEBULA_USE_LINKER} + USER_LINKER_OPTION_INDEX) + +if(${USER_LINKER_OPTION_INDEX} EQUAL -1) + message( + STATUS + "Using custom linker: '${NEBULA_USE_LINKER}', explicitly supported entries are ${USER_LINKER_OPTION_VALUES}") +endif() + +print_config(NEBULA_USE_LINKER) + +if (${NEBULA_USE_LINKER} STREQUAL "mold") + include(FindMoldLinker) +else() + nebula_add_exe_linker_flag(-fuse-ld=${NEBULA_USE_LINKER}) + nebula_add_shared_linker_flag(-fuse-ld=${NEBULA_USE_LINKER}) +endif() + +nebula_add_exe_linker_flag(-static-libstdc++) +nebula_add_exe_linker_flag(-static-libgcc) +nebula_add_exe_linker_flag(-no-pie) +nebula_add_exe_linker_flag(-rdynamic) + +if(NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug") + add_definitions(-D_FORTIFY_SOURCE=2) +else() + # The mips need to add it when build Debug to lift the usual restrictions on the size of the global offset table. + if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "mips64") + add_compile_options(-mxgot) + endif() + + if (NOT ${NEBULA_USE_LINKER} STREQUAL "gold" AND NOT ENABLE_GDB_SCRIPT_SECTION) + # `gold' linker is buggy for `--gc-sections', disabled for it + # `gc-sections' will discard the `.debug_gdb_scripts' section if enabled + add_compile_options(-ffunction-sections) + add_compile_options(-fdata-sections) + nebula_add_exe_linker_flag(-Wl,--gc-sections) + endif() +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + # The mips not supported + if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "mips64") + if(ENABLE_COMPRESSED_DEBUG_INFO) + nebula_add_exe_linker_flag(-Wl,--compress-debug-sections=zlib) + else() + nebula_remove_exe_linker_flag(-Wl,--compress-debug-sections=zlib) + endif() + endif() + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + add_compile_options(-fno-limit-debug-info) + endif() +endif() diff --git a/cmake/nebula/MakeBisonRelocatable.cmake b/cmake/nebula/MakeBisonRelocatable.cmake new file mode 100644 index 0000000..683c936 --- /dev/null +++ b/cmake/nebula/MakeBisonRelocatable.cmake @@ -0,0 +1,17 @@ +set(BISON_EXECUTE_ENV "") +execute_process( + COMMAND ${BISON_EXECUTABLE} --print-datadir + OUTPUT_VARIABLE bison_encoded_data_dir + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(NOT EXISTS ${bison_encoded_data_dir}) + get_filename_component(bison_prefix ${BISON_EXECUTABLE} DIRECTORY) + get_filename_component(bison_prefix ${bison_prefix} DIRECTORY) + if(EXISTS ${bison_prefix}/share/bison) + set(BISON_EXECUTE_ENV "BISON_PKGDATADIR=${bison_prefix}/share/bison") + endif() +endif() +if(NOT ${BISON_EXECUTE_ENV} STREQUAL "") + set(BISON_EXECUTABLE ${CMAKE_COMMAND} -E env ${BISON_EXECUTE_ENV} ${BISON_EXECUTABLE}) +endif() + diff --git a/cmake/nebula/NebulaCMakeMacros.cmake b/cmake/nebula/NebulaCMakeMacros.cmake new file mode 100644 index 0000000..5542f09 --- /dev/null +++ b/cmake/nebula/NebulaCMakeMacros.cmake @@ -0,0 +1,166 @@ +macro(nebula_add_executable) + cmake_parse_arguments( + nebula_exec # prefix + "" # + "NAME" # + "SOURCES;OBJECTS;LIBRARIES" # + ${ARGN} + ) + add_executable( + ${nebula_exec_NAME} + ${nebula_exec_SOURCES} + ${nebula_exec_OBJECTS} + ) + nebula_link_libraries( + ${nebula_exec_NAME} + ${nebula_exec_LIBRARIES} + ) + + if(${nebula_exec_NAME} MATCHES "_test$") + set_target_properties( + ${nebula_exec_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/test + ) + elseif(${nebula_exec_NAME} MATCHES "_bm$") + set_target_properties( + ${nebula_exec_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/bench + ) + endif() + if(TARGET common_project) + add_dependencies( + ${nebula_exec_NAME} + common_project + ) + endif() +endmacro() + +macro(nebula_add_test) + cmake_parse_arguments( + nebula_test # prefix + "DISABLED;FUZZER" # + "NAME" # + "SOURCES;OBJECTS;LIBRARIES" # + ${ARGN} + ) + + nebula_add_executable( + NAME ${nebula_test_NAME} + SOURCES ${nebula_test_SOURCES} + OBJECTS ${nebula_test_OBJECTS} + LIBRARIES ${nebula_test_LIBRARIES} + ) + + if (${nebula_test_FUZZER}) + #Currently only Clang supports fuzz test + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set_target_properties(${nebula_test_NAME} PROPERTIES COMPILE_FLAGS "-g -fsanitize=fuzzer") + set_target_properties(${nebula_test_NAME} PROPERTIES LINK_FLAGS "-fsanitize=fuzzer") + endif() + elseif (NOT ${nebula_test_DISABLED}) + string(REGEX REPLACE "${CMAKE_SOURCE_DIR}/src/(.*)/test" "\\1" test_group ${CMAKE_CURRENT_SOURCE_DIR}) + add_test(NAME ${nebula_test_NAME} COMMAND ${nebula_test_NAME}) + set_tests_properties(${nebula_test_NAME} PROPERTIES LABELS ${test_group}) + # e.g. cmake -DNEBULA_ASAN_PRELOAD=/path/to/libasan.so + # or, cmake -DNEBULA_ASAN_PRELOAD=`/path/to/gcc --print-file-name=libasan.so` + if (NEBULA_ASAN_PRELOAD) + set_property( + TEST ${nebula_test_NAME} + PROPERTY ENVIRONMENT LD_PRELOAD=${NEBULA_ASAN_PRELOAD} + ) + endif() + endif() +endmacro() + +# A wrapper for target_link_libraries() +macro(nebula_link_libraries target) + target_link_libraries( + ${target} + ${ARGN} + folly + fmt + glog + gflags + boost_context + boost_system + boost_regex + boost_filesystem + boost_program_options + event + double-conversion + s2 + ${OPENSSL_SSL_LIBRARY} + ${OPENSSL_CRYPTO_LIBRARY} + ${COMPRESSION_LIBRARIES} + ${JEMALLOC_LIB} + ${LIBUNWIND_LIBRARIES} + resolv + dl + ${GETTIME_LIB} + ${libatomic_link_flags} + -pthread + z + ${COVERAGES} + ${Breakpad_LIBRARY} + ) +endmacro(nebula_link_libraries) + +function(nebula_add_subdirectory dir_name) + if ((NOT ENABLE_TESTING) AND (${dir_name} MATCHES test)) + add_subdirectory(${dir_name} EXCLUDE_FROM_ALL) + return() + endif() + add_subdirectory(${dir_name}) +endfunction() + +macro(nebula_add_exe_linker_flag flag) + string(FIND "${CMAKE_EXE_LINKER_FLAGS}" "${flag}" position) + if(${position} EQUAL -1) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${flag}") + endif() +endmacro() + +macro(nebula_remove_exe_linker_flag flag) + string(REPLACE "${flag}" "" output "${CMAKE_EXE_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${output}") +endmacro() + +macro(nebula_add_shared_linker_flag flag) + string(FIND "${CMAKE_SHARED_LINKER_FLAGS}" "${flag}" position) + if(${position} EQUAL -1) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${flag}") + endif() +endmacro() + +macro(nebula_remove_shared_linker_flag flag) + string(REPLACE "${flag}" "" output "${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${output}") +endmacro() + +function(nebula_string_rpad output width fill_char value) + string(LENGTH ${value} length) + math(EXPR delta "${width} - ${length}") + set(pads "") + if(${delta} GREATER 0) + math(EXPR delta "${delta} - 1") + foreach(loop_var RANGE ${delta}) + set(pads "${pads}${fill_char}") + endforeach() + endif() + set(${output} "${value}${pads}" PARENT_SCOPE) +endfunction() + +macro(print_option name) + get_property(helpstring CACHE ${name} PROPERTY HELPSTRING) + nebula_string_rpad(padded 32 " " "${name}") + string(FIND "${helpstring}" "No help" position) + if(helpstring AND ${position} EQUAL -1) + message(STATUS "${padded}: ${${name}} (${helpstring})") + else() + message(STATUS "${padded}: ${${name}}") + endif() +endmacro() + +macro(print_config name) + print_option(${name}) +endmacro() diff --git a/cmake/nebula/NebulaCustomTargets.cmake b/cmake/nebula/NebulaCustomTargets.cmake new file mode 100644 index 0000000..ee96190 --- /dev/null +++ b/cmake/nebula/NebulaCustomTargets.cmake @@ -0,0 +1,30 @@ +set(NEBULA_CLEAN_ALL_DEPS clean-interface clean-bin) + +add_custom_target( + clean-bin + COMMAND "rm" "-fr" "bin/*" +) + +add_custom_target( + clean-build + COMMAND ${CMAKE_MAKE_PROGRAM} clean + COMMAND "find" "." "-name" "Testing" "|" "xargs" "rm" "-fr" + DEPENDS ${NEBULA_CLEAN_ALL_DEPS} +) + +add_custom_target( + clean-all + COMMAND ${CMAKE_MAKE_PROGRAM} clean + COMMAND "find" "." "-name" "Testing" "|" "xargs" "rm" "-fr" + DEPENDS ${NEBULA_CLEAN_ALL_DEPS} +) + +add_custom_target( + distclean + COMMAND "find" "." "-name" "CMakeFiles" "|" "xargs" "rm" "-fr" + COMMAND "find" "." "-name" "CMakeCache.txt" "|" "xargs" "rm" "-f" + COMMAND "find" "." "-name" "cmake_install.cmake" "|" "xargs" "rm" "-f" + COMMAND "find" "." "-name" "CTestTestfile.cmake" "|" "xargs" "rm" "-f" + COMMAND "find" "." "-name" "Makefile" "|" "xargs" "rm" "-f" + DEPENDS clean-all +) diff --git a/cmake/nebula/PlatformCheck.cmake b/cmake/nebula/PlatformCheck.cmake new file mode 100644 index 0000000..bea9a56 --- /dev/null +++ b/cmake/nebula/PlatformCheck.cmake @@ -0,0 +1,3 @@ +if(${CMAKE_SYSTEM_NAME} MATCHES "(Darwin|FreeBSD|Windows)") + MESSAGE(FATAL_ERROR "Only Linux is supported") +endif () diff --git a/cmake/nebula/SanitizerConfig.cmake b/cmake/nebula/SanitizerConfig.cmake new file mode 100644 index 0000000..9f610e8 --- /dev/null +++ b/cmake/nebula/SanitizerConfig.cmake @@ -0,0 +1,63 @@ +if(ENABLE_ASAN OR ENABLE_UBSAN) + add_definitions(-DBUILT_WITH_SANITIZER) +endif() + +if(ENABLE_ASAN) + set(CMAKE_REQUIRED_FLAGS "-fsanitize=address") + check_cxx_compiler_flag("-fsanitize=address" ENABLE_ASAN_OK) + if (NOT ENABLE_ASAN_OK) + MESSAGE(FATAL_ERROR "The compiler does not support address sanitizer") + endif() + if(ENABLE_STATIC_ASAN AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + add_compile_options(-static-libasan) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") + endif() + add_compile_options(-fsanitize=address) + add_compile_options(-g) + add_compile_options(-fno-omit-frame-pointer) + add_definitions(-DENABLE_ASAN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") +endif() + +if(ENABLE_TSAN) + if (ENABLE_ASAN) + MESSAGE(FATAL_ERROR "ENABLE_TSAN cannot be combined with ENABLE_ASAN") + endif() + set(CMAKE_REQUIRED_FLAGS "-fsanitize=thread") + check_cxx_compiler_flag("-fsanitize=thread" ENABLE_TSAN_OK) + if (NOT ENABLE_TSAN_OK) + MESSAGE(FATAL_ERROR "The compiler does not support thread sanitizer") + endif() + set(ENV{TSAN_OPTIONS} "report_atomic_races=0") + add_compile_options(-fsanitize=thread) + add_compile_options(-g) + add_compile_options(-fno-omit-frame-pointer) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") +endif() + +if(ENABLE_UBSAN) + check_cxx_compiler_flag("-fsanitize=undefined -fno-sanitize=alignment" ENABLE_UBSAN_OK) + if (NOT ENABLE_UBSAN_OK) + MESSAGE(FATAL_ERROR "The compiler does not support Undefined Behavior Sanitizer") + endif() + if(ENABLE_STATIC_UBSAN AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + add_compile_options(-static-libubsan) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") + endif() + add_compile_options(-fsanitize=undefined -fno-sanitize=alignment) + add_compile_options(-fno-sanitize-recover=all) # Exit on failure + # TODO(dutor) Remove the following line when RTTI-enabled RocksDB is ready + add_compile_options(-fno-sanitize=vptr) + if(NOT ENABLE_ASAN) + add_compile_options(-g) + add_compile_options(-fno-omit-frame-pointer) + endif() + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") +endif() + +if(ENABLE_FUZZ_TEST) + check_cxx_compiler_flag("-fsanitize=fuzzer" ENABLE_FUZZ_OK) + if (NOT ENABLE_FUZZ_OK) + MESSAGE(FATAL_ERROR "The compiler does not support fuzz testing") + endif() +endif() diff --git a/cmake/nebula/ThirdPartyConfig.cmake b/cmake/nebula/ThirdPartyConfig.cmake new file mode 100644 index 0000000..03f5c58 --- /dev/null +++ b/cmake/nebula/ThirdPartyConfig.cmake @@ -0,0 +1,172 @@ +message(">>>> Configuring third party for '${PROJECT_NAME}' <<<<") +# The precedence to decide NEBULA_THIRDPARTY_ROOT is: +# 1. The path defined with CMake argument, i.e -DNEBULA_THIRDPARTY_ROOT=path +# 2. ${CMAKE_BINARY_DIR}/third-party/install, if exists +# 3. The path specified with environment variable NEBULA_THIRDPARTY_ROOT=path +# 4. /opt/vesoft/third-party, if exists +# 5. At last, one copy will be downloaded and installed to ${CMAKE_BINARY_DIR}/third-party/install + +set(NEBULA_THIRDPARTY_VERSION "3.3") + +if(${DISABLE_CXX11_ABI}) + SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party-98/install) + if(NOT EXISTS ${CMAKE_BINARY_DIR}/third-party-98/install) + message(STATUS "Install abi 98 third-party") + include(InstallThirdParty) + endif() +else() + if("${NEBULA_THIRDPARTY_ROOT}" STREQUAL "") + if(EXISTS ${CMAKE_BINARY_DIR}/third-party/install) + SET(NEBULA_THIRDPARTY_ROOT ${CMAKE_BINARY_DIR}/third-party/install) + elseif(NOT $ENV{NEBULA_THIRDPARTY_ROOT} STREQUAL "") + SET(NEBULA_THIRDPARTY_ROOT $ENV{NEBULA_THIRDPARTY_ROOT}) + elseif(EXISTS /opt/vesoft/third-party/${NEBULA_THIRDPARTY_VERSION}) + SET(NEBULA_THIRDPARTY_ROOT "/opt/vesoft/third-party/${NEBULA_THIRDPARTY_VERSION}") + else() + include(InstallThirdParty) + endif() + endif() +endif() + +if(NOT ${NEBULA_THIRDPARTY_ROOT} STREQUAL "") + print_config(NEBULA_THIRDPARTY_ROOT) + file(READ ${NEBULA_THIRDPARTY_ROOT}/version-info third_party_build_info) + message(STATUS "Build info of nebula third party:\n${third_party_build_info}") + list(INSERT CMAKE_INCLUDE_PATH 0 ${NEBULA_THIRDPARTY_ROOT}/include) + list(INSERT CMAKE_LIBRARY_PATH 0 ${NEBULA_THIRDPARTY_ROOT}/lib) + list(INSERT CMAKE_LIBRARY_PATH 0 ${NEBULA_THIRDPARTY_ROOT}/lib64) + list(INSERT CMAKE_PROGRAM_PATH 0 ${NEBULA_THIRDPARTY_ROOT}/bin) + include_directories(SYSTEM ${NEBULA_THIRDPARTY_ROOT}/include) + link_directories( + ${NEBULA_THIRDPARTY_ROOT}/lib + ${NEBULA_THIRDPARTY_ROOT}/lib64 + ) +endif() + +if(NOT ${NEBULA_OTHER_ROOT} STREQUAL "") + string(REPLACE ":" ";" DIR_LIST ${NEBULA_OTHER_ROOT}) + list(LENGTH DIR_LIST len) + foreach(DIR IN LISTS DIR_LIST ) + list(INSERT CMAKE_INCLUDE_PATH 0 ${DIR}/include) + list(INSERT CMAKE_LIBRARY_PATH 0 ${DIR}/lib) + list(INSERT CMAKE_PROGRAM_PATH 0 ${DIR}/bin) + include_directories(SYSTEM ${DIR}/include) + link_directories(${DIR}/lib) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${DIR}/lib") + endforeach() +endif() + +print_config(CMAKE_INCLUDE_PATH) +print_config(CMAKE_LIBRARY_PATH) +print_config(CMAKE_PROGRAM_PATH) + +execute_process( + COMMAND ldd --version + COMMAND head -1 + COMMAND cut -d ")" -f 2 + COMMAND cut -d " " -f 2 + OUTPUT_VARIABLE GLIBC_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE +) +print_config(GLIBC_VERSION) + +if (GLIBC_VERSION VERSION_LESS "2.17") + set(GETTIME_LIB rt) +else() + set(GETTIME_LIB) +endif() + +# Breakpad +if (ENABLE_BREAKPAD) + if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug" AND NOT ${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo") + MESSAGE(FATAL_ERROR "Breakpad need debug info.") + endif() +endif() +if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64") + set(ENABLE_BREAKPAD OFF) +endif() +if (ENABLE_BREAKPAD) + add_compile_options(-DENABLE_BREAKPAD=1) +endif() + +message("") + +find_package(Bzip2 REQUIRED) +find_package(DoubleConversion REQUIRED) +find_package(Fatal REQUIRED) +find_package(Fbthrift REQUIRED) +find_package(Folly REQUIRED) +find_package(Gflags REQUIRED) +find_package(Glog REQUIRED) +find_package(Googletest REQUIRED) +if(ENABLE_JEMALLOC) + find_package(Jemalloc REQUIRED) + add_definitions(-DENABLE_JEMALLOC) +endif() +find_package(Libevent REQUIRED) +find_package(Proxygen REQUIRED) +find_package(Rocksdb REQUIRED) +find_package(Snappy REQUIRED) +find_package(Wangle REQUIRED) +find_package(ZLIB REQUIRED) +find_package(Zstd REQUIRED) +find_package(OpenSSL REQUIRED) +find_package(Boost REQUIRED) +find_package(Libunwind REQUIRED) +find_package(BISON 3.0.5 REQUIRED) +include(MakeBisonRelocatable) +find_package(FLEX REQUIRED) +find_package(LibLZMA REQUIRED) +find_package(Fizz REQUIRED) +find_package(Sodium REQUIRED) +if (ENABLE_BREAKPAD) + find_package(Breakpad REQUIRED) +endif() + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${NEBULA_THIRDPARTY_ROOT}/lib") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${NEBULA_THIRDPARTY_ROOT}/lib64") + +# All thrift libraries +set(THRIFT_LIBRARIES + thriftcpp2 + async + thriftprotocol + transport + concurrency + thriftfrozen2 + thrift-core + rpcmetadata + thriftmetadata + wangle + fizz + sodium +) + +set(PROXYGEN_LIBRARIES + proxygenhttpserver + proxygen + wangle + fizz + sodium +) + +set(ROCKSDB_LIBRARIES ${Rocksdb_LIBRARY}) + +# All compression libraries +set(COMPRESSION_LIBRARIES bz2 snappy zstd z lz4) +if (LIBLZMA_FOUND) + include_directories(SYSTEM ${LIBLZMA_INCLUDE_DIRS}) + list(APPEND COMPRESSION_LIBRARIES ${LIBLZMA_LIBRARIES}) +endif() + +if (NOT ENABLE_JEMALLOC OR ENABLE_ASAN OR ENABLE_UBSAN) + set(JEMALLOC_LIB ) +else() + set(JEMALLOC_LIB jemalloc) +endif() + +if (Breakpad_FOUND) + include_directories(AFTER SYSTEM ${Breakpad_INCLUDE_DIR}/breakpad) +endif() + +message(">>>> Configuring third party for '${PROJECT_NAME}' done <<<<") diff --git a/cmake/nebula/TimeTrace.cmake b/cmake/nebula/TimeTrace.cmake new file mode 100644 index 0000000..d209359 --- /dev/null +++ b/cmake/nebula/TimeTrace.cmake @@ -0,0 +1,6 @@ +if(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + option(ENABLE_BUILD_WITH_TIME_TRACE "Enable -ftime-trace to generate time tracing .json files on clang" OFF) + if(ENABLE_BUILD_WITH_TIME_TRACE) + add_compile_options(-ftime-trace) + endif() +endif() diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..9ad5a22 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,13 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +# For more configuration details: +# https://docs.codecov.io/docs/codecov-yaml + +# validate the configuration: +# curl -X POST --data-binary @codecov.yml https://codecov.io/validate + +codecov: + allow_pseudo_compare: True + allow_coverage_offsets: True diff --git a/conf/CMakeLists.txt b/conf/CMakeLists.txt new file mode 100644 index 0000000..0ec9079 --- /dev/null +++ b/conf/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +# These configuration files are for reference to generate your own customized ones. +# Thus, they are installed as read-only, even for the owner. + +if(NOT ENABLE_STANDALONE_VERSION) + +install( + FILES + nebula-graphd.conf.default + nebula-graphd.conf.production + PERMISSIONS + OWNER_READ + GROUP_READ + WORLD_READ + DESTINATION + etc + COMPONENT + graph + ) + +install( + FILES + nebula-metad.conf.default + nebula-metad.conf.production + PERMISSIONS + OWNER_READ + GROUP_READ + WORLD_READ + DESTINATION + etc + COMPONENT + meta + ) + +install( + FILES + nebula-storaged.conf.default + nebula-storaged.conf.production + nebula-storaged-listener.conf.production + PERMISSIONS + OWNER_READ + GROUP_READ + WORLD_READ + DESTINATION + etc + COMPONENT + storage + ) + +else() + +install( + FILES + nebula-standalone.conf.default + PERMISSIONS + OWNER_READ + GROUP_READ + WORLD_READ + DESTINATION + etc + COMPONENT + graph + ) + +endif() diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default new file mode 100644 index 0000000..d57d3f8 --- /dev/null +++ b/conf/nebula-graphd.conf.default @@ -0,0 +1,134 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-graphd.pid +# Whether to enable optimizer +--enable_optimizer=true +# The default charset when a space is created +--default_charset=utf8 +# The default collate when a space is created +--default_collate=utf8_bin +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=graphd-stdout.log +--stderr_log_file=graphd-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# wether logging files' name contain time stamp. +--timestamp_in_logfile_name=true +########## query ########## +# Whether to treat partial success as an error. +# This flag is only used for Read-only access, and Modify access always treats partial success as an error. +--accept_partial_success=false +# Maximum sentence length, unit byte +--max_allowed_query_size=4194304 + +########## networking ########## +# Comma separated Meta Server Addresses +--meta_server_addrs=127.0.0.1:9559 +# Local IP used to identify the nebula-graphd process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=127.0.0.1 +# Network device to listen on +--listen_netdev=any +# Port to listen on +--port=9669 +# To turn on SO_REUSEPORT or not +--reuse_port=false +# Backlog of the listen socket, adjust this together with net.core.somaxconn +--listen_backlog=1024 +# The number of seconds Nebula service waits before closing the idle connections +--client_idle_timeout_secs=28800 +# The number of seconds before idle sessions expire +# The range should be in [1, 604800] +--session_idle_timeout_secs=28800 +# The number of threads to accept incoming connections +--num_accept_threads=1 +# The number of networking IO threads, 0 for # of CPU cores +--num_netio_threads=0 +# Max active connections for all networking threads. 0 means no limit. +# Max connections for each networking thread = num_max_connections / num_netio_threads +--num_max_connections=0 +# The number of threads to execute user queries, 0 for # of CPU cores +--num_worker_threads=0 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19669 +# storage client timeout +--storage_client_timeout_ms=60000 +# slow query threshold in us +--slow_query_threshold_us=200000 +# Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file +--ws_meta_http_port=19559 + +########## authentication ########## +# Enable authorization +--enable_authorize=false +# User login authentication type, password for nebula authentication, ldap for ldap authentication, cloud for cloud authentication +--auth_type=password + +########## memory ########## +# System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0 +--system_memory_high_watermark_ratio=0.8 + +########## metrics ########## +--enable_space_level_metrics=false + +########## experimental feature ########## +# if use experimental features +--enable_experimental_feature=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true + +# enable udf, written in c++ only for now +--enable_udf=true + +# set the directory where the .so files of udf are stored, when enable_udf is true +--udf_path=/home/nebula/dev/nebula/udf/ + +########## session ########## +# Maximum number of sessions that can be created per IP and per user +--max_sessions_per_ip_per_user=300 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 + +########## performance optimization ########## +# The max job size in multi job mode +--max_job_size=1 +# The min batch size for handling dataset in multi job mode, only enabled when max_job_size is greater than 1 +--min_batch_size=8192 +# if true, return directly without go through RPC +--optimize_appendvertices=false +# number of paths constructed by each thread +--path_batch_size=10000 diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production new file mode 100644 index 0000000..f4e9cd2 --- /dev/null +++ b/conf/nebula-graphd.conf.production @@ -0,0 +1,127 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-graphd.pid +# Whether to enable optimizer +--enable_optimizer=true +# Heartbeat interval of communication between meta client and graphd service +--heartbeat_interval_secs=10 +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=graphd-stdout.log +--stderr_log_file=graphd-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# wether logging files' name contain timestamp +--timestamp_in_logfile_name=true + +########## query ########## +# Whether to treat partial success as an error. +# This flag is only used for Read-only access, and Modify access always treats partial success as an error. +--accept_partial_success=false +# Maximum sentence length, unit byte +--max_allowed_query_size=4194304 + +########## networking ########## +# Comma separated Meta Server Addresses +--meta_server_addrs=127.0.0.1:9559 +# Local IP used to identify the nebula-graphd process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=127.0.0.1 +# Network device to listen on +--listen_netdev=any +# Port to listen on +--port=9669 +# To turn on SO_REUSEPORT or not +--reuse_port=false +# Backlog of the listen socket, adjust this together with net.core.somaxconn +--listen_backlog=1024 +# The number of seconds Nebula service waits before closing the idle connections +--client_idle_timeout_secs=28800 +# The number of seconds before idle sessions expire +# The range should be in [1, 604800] +--session_idle_timeout_secs=28800 +# The number of threads to accept incoming connections +--num_accept_threads=1 +# The number of networking IO threads, 0 for # of CPU cores +--num_netio_threads=0 +# Max active connections for all networking threads. 0 means no limit. +# Max connections for each networking thread = num_max_connections / num_netio_threads +--num_max_connections=0 +# The number of threads to execute user queries, 0 for # of CPU cores +--num_worker_threads=0 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19669 +# storage client timeout +--storage_client_timeout_ms=60000 +# slow query threshold in us +--slow_query_threshold_us=200000 +# Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file +--ws_meta_http_port=19559 + +########## authentication ########## +# Enable authorization +--enable_authorize=false +# User login authentication type, password for nebula authentication, ldap for ldap authentication, cloud for cloud authentication +--auth_type=password + +########## memory ########## +# System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0 +--system_memory_high_watermark_ratio=0.8 + +########## metrics ########## +--enable_space_level_metrics=false + +########## experimental feature ########## +# if use experimental features +--enable_experimental_feature=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true + +########## session ########## +# Maximum number of sessions that can be created per IP and per user +--max_sessions_per_ip_per_user=300 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 + +########## performance optimization ########## +# The max job size in multi job mode +--max_job_size=1 +# The min batch size for handling dataset in multi job mode, only enabled when max_job_size is greater than 1 +--min_batch_size=8192 +# if true, return directly without go through RPC +--optimize_appendvertices=false +# number of paths constructed by each thread +--path_batch_size=10000 diff --git a/conf/nebula-metad.conf.default b/conf/nebula-metad.conf.default new file mode 100644 index 0000000..89b594f --- /dev/null +++ b/conf/nebula-metad.conf.default @@ -0,0 +1,53 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-metad.pid + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=metad-stdout.log +--stderr_log_file=metad-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# wether logging files' name contain time stamp, If Using logrotate to rotate logging files, than should set it to true. +--timestamp_in_logfile_name=true + +########## networking ########## +# Comma separated Meta Server addresses +--meta_server_addrs=127.0.0.1:9559 +# Local IP used to identify the nebula-metad process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=127.0.0.1 +# Meta daemon listening port +--port=9559 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19559 +# Port to listen on Storage with HTTP protocol, it corresponds to ws_http_port in storage's configuration file +--ws_storage_http_port=19779 + +########## storage ########## +# Root data path, here should be only single path for metad +--data_path=data/meta + +########## Misc ######### +# The default number of parts when a space is created +--default_parts_num=100 +# The default replica factor when a space is created +--default_replica_factor=1 + +--heartbeat_interval_secs=10 +--agent_heartbeat_interval_secs=60 diff --git a/conf/nebula-metad.conf.production b/conf/nebula-metad.conf.production new file mode 100644 index 0000000..3ea93ff --- /dev/null +++ b/conf/nebula-metad.conf.production @@ -0,0 +1,56 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-metad.pid + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=metad-stdout.log +--stderr_log_file=metad-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# wether logging files' name contain time stamp. +--timestamp_in_logfile_name=true + +########## networking ########## +# Comma separated Meta Server addresses +--meta_server_addrs=192.168.2.1:9559 +# Local IP used to identify the nebula-metad process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=192.168.2.1 +# Meta daemon listening port +--port=9559 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19559 +# Port to listen on Storage with HTTP protocol, it corresponds to ws_http_port in storage's configuration file +--ws_storage_http_port=19779 + +########## storage ########## +# Root data path, here should be only single path for metad +--data_path=data/meta + +########## Misc ######### +# The default number of parts when a space is created +--default_parts_num=100 +# The default replica factor when a space is created +--default_replica_factor=1 + +--heartbeat_interval_secs=10 +--agent_heartbeat_interval_secs=60 + +############## rocksdb Options ############## +--rocksdb_wal_sync=true diff --git a/conf/nebula-standalone.conf.default b/conf/nebula-standalone.conf.default new file mode 100644 index 0000000..e5f7a44 --- /dev/null +++ b/conf/nebula-standalone.conf.default @@ -0,0 +1,163 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-standalone.pid +# Whether to enable optimizer +--enable_optimizer=true +# The default charset when a space is created +--default_charset=utf8 +# The defaule collate when a space is created +--default_collate=utf8_bin +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=standalone-stdout.log +--stderr_log_file=standalone-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 + +########## query ########## +# Whether to treat partial success as an error. +# This flag is only used for Read-only access, and Modify access always treats partial success as an error. +--accept_partial_success=false +# Maximum sentence length, unit byte +--max_allowed_query_size=4194304 + +########## networking ########## +# Comma separated Meta Server Addresses +--meta_server_addrs=127.0.0.1:9559 +# Local IP used to identify the nebula-graphd process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=127.0.0.1 +# Network device to listen on +--listen_netdev=any +# Port to listen on +--port=9669 +--meta_port=9559 +--storage_port=9779 +# To turn on SO_REUSEPORT or not +--reuse_port=false +# Backlog of the listen socket, adjust this together with net.core.somaxconn +--listen_backlog=1024 +# The number of seconds Nebula service waits before closing the idle connections +--client_idle_timeout_secs=28800 +# The number of seconds before idle sessions expire +# The range should be in [1, 604800] +--session_idle_timeout_secs=28800 +# The number of threads to accept incoming connections +--num_accept_threads=1 +# The number of networking IO threads, 0 for # of CPU cores +--num_netio_threads=0 +# The number of threads to execute user queries, 0 for # of CPU cores +--num_worker_threads=0 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19669 +# storage client timeout +--storage_client_timeout_ms=60000 +# Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file +--ws_meta_http_port=19559 +# HTTP service port +--ws_storage_http_port=19779 +# heartbeat with meta service +--heartbeat_interval_secs=10 + + +########## authentication ########## +# Enable authorization +--enable_authorize=false +# User login authentication type, password for nebula authentication, ldap for ldap authentication, cloud for cloud authentication +--auth_type=password + +########## memory ########## +# System memory high watermark ratio +--system_memory_high_watermark_ratio=0.8 + +########## experimental feature ########## +# if use experimental features +--enable_experimental_feature=false + +# if use balance data feature, only work if enable_experimental_feature is true +--enable_data_balance=true + +######### Raft ######### +# Raft election timeout +--raft_heartbeat_interval_secs=30 +# RPC timeout for raft client (ms) +--raft_rpc_timeout_ms=500 +## recycle Raft WAL +--wal_ttl=14400 + +########## Disk ########## +# Root data path. Split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/ +# One path per Rocksdb instance. +--data_path=data/storage + +# Minimum reserved bytes of each data path +--minimum_reserved_bytes=268435456 + +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb + +# Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd +# For the sake of binary compatibility, the default value is snappy. +# Recommend to use: +# * lz4 to gain more CPU performance, with the same compression ratio with snappy +# * zstd to occupy less disk space +# * lz4hc for the read-heavy write-light scenario +--rocksdb_compression=lz4 + +# Set different compressions for different levels +# For example, if --rocksdb_compression is snappy, +# "no:no:lz4:lz4::zstd" is identical to "no:no:lz4:lz4:snappy:zstd:snappy" +# In order to disable compression for level 0/1, set it to "no:no" +--rocksdb_compression_per_level= + +# Whether or not to enable rocksdb's statistics, disabled by default +--enable_rocksdb_statistics=false + +# Statslevel used by rocksdb to collection statistics, optional values are +# * kExceptHistogramOrTimers, disable timer stats, and skip histogram stats +# * kExceptTimers, Skip timer stats +# * kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression. +# * kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock. +# * kAll, Collect all stats +--rocksdb_stats_level=kExceptHistogramOrTimers + +# Whether or not to enable rocksdb's prefix bloom filter, enabled by default. +--enable_rocksdb_prefix_filtering=true +# Whether or not to enable rocksdb's whole key bloom filter, disabled by default. +--enable_rocksdb_whole_key_filtering=false + +############## rocksdb Options ############## +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={"write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={"block_size":"8192"} + +############## meta Options ############## +--meta_data_path=data/meta +--default_replica_factor=1 +--default_parts_num=100 diff --git a/conf/nebula-storaged-listener.conf.default b/conf/nebula-storaged-listener.conf.default new file mode 100644 index 0000000..86d74d3 --- /dev/null +++ b/conf/nebula-storaged-listener.conf.default @@ -0,0 +1,58 @@ +########## nebula-storaged-listener ########### +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-storaged-listener.pid +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs_storaged_listener +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=storaged-listener-stdout.log +--stderr_log_file=storaged-listener-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# Wether logging files' name contain timestamp. +--timestamp_in_logfile_name=true + +########## networking ########## +# Meta server address +--meta_server_addrs=127.0.0.1:9559 +# Local ip +--local_ip=127.0.0.1 +# Storage daemon listening port +--port=9789 +# HTTP service ip +--ws_ip=127.0.0.1 +# HTTP service port +--ws_http_port=19789 +# heartbeat with meta service +--heartbeat_interval_secs=10 + +########## storage ########## +# Listener wal directory. only one path is allowed. +--listener_path=data/listener +# This parameter can be ignored for compatibility. let's fill A default value of "data" +--data_path=data +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple diff --git a/conf/nebula-storaged-listener.conf.production b/conf/nebula-storaged-listener.conf.production new file mode 100644 index 0000000..72467e7 --- /dev/null +++ b/conf/nebula-storaged-listener.conf.production @@ -0,0 +1,58 @@ +########## nebula-storaged-listener ########### +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-storaged-listener.pid +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs_storaged_listener +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=storaged-listener-stdout.log +--stderr_log_file=storaged-listener-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# Wether logging files' name contain timestamp. +--timestamp_in_logfile_name=true + +########## networking ########## +# Meta server address +--meta_server_addrs=192.168.2.1:9559 +# Local ip +--local_ip=192.168.2.4 +# Storage daemon listening port +--port=9789 +# HTTP service ip +--ws_ip=192.168.2.4 +# HTTP service port +--ws_http_port=19789 +# heartbeat with meta service +--heartbeat_interval_secs=10 + +########## storage ########## +# Listener wal directory. only one path is allowed. +--listener_path=data/listener +# This parameter can be ignored for compatibility. let's fill A default value of "data" +--data_path=data +# The type of part manager, [memory | meta] +--part_man_type=memory +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb +# The type of part, `simple', `consensus'... +--part_type=simple diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default new file mode 100644 index 0000000..0d29f55 --- /dev/null +++ b/conf/nebula-storaged.conf.default @@ -0,0 +1,143 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-storaged.pid +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=storaged-stdout.log +--stderr_log_file=storaged-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# Wether logging files' name contain time stamp. +--timestamp_in_logfile_name=true + +########## networking ########## +# Comma separated Meta server addresses +--meta_server_addrs=127.0.0.1:9559 +# Local IP used to identify the nebula-storaged process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=127.0.0.1 +# Storage daemon listening port +--port=9779 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19779 +# heartbeat with meta service +--heartbeat_interval_secs=10 + +######### Raft ######### +# Raft election timeout +--raft_heartbeat_interval_secs=30 +# RPC timeout for raft client (ms) +--raft_rpc_timeout_ms=500 +## recycle Raft WAL +--wal_ttl=14400 + +########## Disk ########## +# Root data path. Split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/ +# One path per Rocksdb instance. +--data_path=data/storage + +# Minimum reserved bytes of each data path +--minimum_reserved_bytes=268435456 + +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. +# The unit is MB. +--rocksdb_block_cache=4 +# The type of storage engine, `rocksdb', `memory', etc. +--engine_type=rocksdb + +# Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd +# For the sake of binary compatibility, the default value is snappy. +# Recommend to use: +# * lz4 to gain more CPU performance, with the same compression ratio with snappy +# * zstd to occupy less disk space +# * lz4hc for the read-heavy write-light scenario +--rocksdb_compression=lz4 + +# Set different compressions for different levels +# For example, if --rocksdb_compression is snappy, +# "no:no:lz4:lz4::zstd" is identical to "no:no:lz4:lz4:snappy:zstd:snappy" +# In order to disable compression for level 0/1, set it to "no:no" +--rocksdb_compression_per_level= + +# Whether or not to enable rocksdb's statistics, disabled by default +--enable_rocksdb_statistics=false + +# Statslevel used by rocksdb to collection statistics, optional values are +# * kExceptHistogramOrTimers, disable timer stats, and skip histogram stats +# * kExceptTimers, Skip timer stats +# * kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression. +# * kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock. +# * kAll, Collect all stats +--rocksdb_stats_level=kExceptHistogramOrTimers + +# Whether or not to enable rocksdb's prefix bloom filter, enabled by default. +--enable_rocksdb_prefix_filtering=true +# Whether or not to enable rocksdb's whole key bloom filter, disabled by default. +--enable_rocksdb_whole_key_filtering=false + +############## rocksdb Options ############## +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={"write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={"block_size":"8192"} + +############### misc #################### +# Whether turn on query in multiple thread +--query_concurrently=true +# Whether remove outdated space data +--auto_remove_invalid_space=true +# Network IO threads number +--num_io_threads=16 +# Max active connections for all networking threads. 0 means no limit. +# Max connections for each networking thread = num_max_connections / num_netio_threads +--num_max_connections=0 +# Worker threads number to handle request +--num_worker_threads=32 +# Maximum subtasks to run admin jobs concurrently +--max_concurrent_subtasks=10 +# The rate limit in bytes when leader synchronizes snapshot data +--snapshot_part_rate_limit=10485760 +# The amount of data sent in each batch when leader synchronizes snapshot data +--snapshot_batch_size=1048576 +# The rate limit in bytes when leader synchronizes rebuilding index +--rebuild_index_part_rate_limit=4194304 +# The amount of data sent in each batch when leader synchronizes rebuilding index +--rebuild_index_batch_size=1048576 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production new file mode 100644 index 0000000..f62b285 --- /dev/null +++ b/conf/nebula-storaged.conf.production @@ -0,0 +1,144 @@ +########## basics ########## +# Whether to run as a daemon process +--daemonize=true +# The file to host the process id +--pid_file=pids/nebula-storaged-listener.pid +# Whether to use the configuration obtained from the configuration file +--local_config=true + +########## logging ########## +# The directory to host logging files +--log_dir=logs +# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively +--minloglevel=0 +# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging +--v=0 +# Maximum seconds to buffer the log messages +--logbufsecs=0 +# Whether to redirect stdout and stderr to separate output files +--redirect_stdout=true +# Destination filename of stdout and stderr, which will also reside in log_dir. +--stdout_log_file=storaged-listener-stdout.log +--stderr_log_file=storaged-listener-stderr.log +# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. +--stderrthreshold=3 +# Wether logging files' name contain timestamp. +--timestamp_in_logfile_name=true + +########## networking ########## +# Comma separated Meta server addresses +--meta_server_addrs=192.168.2.1:45500 +# Local IP used to identify the nebula-storaged process. +# Change it to an address other than loopback if the service is distributed or +# will be accessed remotely. +--local_ip=192.168.2.3 +# Storage daemon listening port +--port=9779 +# HTTP service ip +--ws_ip=0.0.0.0 +# HTTP service port +--ws_http_port=19779 +# heartbeat with meta service +--heartbeat_interval_secs=10 + +######### Raft ######### +# Raft election timeout +--raft_heartbeat_interval_secs=30 +# RPC timeout for raft client (ms) +--raft_rpc_timeout_ms=500 +## recycle Raft WAL +--wal_ttl=14400 + +########## Disk ########## +# Root data path. split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/ +# One path per Rocksdb instance. +--data_path=data/storage + +# Minimum reserved bytes of each data path +--minimum_reserved_bytes=268435456 + +# The default reserved bytes for one batch operation +--rocksdb_batch_size=4096 +# The default block cache size used in BlockBasedTable. (MB) +# recommend: 1/3 of all memory +--rocksdb_block_cache=4096 +# Disable page cache to better control memory used by rocksdb. +# Caution: Make sure to allocate enough block cache if disabling page cache! +--disable_page_cache=false + +# Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd +# For the sake of binary compatibility, the default value is snappy. +# Recommend to use: +# * lz4 to gain more CPU performance, with the same compression ratio with snappy +# * zstd to occupy less disk space +# * lz4hc for the read-heavy write-light scenario +--rocksdb_compression=lz4 + +# Set different compressions for different levels +# For example, if --rocksdb_compression is snappy, +# "no:no:lz4:lz4::zstd" is identical to "no:no:lz4:lz4:snappy:zstd:snappy" +# In order to disable compression for level 0/1, set it to "no:no" +--rocksdb_compression_per_level= + +############## rocksdb Options ############## +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={"max_subcompactions":"4","max_background_jobs":"4","skip_checking_sst_file_sizes_on_db_open":"true"} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={"disable_auto_compactions":"false","write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={"block_size":"8192"} + +# Whether or not to enable rocksdb's statistics, disabled by default +--enable_rocksdb_statistics=false + +# Statslevel used by rocksdb to collection statistics, optional values are +# * kExceptHistogramOrTimers, disable timer stats, and skip histogram stats +# * kExceptTimers, Skip timer stats +# * kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression. +# * kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock. +# * kAll, Collect all stats +--rocksdb_stats_level=kExceptHistogramOrTimers + +# Whether or not to enable rocksdb's prefix bloom filter, enabled by default. +--enable_rocksdb_prefix_filtering=true +# Whether or not to enable rocksdb's whole key bloom filter, disabled by default. +--enable_rocksdb_whole_key_filtering=false + +############### misc #################### +# Whether turn on query in multiple thread +--query_concurrently=true +# Whether remove outdated space data +--auto_remove_invalid_space=true +# Network IO threads number +--num_io_threads=16 +# Max active connections for all networking threads. 0 means no limit. +# Max connections for each networking thread = num_max_connections / num_netio_threads +--num_max_connections=0 +# Worker threads number to handle request +--num_worker_threads=32 +# Maximum subtasks to run admin jobs concurrently +--max_concurrent_subtasks=10 +# The rate limit in bytes when leader synchronizes snapshot data +--snapshot_part_rate_limit=10485760 +# The amount of data sent in each batch when leader synchronizes snapshot data +--snapshot_batch_size=1048576 +# The rate limit in bytes when leader synchronizes rebuilding index +--rebuild_index_part_rate_limit=4194304 +# The amount of data sent in each batch when leader synchronizes rebuilding index +--rebuild_index_batch_size=1048576 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 diff --git a/conf/tuned/README.md b/conf/tuned/README.md new file mode 100644 index 0000000..dfcc0bc --- /dev/null +++ b/conf/tuned/README.md @@ -0,0 +1,8 @@ +# Summary + +These are tuned profile to configure the system to optimize for the NebulaGraph service. + +Follow below steps to utilize: + * Install the tuned service if absent, and enable it with `systemctl`. + * Copy the __nebula__ directory into `/etc/tuned`. + * Execute `tuned-adm profile nebula` to activate the profile. diff --git a/conf/tuned/nebula/tuned.conf b/conf/tuned/nebula/tuned.conf new file mode 100644 index 0000000..8951652 --- /dev/null +++ b/conf/tuned/nebula/tuned.conf @@ -0,0 +1,32 @@ +[main] +summary=Optimize for NebulaGraph DBMS +include=latency-performance + + +[vm] +transparent_hugepages=never + + +[sysctl] +kernel.core_pattern=core +kernel.core_uses_pid=1 +kernel.numa_balancing=0 + +vm.swappiness=0 +vm.oom_dump_tasks=1 +# min_free_kbytes is suggested to set to approximately 2% of the total memory. +# 1GB at least and 5GB at most. +vm.min_free_kbytes=5242880 +vm.max_map_count=131060 +vm.dirty_background_ratio = 3 +vm.dirty_ratio = 20 +vm.dirty_expire_centisecs = 500 +vm.dirty_writeback_centisecs = 100 + +net.core.busy_read=50 +net.core.busy_poll=50 +net.core.somaxconn=4096 +net.ipv4.tcp_max_syn_backlog=4096 +net.core.netdev_max_backlog=10240 +net.ipv4.tcp_fastopen=3 +net.ipv4.tcp_slow_start_after_idle=0 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..ea2e33a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,66 @@ +FROM vesoft/nebula-dev:centos7 as builder + +ENV BUILD_IN_DOCKER TRUE +ARG VERSION= + +COPY . /home/nebula/BUILD + +RUN cd /home/nebula/BUILD/package \ + && ./package.sh -v "${VERSION}" + +FROM centos:7 as graphd + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-graph.rpm /usr/local/nebula/nebula-graphd.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9669 19669 19670 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"] + +FROM centos:7 as metad + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-meta.rpm /usr/local/nebula/nebula-metad.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9559 9560 19559 19560 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"] + +FROM centos:7 as storaged + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-storage.rpm /usr/local/nebula/nebula-storaged.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9777 9778 9779 9780 19779 19780 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"] + +FROM centos:7 as tools + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-tool.rpm /usr/local/nebula/nebula-tool.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && rm -rf *.rpm + +# default entrypoint +ENTRYPOINT ["/usr/local/nebula/bin/db_dump"] diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd new file mode 100644 index 0000000..80c2f1a --- /dev/null +++ b/docker/Dockerfile.graphd @@ -0,0 +1,24 @@ +FROM vesoft/nebula-dev:centos7 as builder + +ENV BUILD_IN_DOCKER TRUE +ARG VERSION= + +COPY . /home/nebula/BUILD + +RUN cd /home/nebula/BUILD/package \ + && ./package.sh -v "${VERSION}" + +FROM centos:7 + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-graph.rpm /usr/local/nebula/nebula-graphd.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9669 19669 19670 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"] diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad new file mode 100644 index 0000000..ba13f9e --- /dev/null +++ b/docker/Dockerfile.metad @@ -0,0 +1,24 @@ +FROM vesoft/nebula-dev:centos7 as builder + +ENV BUILD_IN_DOCKER TRUE +ARG VERSION= + +COPY . /home/nebula/BUILD + +RUN cd /home/nebula/BUILD/package \ + && ./package.sh -v "${VERSION}" + +FROM centos:7 + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-meta.rpm /usr/local/nebula/nebula-metad.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9559 9560 19559 19560 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"] diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged new file mode 100644 index 0000000..da16bfc --- /dev/null +++ b/docker/Dockerfile.storaged @@ -0,0 +1,24 @@ +FROM vesoft/nebula-dev:centos7 as builder + +ENV BUILD_IN_DOCKER TRUE +ARG VERSION= + +COPY . /home/nebula/BUILD + +RUN cd /home/nebula/BUILD/package \ + && ./package.sh -v "${VERSION}" + +FROM centos:7 + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-storage.rpm /usr/local/nebula/nebula-storaged.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && mkdir -p ./{logs,data,pids} \ + && rm -rf *.rpm + +EXPOSE 9777 9778 9779 9780 19779 19780 + +ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"] diff --git a/docker/Dockerfile.tools b/docker/Dockerfile.tools new file mode 100644 index 0000000..fe361bc --- /dev/null +++ b/docker/Dockerfile.tools @@ -0,0 +1,21 @@ +FROM vesoft/nebula-dev:centos7 as builder + +ENV BUILD_IN_DOCKER TRUE +ARG VERSION= + +COPY . /home/nebula/BUILD + +RUN cd /home/nebula/BUILD/package \ + && ./package.sh -v "${VERSION}" + +FROM centos:7 + +COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-tool.rpm /usr/local/nebula/nebula-tool.rpm + +WORKDIR /usr/local/nebula + +RUN rpm -ivh *.rpm \ + && rm -rf *.rpm + +# default entrypoint +ENTRYPOINT ["/usr/local/nebula/bin/db_dump"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..6cd7391 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,8 @@ +# Docker Files for NebulaGraph Services + +Following docker images will be ready in production. + +- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd` +- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad` +- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged` +- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader diff --git a/docs/rfcs/0000-template.md b/docs/rfcs/0000-template.md new file mode 100644 index 0000000..b80bb29 --- /dev/null +++ b/docs/rfcs/0000-template.md @@ -0,0 +1,64 @@ +--- +name: Request for Comments +about: Detail an idea for this project + +--- + + + + +# Summary + +The overview explanation of the feature. + +# Motivation + +Why are we doing this? What use cases does it support? What is the expected outcome? + +# Usage explanation + +Explain usage of the proposal as if it was already introduced in the project. That generally means: + +- Introducing new named concepts, syntax or configuration. +- Explaining the feature largely in terms of examples. If applicable, provide sample usage, output or error messages. +- Explaining the compatibility in several levels like query language, RPC structure and storage format etc. + +# Design explanation + +This is the technical section of the RFC. Explain the design in sufficient detail that: + +- Detail how the feature will be implemented. +- From syntax, AST, planning, executing and even deep into storage etc. + +# Rationale and alternatives + +- Explain the ultimate purpose of the proposal. Why is this design the best in the space of possible designs? +- What other designs have been considered and what is the rationale for not choosing them? + +# Drawbacks + +Why should we *not* accept this? List the negative affection or shortage if exists. + +# Prior art + +Discuss prior art, both the good and the bad, in relation to this proposal. +A few examples of what this can include are: + +- For practical proposals: Does this feature exist in other database and what experience have their community had? +- Papers: Are there any published papers or great posts that discuss this? If you have some relevant papers to refer to, this can serve as a more detailed theoretical background. + +The proposal won't be accepted for they are in other projects, but this will show us more productive meaning. + +# Unresolved questions + +- What parts of the design do you expect to resolve through the RFC process before this gets accepted? +- What parts of the design do you expect *can't* to resolve eventually in this RFC? +- What related issues do you consider out of scope for this RFC but could be addressed in the future independently of the solution that comes out of this RFC? + +# Future possibilities + +List the expected extensions of this proposal in future, and why better not include in current stage. + +But all content in this section has no effort to determine whether we accept this proposal, in fact, it's just additional notes or even brainstorming. diff --git a/package/package.sh b/package/package.sh new file mode 100644 index 0000000..7a006ee --- /dev/null +++ b/package/package.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# +# Package nebula as deb/rpm package +# +# introduce the args +# -v: The version of package, the version should be match tag name, default value is null +# -n: Package to one or multi-packages, `ON` means one package, `OFF` means multi packages, default value is `ON` +# -s: Whether to strip the package, default value is `FALSE` +# -d: Whether to enable sanitizer, default OFF +# -t: Build type, default Release +# -j: Number of threads, default $(nproc) +# -r: Whether to enable compressed debug info, default ON +# -p: Whether to dump the symbols from binary by dump_syms +# -c: Whether to enable console building, default ON +# -k: Whether to enable breakpad, default OFF +# +# usage: ./package.sh -v -n -s -c +# + +set -e + +usage="Usage: ${0} -v -n -s -g -j -t " +project_dir="$(cd "$(dirname "$0")" && pwd)/.." +build_dir=${project_dir}/pkg-build +install_prefix=/usr/local/nebula + +BUILD_IN_DOCKER=${BUILD_IN_DOCKER:-FALSE} + +# default variables when building release. +function _default_release_variables { + version="" + package_one="ON" + strip_enable="TRUE" + enablesanitizer="OFF" + static_sanitizer="OFF" + build_type="RelWithDebInfo" + build_console="OFF" + jobs=$(nproc) + enable_compressed_debug_info="OFF" + dump_symbols="ON" + dump_syms_tool_dir= + system_name= + enable_breakpad="ON" + +} + +# change default variables. e.g. build release in docker +function _extra_release_variables { + if [[ $BUILD_IN_DOCKER == TRUE ]]; then + package_one="OFF" + enable_compressed_debug_info="ON" + dump_symbols="OFF" + fi +} + +_default_release_variables +_extra_release_variables + +while getopts v:n:s:d:t:r:p:j:c:k: opt; +do + case $opt in + v) + version=$OPTARG + ;; + n) + package_one=$OPTARG + ;; + s) + strip_enable=$OPTARG + ;; + d) + enablesanitizer="ON" + if [ "$OPTARG" == "static" ]; then + static_sanitizer="ON" + fi + build_type="RelWithDebInfo" + ;; + t) + build_type=$OPTARG + ;; + c) + build_console=$OPTARG + ;; + j) + jobs=$OPTARG + ;; + r) + enable_compressed_debug_info=$OPTARG + ;; + p) + dump_symbols=$OPTARG + ;; + k) + enable_breakpad=$OPTARG + ;; + ?) + echo "Invalid option, use default arguments" + ;; + esac +done + +# version is null, get from tag name +[[ -z $version ]] && version=$(git describe --exact-match --abbrev=0 --tags | sed 's/^v//') +# version is null, use UTC date as version +[[ -z $version ]] && version=$(date -u +%Y.%m.%d)-nightly + +if [[ -z $version ]]; then + echo "version is null, exit" + echo ${usage} + exit 1 +fi + + +if [[ $strip_enable != TRUE ]] && [[ $strip_enable != FALSE ]]; then + echo "strip enable is wrong, exit" + echo ${usage} + exit 1 +fi + +cat << EOF +Configuration for this shell: +version: $version +strip_enable: $strip_enable +enablesanitizer: $enablesanitizer +static_sanitizer: $static_sanitizer +build_type: $build_type +build_console: $build_console +branch: $branch +enable_compressed_debug_info: $enable_compressed_debug_info +dump_symbols: $dump_symbols + +EOF + +function _build_graph { + pushd ${build_dir} + cmake -DCMAKE_BUILD_TYPE=${build_type} \ + -DNEBULA_BUILD_VERSION=${version} \ + -DENABLE_ASAN=${san} \ + -DENABLE_UBSAN=${san} \ + -DENABLE_STATIC_ASAN=${ssan} \ + -DENABLE_STATIC_UBSAN=${ssan} \ + -DCMAKE_INSTALL_PREFIX=${install_prefix} \ + -DENABLE_TESTING=OFF \ + -DENABLE_CONSOLE_COMPILATION=${build_console} \ + -DENABLE_PACK_ONE=${package_one} \ + -DENABLE_COMPRESSED_DEBUG_INFO=${enable_compressed_debug_info} \ + -DENABLE_PACKAGE_TAR=${package_tar} \ + -DENABLE_BREAKPAD=${enable_breakpad} \ + ${project_dir} + + if ! ( make -j ${jobs} ); then + echo ">>> build NebulaGraph failed <<<" + exit 1 + fi + popd + echo ">>> build NebulaGraph successfully <<<" +} + +# args: +function build { + version=$1 + san=$2 + ssan=$3 + build_type=$4 + package_tar=$5 + install_prefix=$6 + + mkdir -p ${build_dir} + + _build_graph +} + +# args: +function package { + pushd ${build_dir} + strip_enable=$1 + + args="" + [[ $strip_enable == TRUE ]] && args="-D CPACK_STRIP_FILES=TRUE -D CPACK_RPM_SPEC_MORE_DEFINE=" + + if ! ( cpack --verbose $args ); then + echo ">>> package NebulaGraph failed <<<" + exit 1 + else + # rename package file + outputDir=$build_dir/cpack_output + mkdir -p ${outputDir} + for pkg_name in $(ls ./*nebula*-${version}*); do + mv ${pkg_name} ${outputDir}/ + echo "####### target package file is ${outputDir}/${pkg_name}" + done + fi + + popd +} + +function _find_dump_syms_tool { + if [[ -x ${build_dir}/third-party/install/bin/dump_syms ]]; then + dump_syms_tool_dir=${build_dir}/third-party/install/bin + elif [[ -x /opt/vesoft/third-party/3.3/bin/dump_syms ]]; then + dump_syms_tool_dir=/opt/vesoft/third-party/3.3/bin + else + echo ">>> Failed to find the dump_syms tool <<<" + exit 1 + fi +} + +# This is only for releasing the disk resources. +function _strip_unnecessary_binaries { + for bin in $(ls -1 -F ${build_dir}/bin/ | grep -v [/$] | sed -e '/nebula-metad/d;/nebula-graphd/d;/nebula-storaged/d'); do + if ! (strip ${build_dir}/bin/${bin}); then + echo ">>> strip ${bin} failed: $?. <<<" + exit 1 + fi + done +} + +function dump_syms { + _strip_unnecessary_binaries + _find_dump_syms_tool + dump_syms=${dump_syms_tool_dir}/dump_syms + + syms_dir=${build_dir}/symbols/ + rm -rf ${syms_dir} && mkdir -p ${syms_dir} + + pack=`ls ${build_dir}/cpack_output/` + tmp=${pack#nebula-graph} + ver=${tmp%.*} + + hash objcopy &> /dev/null || { + echo "'objcopy' is not installed" + exit 1 + } + + for bin in nebula-graphd nebula-storaged nebula-metad; do + # Create separate debuginfo for GDB + objcopy --only-keep-debug ${build_dir}/bin/${bin} ${syms_dir}/${bin}${ver}.debug + # Create separate debuginfo for breakpad + if ! (${dump_syms} ${build_dir}/bin/${bin} > ${syms_dir}/${bin}${ver}.sym); then + echo ">>> dump ${bin} symbols failed: $?. <<<" + exit 1 + fi + done +} + +function add_commit { + hash git &> /dev/null + if [ $? -eq 0 ];then + if [ -d ".git" ];then + git rev-parse --short HEAD > ${build_dir}/cpack_output/commit.txt + fi + fi +} + +# The main +build $version $enablesanitizer $static_sanitizer $build_type "OFF" "/usr/local/nebula" +package $strip_enable +if [[ "$dump_symbols" == "ON" ]]; then + echo ">>> start dump symbols <<<" + dump_syms +fi + +# tar package +build $version $enablesanitizer $static_sanitizer $build_type "ON" "/" +package $strip_enable + +# add commit information +add_commit diff --git a/package/postinst b/package/postinst new file mode 100644 index 0000000..0d56305 --- /dev/null +++ b/package/postinst @@ -0,0 +1,12 @@ +#!/bin/bash + +install_dir=/usr/local/nebula +daemons=(metad graphd storaged) +for daemon in ${daemons[@]} +do + if [[ ! -f ${install_dir}/etc/nebula-${daemon}.conf ]] && [[ -f ${install_dir}/etc/nebula-${daemon}.conf.default ]]; then + cp ${install_dir}/etc/nebula-${daemon}.conf.default ${install_dir}/etc/nebula-${daemon}.conf + chmod 644 ${install_dir}/etc/nebula-${daemon}.conf + fi +done + diff --git a/package/rpm_postinst b/package/rpm_postinst new file mode 100644 index 0000000..ba17d5d --- /dev/null +++ b/package/rpm_postinst @@ -0,0 +1,11 @@ +#!/bin/bash + +daemons=(metad graphd storaged) +for daemon in ${daemons[@]} +do + if [[ ! -f $RPM_INSTALL_PREFIX/etc/nebula-${daemon}.conf ]] && [[ -f $RPM_INSTALL_PREFIX/etc/nebula-${daemon}.conf.default ]]; then + cp $RPM_INSTALL_PREFIX/etc/nebula-${daemon}.conf.default $RPM_INSTALL_PREFIX/etc/nebula-${daemon}.conf + chmod 644 $RPM_INSTALL_PREFIX/etc/nebula-${daemon}.conf + fi +done + diff --git a/resources/CMakeLists.txt b/resources/CMakeLists.txt new file mode 100644 index 0000000..2368900 --- /dev/null +++ b/resources/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +install( + FILES + gflags.json + date_time_zonespec.csv + PERMISSIONS + OWNER_WRITE OWNER_READ + GROUP_READ + WORLD_READ + DESTINATION + share/resources + COMPONENT + common +) diff --git a/resources/README.md b/resources/README.md new file mode 100644 index 0000000..8c30837 --- /dev/null +++ b/resources/README.md @@ -0,0 +1,3 @@ +# Overview + +This directory holds docs and resources shipped with the **NebulaGraph** package. diff --git a/resources/date_time_zonespec.csv b/resources/date_time_zonespec.csv new file mode 100644 index 0000000..c528d57 --- /dev/null +++ b/resources/date_time_zonespec.csv @@ -0,0 +1,520 @@ +"ID","STD ABBR","STD NAME","DST ABBR","DST NAME","GMT offset","DST adjustment","DST Start Date rule","Start time","DST End date rule","End time" +"Africa/Abidjan","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Accra","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Addis_Ababa","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Algiers","CET","CET","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Asmara","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Asmera","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Bamako","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Bangui","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Banjul","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Bissau","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Blantyre","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Brazzaville","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Bujumbura","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Cairo","EET","EET","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Casablanca","WET","WET","WEST","WEST","+00:00:00","+01:00:00","2;0;7","+02:00:00","-1;0;10","+03:00:00" +"Africa/Ceuta","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Africa/Conakry","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Dakar","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Dar_es_Salaam","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Djibouti","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Douala","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/El_Aaiun","WET","WET","WEST","WEST","+00:00:00","+01:00:00","2;0;7","+02:00:00","-1;0;10","+03:00:00" +"Africa/Freetown","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Gaborone","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Harare","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Johannesburg","SAST","SAST","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Juba","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Kampala","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Khartoum","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Kigali","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Kinshasa","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Lagos","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Libreville","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Lome","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Luanda","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Lubumbashi","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Lusaka","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Malabo","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Maputo","CAT","CAT","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Maseru","SAST","SAST","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Mbabane","SAST","SAST","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Mogadishu","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Monrovia","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Nairobi","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Africa/Ndjamena","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Niamey","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Nouakchott","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Ouagadougou","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Porto-Novo","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Porto_Novo","WAT","WAT","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Sao_Tome","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Timbuktu","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Africa/Tripoli","EET","EET","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Africa/Tunis","CET","CET","","","+01:00:00","+00:00:00","","","","+00:00:00" +"Africa/Windhoek","WAT","WAT","WAST","WAST","+01:00:00","+01:00:00","1;0;9","+02:00:00","1;0;4","+02:00:00" +"America/Adak","HST","HST","HDT","HDT","-10:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Anchorage","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Anguilla","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Antigua","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Araguaina","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Buenos_Aires","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Catamarca","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/ComodRivadavia","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Cordoba","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Jujuy","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/La_Rioja","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Mendoza","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Rio_Gallegos","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Salta","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/San_Juan","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/San_Luis","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Tucuman","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Argentina/Ushuaia","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Aruba","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Asuncion","PYT","PYT","PYST","PYST","-04:00:00","+01:00:00","1;0;10","+00:00:00","-1;0;3","+00:00:00" +"America/Atikokan","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Atka","HST","HST","HDT","HDT","-10:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Bahia","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Bahia_Banderas","CST","CST","CDT","CDT","-06:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/Barbados","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Belem","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Belize","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Beulah","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Blanc-Sablon","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Blanc_Sablon","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Boa_Vista","AMT","AMT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Bogota","COT","COT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Boise","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Buenos_Aires","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Cambridge_Bay","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Campo_Grande","AMT","AMT","AMST","AMST","-04:00:00","+01:00:00","3;0;10","+00:00:00","3;0;2","+00:00:00" +"America/Cancun","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Caracas","VET","VET","","","-04:30:00","+00:00:00","","","","+00:00:00" +"America/Catamarca","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Cayenne","GFT","GFT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Cayman","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Center","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Chicago","CST","Central Standard Time","CDT","Central Daylight Time","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Chihuahua","MST","MST","MDT","MDT","-07:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/ComodRivadavia","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Coral_Harbour","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Cordoba","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Costa_Rica","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Creston","MST","MST","","","-07:00:00","+00:00:00","","","","+00:00:00" +"America/Cuiaba","AMT","AMT","AMST","AMST","-04:00:00","+01:00:00","3;0;10","+00:00:00","3;0;2","+00:00:00" +"America/Curacao","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Danmarkshavn","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"America/Dawson","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Dawson_Creek","MST","MST","","","-07:00:00","+00:00:00","","","","+00:00:00" +"America/Denver","MST","Mountain Standard Time","MDT","Mountain Daylight Time","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Detroit","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Dominica","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Edmonton","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Eirunepe","ACT","ACT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/El_Salvador","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Ensenada","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Fortaleza","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Fort_Wayne","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Glace_Bay","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Godthab","WGT","WGT","WGST","WGST","-03:00:00","+01:00:00","-1;6;3","+22:00:00","-1;6;10","+23:00:00" +"America/Goose_Bay","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Grand_Turk","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Grenada","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Guadeloupe","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Guatemala","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Guayaquil","ECT","ECT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Guyana","GYT","GYT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Halifax","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Havana","CST","CST","CDT","CDT","-05:00:00","+01:00:00","2;0;3","+00:00:00","1;0;11","+01:00:00" +"America/Hermosillo","MST","MST","","","-07:00:00","+00:00:00","","","","+00:00:00" +"America/Indiana/Indianapolis","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Knox","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Marengo","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Petersburg","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Tell_City","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Vevay","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Vincennes","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indiana/Winamac","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Indianapolis","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Inuvik","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Iqaluit","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Jamaica","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Jujuy","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Juneau","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Kentucky/Louisville","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Kentucky/Monticello","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Knox","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Knox_IN","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Kralendijk","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/La_Paz","BOT","BOT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/La_Rioja","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Lima","PET","PET","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Los_Angeles","PST","Pacific Standard Time","PDT","Pacific Daylight Time","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Louisville","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Lower_Princes","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Maceio","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Managua","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Manaus","AMT","AMT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Marengo","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Marigot","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Martinique","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Matamoros","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Mazatlan","MST","MST","MDT","MDT","-07:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/Mendoza","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Menominee","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Merida","CST","CST","CDT","CDT","-06:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/Metlakatla","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Mexico_City","CST","CST","CDT","CDT","-06:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/Miquelon","PMST","PMST","PMDT","PMDT","-03:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Moncton","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Monterrey","CST","CST","CDT","CDT","-06:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"America/Montevideo","UYT","UYT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Monticello","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Montreal","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Montserrat","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Nassau","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/New_Salem","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/New_York","EST","Eastern Standard Time","EDT","Eastern Daylight Time","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Nipigon","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Nome","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Noronha","FNT","FNT","","","-02:00:00","+00:00:00","","","","+00:00:00" +"America/North_Dakota/Beulah","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/North_Dakota/Center","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/North_Dakota/New_Salem","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Ojinaga","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Panama","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Pangnirtung","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Paramaribo","SRT","SRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Petersburg","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Phoenix","MST","Mountain Standard Time","","","-07:00:00","+00:00:00","","","","+00:00:00" +"America/Port-au-Prince","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Porto_Acre","ACT","ACT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Porto_Velho","AMT","AMT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Port_au_Prince","EST","EST","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Port_of_Spain","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Puerto_Rico","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Rainy_River","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Rankin_Inlet","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Recife","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Regina","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Resolute","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Rio_Branco","ACT","ACT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"America/Rio_Gallegos","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Rosario","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Salta","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Santarem","BRT","BRT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Santa_Isabel","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Santiago","CLT","CLT","CLST","CLST","-04:00:00","+01:00:00","2;0;8","+00:00:00","3;0;5","+00:00:00" +"America/Santo_Domingo","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/San_Juan","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/San_Luis","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Sao_Paulo","BRT","BRT","BRST","BRST","-03:00:00","+01:00:00","3;0;10","+00:00:00","3;0;2","+00:00:00" +"America/Scoresbysund","EGT","EGT","EGST","EGST","-01:00:00","+01:00:00","-1;0;3","+00:00:00","-1;0;10","+01:00:00" +"America/Shiprock","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Sitka","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/St_Barthelemy","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/St_Johns","NST","NST","NDT","NDT","-03:30:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/St_Kitts","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/St_Lucia","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/St_Thomas","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/St_Vincent","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Swift_Current","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Tegucigalpa","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"America/Tell_City","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Thule","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Thunder_Bay","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Tijuana","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Toronto","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Tortola","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Tucuman","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Ushuaia","ART","ART","","","-03:00:00","+00:00:00","","","","+00:00:00" +"America/Vancouver","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Vevay","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Vincennes","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Virgin","AST","AST","","","-04:00:00","+00:00:00","","","","+00:00:00" +"America/Whitehorse","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Winamac","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Winnipeg","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Yakutat","AKST","AKST","AKDT","AKDT","-09:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"America/Yellowknife","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Antarctica/Casey","AWST","AWST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/Davis","DAVT","DAVT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/DumontDUrville","DDUT","DDUT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/Macquarie","MIST","MIST","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/Mawson","MAWT","MAWT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/McMurdo","NZST","NZST","NZDT","NZDT","+12:00:00","+01:00:00","-1;0;9","+02:00:00","1;0;4","+03:00:00" +"Antarctica/Palmer","CLT","CLT","CLST","CLST","-04:00:00","+01:00:00","2;0;8","+00:00:00","3;0;5","+00:00:00" +"Antarctica/Rothera","ROTT","ROTT","","","-03:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/South_Pole","NZST","NZST","NZDT","NZDT","+12:00:00","+01:00:00","-1;0;9","+02:00:00","1;0;4","+03:00:00" +"Antarctica/Syowa","SYOT","SYOT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Antarctica/Troll","UTC","UTC","CEST","CEST","+00:00:00","+02:00:00","-1;0;3","+01:00:00","-1;0;10","+03:00:00" +"Antarctica/Vostok","VOST","VOST","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Arctic/Longyearbyen","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Asia/Aden","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Almaty","ALMT","ALMT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Amman","EET","EET","EEST","EEST","+02:00:00","+01:00:00","1;5;4","+00:00:00","-1;5;10","+01:00:00" +"Asia/Anadyr","ANAT","ANAT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Asia/Aqtau","AQTT","AQTT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Aqtobe","AQTT","AQTT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ashgabat","TMT","TMT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ashkhabad","TMT","TMT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Baghdad","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Bahrain","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Baku","AZT","AZT","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Asia/Bangkok","ICT","ICT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Beirut","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+00:00:00","-1;0;10","+00:00:00" +"Asia/Bishkek","KGT","KGT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Brunei","BNT","BNT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Calcutta","IST","IST","","","+05:30:00","+00:00:00","","","","+00:00:00" +"Asia/Chita","YAKT","YAKT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Choibalsan","CHOT","CHOT","CHOST","CHOST","+08:00:00","+01:00:00","-1;6;3","+02:00:00","-1;6;9","+00:00:00" +"Asia/Chongqing","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Chungking","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Colombo","IST","IST","","","+05:30:00","+00:00:00","","","","+00:00:00" +"Asia/Dacca","BDT","BDT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Damascus","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;5;3","+00:00:00","-1;5;10","+00:00:00" +"Asia/Dhaka","BDT","BDT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Dili","TLT","TLT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Dubai","GST","GST","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Asia/Dushanbe","TJT","TJT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Gaza","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;6;3","+01:00:00","3;5;10","+00:00:00" +"Asia/Harbin","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Hebron","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;6;3","+01:00:00","3;5;10","+00:00:00" +"Asia/Hong_Kong","HKT","HKT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Hovd","HOVT","HOVT","HOVST","HOVST","+07:00:00","+01:00:00","-1;6;3","+02:00:00","-1;6;9","+00:00:00" +"Asia/Ho_Chi_Minh","ICT","ICT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Irkutsk","IRKT","IRKT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Istanbul","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Asia/Jakarta","WIB","WIB","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Jayapura","WIT","WIT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Jerusalem","IST","IST","IDT","IDT","+02:00:00","+01:00:00","-1;5;3","+02:00:00","-1;0;10","+02:00:00" +"Asia/Kabul","AFT","AFT","","","+04:30:00","+00:00:00","","","","+00:00:00" +"Asia/Kamchatka","PETT","PETT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Asia/Karachi","PKT","PKT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kashgar","XJT","XJT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kathmandu","NPT","NPT","","","+05:45:00","+00:00:00","","","","+00:00:00" +"Asia/Katmandu","NPT","NPT","","","+05:45:00","+00:00:00","","","","+00:00:00" +"Asia/Khandyga","YAKT","YAKT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kolkata","IST","IST","","","+05:30:00","+00:00:00","","","","+00:00:00" +"Asia/Krasnoyarsk","KRAT","KRAT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kuala_Lumpur","MYT","MYT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kuching","MYT","MYT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Kuwait","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Macao","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Macau","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Magadan","MAGT","MAGT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Asia/Makassar","WITA","WITA","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Manila","PHT","PHT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Muscat","GST","GST","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Asia/Nicosia","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Asia/Novokuznetsk","KRAT","KRAT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Novosibirsk","NOVT","NOVT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Omsk","OMST","OMST","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Oral","ORAT","ORAT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Phnom_Penh","ICT","ICT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Pontianak","WIB","WIB","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Pyongyang","KST","KST","","","+08:30:00","+00:00:00","","","","+00:00:00" +"Asia/Qatar","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Qyzylorda","QYZT","QYZT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Rangoon","MMT","MMT","","","+06:30:00","+00:00:00","","","","+00:00:00" +"Asia/Riyadh","AST","AST","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Asia/Saigon","ICT","ICT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Sakhalin","SAKT","SAKT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Asia/Samarkand","UZT","UZT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Seoul","KST","KST","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Shanghai","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Singapore","SGT","SGT","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Srednekolymsk","SRET","SRET","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Asia/Taipei","CST","CST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Tashkent","UZT","UZT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Tbilisi","GET","GET","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Asia/Tehran","IRST","IRST","IRDT","IRDT","+03:30:00","+01:00:00","3;1;3","+00:00:00","3;3;9","+00:00:00" +"Asia/Tel_Aviv","IST","IST","IDT","IDT","+02:00:00","+01:00:00","-1;5;3","+02:00:00","-1;0;10","+02:00:00" +"Asia/Thimbu","BTT","BTT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Thimphu","BTT","BTT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Tokyo","JST","JST","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ujung_Pandang","WITA","WITA","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ulaanbaatar","ULAT","ULAT","ULAST","ULAST","+08:00:00","+01:00:00","-1;6;3","+02:00:00","-1;6;9","+00:00:00" +"Asia/Ulan_Bator","ULAT","ULAT","ULAST","ULAST","+08:00:00","+01:00:00","-1;6;3","+02:00:00","-1;6;9","+00:00:00" +"Asia/Urumqi","XJT","XJT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ust-Nera","VLAT","VLAT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Asia/Ust_Nera","VLAT","VLAT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Asia/Vientiane","ICT","ICT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Asia/Vladivostok","VLAT","VLAT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Asia/Yakutsk","YAKT","YAKT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Asia/Yekaterinburg","YEKT","YEKT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Asia/Yerevan","AMT","AMT","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Atlantic/Azores","AZOT","AZOT","AZOST","AZOST","-01:00:00","+01:00:00","-1;0;3","+00:00:00","-1;0;10","+01:00:00" +"Atlantic/Bermuda","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Atlantic/Canary","WET","WET","WEST","WEST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Atlantic/Cape_Verde","CVT","CVT","","","-01:00:00","+00:00:00","","","","+00:00:00" +"Atlantic/Faeroe","WET","WET","WEST","WEST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Atlantic/Faroe","WET","WET","WEST","WEST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Atlantic/Jan_Mayen","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Atlantic/Madeira","WET","WET","WEST","WEST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Atlantic/Reykjavik","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Atlantic/South_Georgia","GST","GST","","","-02:00:00","+00:00:00","","","","+00:00:00" +"Atlantic/Stanley","FKST","FKST","","","-03:00:00","+00:00:00","","","","+00:00:00" +"Atlantic/St_Helena","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Australia/ACT","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Adelaide","ACST","ACST","ACDT","ACDT","+09:30:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Brisbane","AEST","AEST","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Australia/Broken_Hill","ACST","ACST","ACDT","ACDT","+09:30:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Canberra","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Currie","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Darwin","ACST","ACST","","","+09:30:00","+00:00:00","","","","+00:00:00" +"Australia/Eucla","ACWST","ACWST","","","+08:45:00","+00:00:00","","","","+00:00:00" +"Australia/Hobart","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/LHI","LHST","LHST","LHDT","LHDT","+10:30:00","+00:30:00","1;0;10","+02:00:00","1;0;4","+02:00:00" +"Australia/Lindeman","AEST","AEST","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Australia/Lord_Howe","LHST","LHST","LHDT","LHDT","+10:30:00","+00:30:00","1;0;10","+02:00:00","1;0;4","+02:00:00" +"Australia/Melbourne","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/North","ACST","ACST","","","+09:30:00","+00:00:00","","","","+00:00:00" +"Australia/NSW","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Perth","AWST","AWST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Australia/Queensland","AEST","AEST","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Australia/South","ACST","ACST","ACDT","ACDT","+09:30:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Sydney","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Tasmania","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/Victoria","AEST","AEST","AEDT","AEDT","+10:00:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Australia/West","AWST","AWST","","","+08:00:00","+00:00:00","","","","+00:00:00" +"Australia/Yancowinna","ACST","ACST","ACDT","ACDT","+09:30:00","+01:00:00","1;0;10","+02:00:00","1;0;4","+03:00:00" +"Brazil/Acre","ACT","ACT","","","-05:00:00","+00:00:00","","","","+00:00:00" +"Brazil/DeNoronha","FNT","FNT","","","-02:00:00","+00:00:00","","","","+00:00:00" +"Brazil/East","BRT","BRT","BRST","BRST","-03:00:00","+01:00:00","3;0;10","+00:00:00","3;0;2","+00:00:00" +"Brazil/West","AMT","AMT","","","-04:00:00","+00:00:00","","","","+00:00:00" +"Canada/Atlantic","AST","AST","ADT","ADT","-04:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/Central","CST","CST","CDT","CDT","-06:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/East-Saskatchewan","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"Canada/Eastern","EST","EST","EDT","EDT","-05:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/East_Saskatchewan","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"Canada/Mountain","MST","MST","MDT","MDT","-07:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/Newfoundland","NST","NST","NDT","NDT","-03:30:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/Pacific","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Canada/Saskatchewan","CST","CST","","","-06:00:00","+00:00:00","","","","+00:00:00" +"Canada/Yukon","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Chile/Continental","CLT","CLT","CLST","CLST","-04:00:00","+01:00:00","2;0;8","+00:00:00","3;0;5","+00:00:00" +"Chile/EasterIsland","EAST","EAST","EASST","EASST","-06:00:00","+01:00:00","2;6;8","+22:00:00","2;6;5","+22:00:00" +"Etc/GMT","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Etc/Greenwich","GMT","GMT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Etc/UCT","UCT","UCT","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Etc/Universal","UTC","UTC","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Etc/UTC","UTC","UTC","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Etc/Zulu","UTC","UTC","","","+00:00:00","+00:00:00","","","","+00:00:00" +"Europe/Amsterdam","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Andorra","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Athens","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Belfast","GMT","GMT","BST","BST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Belgrade","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Berlin","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Bratislava","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Brussels","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Bucharest","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Budapest","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Busingen","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Chisinau","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Copenhagen","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Dublin","GMT","GMT","IST","IST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Gibraltar","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Guernsey","GMT","GMT","BST","BST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Helsinki","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Isle_of_Man","GMT","GMT","BST","BST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Istanbul","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Jersey","GMT","GMT","BST","BST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Kaliningrad","EET","EET","","","+02:00:00","+00:00:00","","","","+00:00:00" +"Europe/Kiev","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Lisbon","WET","WET","WEST","WEST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Ljubljana","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/London","GMT","GMT","BST","BST","+00:00:00","+01:00:00","-1;0;3","+01:00:00","-1;0;10","+02:00:00" +"Europe/Luxembourg","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Madrid","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Malta","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Mariehamn","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Minsk","MSK","MSK","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Europe/Monaco","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Moscow","MSK","MSK","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Europe/Nicosia","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Oslo","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Paris","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Podgorica","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Prague","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Riga","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Rome","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Samara","SAMT","SAMT","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Europe/San_Marino","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Sarajevo","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Simferopol","MSK","MSK","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Europe/Skopje","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Sofia","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Stockholm","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Tallinn","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Tirane","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Tiraspol","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Uzhgorod","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Vaduz","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Vatican","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Vienna","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Vilnius","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Volgograd","MSK","MSK","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Europe/Warsaw","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Zagreb","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Europe/Zaporozhye","EET","EET","EEST","EEST","+02:00:00","+01:00:00","-1;0;3","+03:00:00","-1;0;10","+04:00:00" +"Europe/Zurich","CET","CET","CEST","CEST","+01:00:00","+01:00:00","-1;0;3","+02:00:00","-1;0;10","+03:00:00" +"Indian/Antananarivo","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Indian/Chagos","IOT","IOT","","","+06:00:00","+00:00:00","","","","+00:00:00" +"Indian/Christmas","CXT","CXT","","","+07:00:00","+00:00:00","","","","+00:00:00" +"Indian/Cocos","CCT","CCT","","","+06:30:00","+00:00:00","","","","+00:00:00" +"Indian/Comoro","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Indian/Kerguelen","TFT","TFT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Indian/Mahe","SCT","SCT","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Indian/Maldives","MVT","MVT","","","+05:00:00","+00:00:00","","","","+00:00:00" +"Indian/Mauritius","MUT","MUT","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Indian/Mayotte","EAT","EAT","","","+03:00:00","+00:00:00","","","","+00:00:00" +"Indian/Reunion","RET","RET","","","+04:00:00","+00:00:00","","","","+00:00:00" +"Mexico/BajaNorte","PST","PST","PDT","PDT","-08:00:00","+01:00:00","2;0;3","+02:00:00","1;0;11","+02:00:00" +"Mexico/BajaSur","MST","MST","MDT","MDT","-07:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"Mexico/General","CST","CST","CDT","CDT","-06:00:00","+01:00:00","1;0;4","+02:00:00","-1;0;10","+02:00:00" +"Pacific/Apia","WSST","WSST","WSDT","WSDT","+13:00:00","+01:00:00","-1;0;9","+03:00:00","1;0;4","+04:00:00" +"Pacific/Auckland","NZST","NZST","NZDT","NZDT","+12:00:00","+01:00:00","-1;0;9","+02:00:00","1;0;4","+03:00:00" +"Pacific/Chatham","CHAST","CHAST","CHADT","CHADT","+12:45:00","+01:00:00","-1;0;9","+02:00:00","1;0;4","+03:00:00" +"Pacific/Chuuk","CHUT","CHUT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Easter","EAST","EAST","EASST","EASST","-06:00:00","+01:00:00","2;6;8","+22:00:00","2;6;5","+22:00:00" +"Pacific/Efate","VUT","VUT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Enderbury","PHOT","PHOT","","","+13:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Fakaofo","TKT","TKT","","","+13:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Fiji","FJT","FJT","FJST","FJST","+12:00:00","+01:00:00","1;0;11","+02:00:00","3;0;1","+03:00:00" +"Pacific/Funafuti","TVT","TVT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Galapagos","GALT","GALT","","","-06:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Gambier","GAMT","GAMT","","","-09:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Guadalcanal","SBT","SBT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Guam","ChST","ChST","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Honolulu","HST","HST","","","-10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Johnston","HST","HST","","","-10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Kiritimati","LINT","LINT","","","+14:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Kosrae","KOST","KOST","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Kwajalein","MHT","MHT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Majuro","MHT","MHT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Marquesas","MART","MART","","","-09:30:00","+00:00:00","","","","+00:00:00" +"Pacific/Midway","SST","SST","","","-11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Nauru","NRT","NRT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Niue","NUT","NUT","","","-11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Norfolk","NFT","NFT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Noumea","NCT","NCT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Pago_Pago","SST","SST","","","-11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Palau","PWT","PWT","","","+09:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Pitcairn","PST","PST","","","-08:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Pohnpei","PONT","PONT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Ponape","PONT","PONT","","","+11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Port_Moresby","PGT","PGT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Rarotonga","CKT","CKT","","","-10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Saipan","ChST","ChST","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Samoa","SST","SST","","","-11:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Tahiti","TAHT","TAHT","","","-10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Tarawa","GILT","GILT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Tongatapu","TOT","TOT","","","+13:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Truk","CHUT","CHUT","","","+10:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Wake","WAKT","WAKT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Wallis","WFT","WFT","","","+12:00:00","+00:00:00","","","","+00:00:00" +"Pacific/Yap","CHUT","CHUT","","","+10:00:00","+00:00:00","","","","+00:00:00" diff --git a/resources/gflags.json b/resources/gflags.json new file mode 100644 index 0000000..bc928e7 --- /dev/null +++ b/resources/gflags.json @@ -0,0 +1,26 @@ +{ + "MUTABLE": [ + "minloglevel", + "v", + "heartbeat_interval_secs", + "meta_client_retry_times", + "clean_wal_interval_secs", + "wal_ttl", + "clean_wal_interval_secs", + "accept_partial_success", + "system_memory_high_watermark_ratio", + "num_rows_to_check_memory", + "session_idle_timeout_secs", + "session_reclaim_interval_secs", + "max_allowed_connections", + "disable_octal_escape_char", + "max_sessions_per_ip_per_user", + "timezone_name" + ], + "NESTED": [ + "rocksdb_db_options", + "rocksdb_column_family_options", + "rocksdb_block_based_table_options" + ] +} + diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt new file mode 100644 index 0000000..5423139 --- /dev/null +++ b/scripts/CMakeLists.txt @@ -0,0 +1,113 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +if(NOT ENABLE_STANDALONE_VERSION) + +install( + FILES + nebula-storaged.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + storage +) + +install( + FILES + nebula-metad.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + meta +) + +install( + FILES + nebula-graphd.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + graph +) + +else() + +install( + FILES + nebula-standalone.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + graph +) + +endif() + +install( + FILES + nebula.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + common +) + +install( + FILES + utils.sh + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + common +) + +install( + FILES + meta-transfer-tools.sh + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + meta +) + +install( + FILES + nebula-storaged-listener.service + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + DESTINATION + scripts + COMPONENT + storage +) diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..902cb29 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,29 @@ +# Overview + +This document handles **NebulaGraph** installation, and all the operations are done on single hosts. + +Firstly, you need to create the config files for the three daemons in the `conf` directory based on their `*.default` counterparts. + +You could operate on all of the three daemons with `scripts/nebula.service`: + +```shell +$ scripts/nebula.service start all +... +$ scripts/nebula.service status all +... +$ scripts/nebula.service stop all +... +$ scripts/nebula.service restart all +... +``` + +Or, you could do some of the same action on a specific daemon: + +```shell +$ scripts/nebula.service start metad # Or, scripts/nebula-metad.service start +... +$ scripts/nebula.service status graphd # Or, scripts/nebula-graphd.service status +... +$ scripts/nebula.service restart storaged # Or, scripts/nebula-storaged.service restart +... +``` diff --git a/scripts/meta-transfer-tools.sh b/scripts/meta-transfer-tools.sh new file mode 100644 index 0000000..6e773ab --- /dev/null +++ b/scripts/meta-transfer-tools.sh @@ -0,0 +1,91 @@ +#! /bin/bash +# +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# + +src= +dst= +metaDirName= +configs= + +function usage { +Usage="this tool is a simple wrapper to scp to copy local folder of metad to another (remote)place \n \ + it afford 4 options: \n \ + 1. -f (from) local metad dir \n \ + 2. -t (to) remote destination \n \ + 3. -u (update) any configs need to be changed \n \ + different configs should be separated by ':' \n \ + each config has to be the form of "local_ip=172.0.0.1" \n \ + \n \ + for example \n \ + ./meta-transfer-tools.sh -f /path-to-src/metad1 -t bob@172.0.0.1:/path-to-dst -u local_ip=172.0.0.1:port=10086 \n \ + " + + +echo -e $Usage +} + +while getopts ":hf:t:u:" opt; do + case $opt in + h) + usage + exit 0 + ;; + f) + echo "trying to copy metad from: " $OPTARG + src=$OPTARG + metaDirName=${OPTARG##*/} + ;; + t) + echo "trying to copy metad to: " $OPTARG + dst=$OPTARG + ;; + u) + echo "-u: " $OPTARG + configs=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 1 + ;; + esac +done + +if [[ ! -d ${src} ]]; then + echo "${src} does not exist or not a folder" + exit 1 +fi + +configSurfix=/etc/nebula-metad.conf +if [[ ! -f ${src}${configSurfix} ]]; then + echo "${src}${configSurfix} does not exist" + exit 1 +fi + +tmpConfigFile=nebula-metad.conf.bak + +cp ${src}${configSurfix} ${tmpConfigFile} +if [[ -z $configs ]]; then + echo no configs need to replace +else + echo going to parse configs: $configs + array=(${configs//:/ }) + for var in ${array[@]} + do + key=${var%=*} + val=${var#*=} + sed -iE "s/--${key}.*$/--${key}=${val}/" ${tmpConfigFile} + done +fi + +echo "trying to copy metad from $src to $dst" + +set -x +scp -r $src $dst +scp $tmpConfigFile $dst/$metaDirName$configSurfix +set +x + +rm $tmpConfigFile* diff --git a/scripts/nebula-graphd.service b/scripts/nebula-graphd.service new file mode 100644 index 0000000..c850290 --- /dev/null +++ b/scripts/nebula-graphd.service @@ -0,0 +1,6 @@ +#! /bin/bash + +SCRIPT_PATH=$(readlink -f $0) +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +export USAGE_INFO="${0} [-v] [-c /path/to/conf] " +${SCRIPT_DIR}/nebula.service $@ graphd diff --git a/scripts/nebula-metad.service b/scripts/nebula-metad.service new file mode 100644 index 0000000..2660fab --- /dev/null +++ b/scripts/nebula-metad.service @@ -0,0 +1,6 @@ +#! /bin/bash + +SCRIPT_PATH=$(readlink -f $0) +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +export USAGE_INFO="${0} [-v] [-c /path/to/conf] " +${SCRIPT_DIR}/nebula.service $@ metad diff --git a/scripts/nebula-standalone.service b/scripts/nebula-standalone.service new file mode 100644 index 0000000..3405081 --- /dev/null +++ b/scripts/nebula-standalone.service @@ -0,0 +1,6 @@ +#! /bin/bash + +SCRIPT_PATH=$(readlink -f $0) +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +export USAGE_INFO="${0} [-v] [-c /path/to/conf] " +${SCRIPT_DIR}/nebula.service $@ standalone diff --git a/scripts/nebula-storaged-listener.service b/scripts/nebula-storaged-listener.service new file mode 100644 index 0000000..7837458 --- /dev/null +++ b/scripts/nebula-storaged-listener.service @@ -0,0 +1,6 @@ +#! /bin/bash + +SCRIPT_PATH=$(readlink -f $0) +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +export USAGE_INFO="${0} [-v] [-c /path/to/conf] " +${SCRIPT_DIR}/nebula.service $@ storaged-listener diff --git a/scripts/nebula-storaged.service b/scripts/nebula-storaged.service new file mode 100644 index 0000000..f19ee2a --- /dev/null +++ b/scripts/nebula-storaged.service @@ -0,0 +1,6 @@ +#! /bin/bash + +SCRIPT_PATH=$(readlink -f $0) +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +export USAGE_INFO="${0} [-v] [-c /path/to/conf] " +${SCRIPT_DIR}/nebula.service $@ storaged diff --git a/scripts/nebula.service b/scripts/nebula.service new file mode 100644 index 0000000..ff899e0 --- /dev/null +++ b/scripts/nebula.service @@ -0,0 +1,301 @@ +#! /bin/bash + +function usage { + echo "Usage: ${USAGE_INFO}" +} + +[[ -z ${USAGE_INFO} ]] && USAGE_INFO="${0} [-v] [-c /path/to/config] " + +if [[ $# == 0 ]]; then + usage + exit 1 +fi + +# Original directory +OLD_DIR=$(pwd) +# Path to this script +SCRIPT_PATH=$(readlink -f $0) +# Directory of this script +SCRIPT_DIR=$(dirname ${SCRIPT_PATH}) +# Installation directory, i.e. parent of SCRIPT_DIR +# For now we assume that the directory hierarchy of the nebula installation is: +# root/bin, root/etc/, root/scripts, etc. +INSTALL_ROOT=$(cd ${SCRIPT_DIR}/.. &>/dev/null; pwd) +UTILS_PATH=${SCRIPT_DIR}/utils.sh + +cd ${INSTALL_ROOT} + + +[[ -f ${UTILS_PATH} ]] || ERROR_AND_EXIT "${UTILS_PATH} not exist" + +source ${UTILS_PATH} || exit 1 + +function on_exit { + cd ${OLD_DIR} &>/dev/null +} + +trap on_exit EXIT + + +ACTION= +CONFIG= +DAEMON= +TARGET= +TARGETS=() +VERBOSE=0 + +# Parsing options from arguments +while getopts ":hvc:" opt; do + case $opt in + h) + usage + exit 0 + ;; + c) + CONFIG=${OPTARG} + ;; + v) + VERBOSE=1 + ;; + \?) + ERROR "Invalid option: -${OPTARG}" + usage + exit 1 + ;; + :) + ERROR_AND_EXIT "Option -${OPTARG} requires an argument." + ;; + esac +done + +# Regard the last two operands as `action' and `target' +shift $((OPTIND - 1)) +if [[ $# != 2 ]]; then + usage + exit 1 +fi +ACTION=${1} +TARGET=${2} + +if [ -e ${INSTALL_ROOT}/bin/nebula-standalone ] +then + ALLTARGET=(standalone) +else + ALLTARGET=(metad graphd storaged) +fi + +# Collect the daemons on which we perform the action on +case ${TARGET} in + metad) + TARGETS=(${TARGET}) + ;; + graphd) + TARGETS=(${TARGET}) + ;; + storaged) + TARGETS=(${TARGET}) + ;; + storaged-listener) + TARGETS=(${TARGET}) + ;; + standalone) + TARGETS=(${TARGET}) + ;; + all) + TARGETS=${ALLTARGET[*]} + ;; + *) + ERROR "Unknown daemon \`${DAEMON}'" + usage + exit 1 +esac + +# If the target is `all', the specified config file will be discarded +if [[ -n ${CONFIG} ]] && [[ ${TARGET} == "all" ]]; then + WARN "Config file specified but the target is \`all'" + CONFIG= +fi + + +# args: [config file] +function start_daemon { + local daemon_name=nebula-${1} + local config=${2} + local executable=${INSTALL_ROOT}/bin/${daemon_name} + if [[ $daemon_name == *-listener ]]; then + executable=${INSTALL_ROOT}/bin/${daemon_name%-listener*} + fi + + [[ -e ${executable} ]] || ERROR_AND_EXIT "${executable} not found" + + [[ -z ${config} ]] && config=${INSTALL_ROOT}/etc/${daemon_name}.conf + + [[ -f ${config} ]] || ERROR_AND_EXIT "${config} not found" + + pid_file=$(get_pid_file_from_config ${config}) + if [[ -f ${pid_file} ]]; then + if is_process_running ${pid_file}; then + ERROR "${daemon_name} already running: $(cat ${pid_file})" + exit 1 + fi + fi + + log_dir=$(get_log_dir_from_config ${config}) + [[ -z ${log_dir} ]] && log_dir=logs + make_log_dir_if_absent ${log_dir} || ERROR_AND_EXIT "Failed to create ${log_dir}" + + local command="${executable} --flagfile ${config}" + INFO "Starting ${daemon_name}..." + eval ${command} + code=$? + INFO "Done" + [[ ${code} -ne 0 ]] && exit ${code} +} + + +# args: [signal name] +function stop_by_pid_file { + local pid_file=${1} + local signame=${2} + [[ -z ${signame} ]] && signame=TERM + + if is_process_running ${pid_file}; then + [[ ${VERBOSE} -ne 0 ]] && INFO "Send SIG${signame} to $(cat ${pid_file})" + kill -s ${signame} $(cat ${pid_file}) + else + [[ ${VERBOSE} -ne 0 ]] && INFO "No such process" || code=$? + fi +} + + +# args: [config file] +function stop_daemon { + local daemon_name=nebula-${1} + local config=${2} + local executable=${INSTALL_ROOT}/bin/${daemon_name} + if [[ $daemon_name == *-listener ]]; then + executable=${INSTALL_ROOT}/bin/${daemon_name%-listener*} + fi + + [[ -e ${executable} ]] || ERROR_AND_EXIT "${executable} not found" + + [[ -z ${config} ]] && config=${INSTALL_ROOT}/etc/${daemon_name}.conf + + [[ -f ${config} ]] || ERROR_AND_EXIT "${config} not found" + + INFO "Stopping ${daemon_name}..." + stop_by_pid_file $(get_pid_file_from_config ${config}) + code=$? + INFO "Done" + [[ ${code} -ne 0 ]] && exit ${code} +} + + +# args: [config file] +function wait_daemon_to_exit { + local daemon_name=nebula-${1} + local config=${2} + local executable=${INSTALL_ROOT}/bin/${daemon_name} + if [[ $daemon_name == *-listener ]]; then + executable=${INSTALL_ROOT}/bin/${daemon_name%-listener*} + fi + + [[ -e ${executable} ]] || ERROR_AND_EXIT "${executable} not found" + + [[ -z ${config} ]] && config=${INSTALL_ROOT}/etc/${daemon_name}.conf + + [[ -f ${config} ]] || ERROR_AND_EXIT "${config} not found" + + wait_for_exit $(get_pid_file_from_config ${config}) 5 +} + + +# args: [config file] +function restart_daemon { + stop_daemon $@ + wait_daemon_to_exit $@ + start_daemon $@ +} + + +# args: [config file] +function kill_daemon { + local daemon_name=nebula-${1} + local config=${2} + + [[ -z ${config} ]] && config=${INSTALL_ROOT}/etc/${daemon_name}.conf + + [[ -f ${config} ]] || ERROR_AND_EXIT "${config} not found" + + INFO "Force killing ${daemon_name}..." + stop_by_pid_file $(get_pid_file_from_config ${config}) "KILL" + code=$? + INFO "Done" + [[ ${code} -ne 0 ]] && exit ${code} +} + + +# args: [config file] +function status_daemon { + local daemon_name=nebula-${1} + local config=${2} + + local executable=${INSTALL_ROOT}/bin/${daemon_name} + if [[ $daemon_name == *-listener ]]; then + executable=${INSTALL_ROOT}/bin/${daemon_name%-listener*} + fi + + local daemon_version=$(daemon_version ${executable}) + + [[ -z ${config} ]] && config=${INSTALL_ROOT}/etc/${daemon_name}.conf + + [[ -f ${config} ]] || ERROR_AND_EXIT "${config} not found" + + local pid_file=$(get_pid_file_from_config ${config}) + if is_process_running ${pid_file}; then + local port=$(get_port_from_config ${config}) + if is_port_listened_on ${port}; then + port=${GREEN}${port}${NC} + else + port=${BLINK}${RED}${port}${NC} + if [[ $daemon_name == nebula-storaged ]]; then + WARN "${daemon_name} after v3.0.0 will not start service until it is added to cluster." + WARN "See Manage Storage hosts:${RED}ADD HOSTS${NC} in https://docs.nebula-graph.io/" + fi + fi + INFO "${daemon_name}(${daemon_version}): Running as $(cat ${pid_file}), Listening on ${port}" + else + INFO "${daemon_name}(${daemon_version}): Exited" + fi +} + +# To perform some environment checking +env_check + +# The main driver +for DAEMON in ${TARGETS[@]} +do + case ${ACTION} in + start) + start_daemon ${DAEMON} ${CONFIG} + ;; + stop) + stop_daemon ${DAEMON} ${CONFIG} + ;; + restart) + restart_daemon ${DAEMON} ${CONFIG} + ;; + kill) + kill_daemon ${DAEMON} ${CONFIG} + ;; + status) + status_daemon ${DAEMON} ${CONFIG} + ;; + *) + usage + exit 1 + ;; + esac +done + +exit 0 diff --git a/scripts/services.sh b/scripts/services.sh new file mode 100644 index 0000000..6f9739e --- /dev/null +++ b/scripts/services.sh @@ -0,0 +1,68 @@ +#! /bin/bash + +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +SCRIPT_PATH=$(cd "$(dirname "$0")";pwd) + +if [ $# != 1 ]; then + echo "USAGE: $0 " + exit 1; +fi + +if [ -z $NEBULA_HOME ]; then + echo "NEBULA_HOME is not setting, using /usr/local as default value" + NEBULA_HOME=/usr/local +fi + +echo "Processing Meta Service ..." + +if [ ! -f ${SCRIPT_PATH}"/meta.hosts" ]; then + echo "${SCRIPT_PATH}/metas not exist" + exit 0 +fi + +for host in `cat ${SCRIPT_PATH}/meta.hosts` +do + echo "start $host" + if [ $1 = "status" ]; then + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-metad.conf $1 metad" + else + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-metad.conf $1 metad > /dev/null 2>&1 &" + fi +done + +echo "Processing Storage Service ..." + +if [ ! -f ${SCRIPT_PATH}"/storage.hosts" ]; then + echo "${SCRIPT_PATH}/storage.hosts not exist" + exit 0 +fi + +for host in `cat ${SCRIPT_PATH}/storage.hosts` +do + echo "start $host" + if [ $1 = "status" ]; then + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-storaged.conf $1 storaged" + else + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-storaged.conf $1 storaged > /dev/null 2>&1 &" + fi +done + +echo "Processing Graph Service ..." + +if [ ! -f ${SCRIPT_PATH}"/graph.hosts" ]; then + echo "${SCRIPT_PATH}/graph.hosts not exist" + exit 0 +fi + +for host in `cat ${SCRIPT_PATH}/graph.hosts` +do + echo "start $host" + if [ $1 = "status" ]; then + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-graphd.conf $1 graphd" + else + ssh $host "${NEBULA_HOME}/scripts/nebula.service -c ${NEBULA_HOME}/etc/nebula-graphd.conf $1 graphd > /dev/null 2>&1 &" + fi +done diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 0000000..1d115ac --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,162 @@ +# +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +# +# Some color code definitions +RED= +GREEN= +YELLOW= +BLUE= +BLINK= +NC= + +# To detect if stdout or stderr is being attached to terminal +function is_tty { + [[ -t 1 ]] && [[ -t 2 ]] + return $? +} + +# Enable colorful output only if no terminal attached +is_tty && RED=$(echo -e "\001\033[1;31m\002") +is_tty && GREEN=$(echo -e "\001\033[1;32m\002") +is_tty && YELLOW=$(echo -e "\001\033[1;33m\002") +is_tty && BLUE=$(echo -e "\001\033[1;34m\002") +is_tty && BLINK=$(echo -e "\001\033[5m\002") +is_tty && NC=$(echo -e "\001\033[0m\002") + + +function INFO { + echo "${GREEN}[INFO]${NC}" $@ +} + + +function WARN { + echo "${YELLOW}[WARN]${NC}" $@ +} + + +function ERROR { + echo "${RED}[ERROR]${NC}" $@ >&2 +} + + +function ERROR_AND_EXIT { + ERROR $@ + exit 1 +} + + +# Tell if a path is an absolute one +# args: +function is_absolute_path { + [[ ${1} = /* ]] + return $? +} + + +# Tell if a path is an existing directory +# args: +function is_existing_dir { + [[ -d ${1} ]] + return $? +} + + +# Tell if a process is running, via a file hosting its pid +# args: +function is_process_running { + local pid_file=${1} + [[ -f ${pid_file} ]] || return 1 + + local pid=$(cat ${pid_file}) + [[ -z ${pid} ]] && return 1 + + ps -p ${pid} &>/dev/null + return $? +} + + +# Tell if some process is listening on the target port +# args: +function is_port_listened_on { + local port=${1} + if which ss &>/dev/null; then + ss -lnt | grep ${port} &>/dev/null && return 0 + fi +} + + +# To wait for a process to exit +# args: +function wait_for_exit { + local pid_file=${1} + local seconds=${2} + is_process_running ${pid_file} || return 0 + while [[ ${seconds} > 0 ]]; do + sleep 0.1 + is_process_running ${pid_file} || return 0 + seconds=$(echo "${seconds} - 0.1" | bc -l) + done +} + +# To read a config item's value from the config file +# args: +function get_item_from_config { + sed -n -e "s/^--${2}=\(.*\)$/\1/p" ${1} 2>/dev/null | sed -n "1p" +} + + +# To read the path to the pid file from the config file +# args: +function get_pid_file_from_config { + get_item_from_config ${1} "pid_file" +} + + +# To read to listen port number from the config file +# args: +function get_port_from_config { + get_item_from_config ${1} "port" +} + + +# To read logging directory from the config file +# args: +function get_log_dir_from_config { + get_item_from_config ${1} "log_dir" +} + + +# To create the logging directory if absent +# args: +function make_log_dir_if_absent { + [[ -d ${1} ]] || mkdir -p ${1} >/dev/null +} + + +# Perform some environment checking +function env_check { + local nfile=$(ulimit -n 2>/dev/null) + local core=$(ulimit -c 2>/dev/null) + local cputime=$(ulimit -t 2>/dev/null) + + if [[ ${nfile} -le 1024 ]]; then + WARN "The maximum files allowed to open might be too few: ${nfile}" + fi + + if [[ ${core} -eq 0 ]]; then + ulimit -c unlimited &>/dev/null + fi + + if [[ ${cputime} != "unlimited" ]]; then + WARN "The CPU time a process can consume is restricted to ${cputime}" + fi +} + +# Get the daemon Git commit version information +function daemon_version { + local version=$(${1} --version | head -n 1) + echo ${version} | sed 's/.*Git: \([[:alnum:]]*\).*/\1/g' +} + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..4817a15 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_subdirectory(clients) +nebula_add_subdirectory(codec) +nebula_add_subdirectory(common) +nebula_add_subdirectory(daemons) +nebula_add_subdirectory(graph) +nebula_add_subdirectory(interface) +nebula_add_subdirectory(kvstore) +nebula_add_subdirectory(meta) +nebula_add_subdirectory(mock) +nebula_add_subdirectory(parser) +nebula_add_subdirectory(storage) +nebula_add_subdirectory(tools) +nebula_add_subdirectory(version) +nebula_add_subdirectory(webservice) +nebula_add_subdirectory(console) diff --git a/src/clients/CMakeLists.txt b/src/clients/CMakeLists.txt new file mode 100644 index 0000000..e2bb5d3 --- /dev/null +++ b/src/clients/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_subdirectory(meta) +nebula_add_subdirectory(storage) diff --git a/src/clients/graph/README.md b/src/clients/graph/README.md new file mode 100644 index 0000000..dc2e263 --- /dev/null +++ b/src/clients/graph/README.md @@ -0,0 +1,8 @@ +Clients for other languages are in the separate repositories: + +* [nebula-go](https://github.com/vesoft-inc/nebula-go) +* [nebula-java](https://github.com/vesoft-inc/nebula-java) +* [nebula-python](https://github.com/vesoft-inc/nebula-python) +* [nebula-cpp](https://github.com/vesoft-inc/nebula-cpp) +* [nebula-rust](https://github.com/vesoft-inc/nebula-rust) +* [nebula-node](https://github.com/vesoft-inc/nebula-node) diff --git a/src/clients/meta/CMakeLists.txt b/src/clients/meta/CMakeLists.txt new file mode 100644 index 0000000..c5175f5 --- /dev/null +++ b/src/clients/meta/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + meta_client_obj OBJECT + MetaClient.cpp +) + +nebula_add_library( + file_based_cluster_id_man_obj OBJECT + FileBasedClusterIdMan.cpp +) + +nebula_add_subdirectory(stats) +nebula_add_subdirectory(test) diff --git a/src/clients/meta/FileBasedClusterIdMan.cpp b/src/clients/meta/FileBasedClusterIdMan.cpp new file mode 100644 index 0000000..7043729 --- /dev/null +++ b/src/clients/meta/FileBasedClusterIdMan.cpp @@ -0,0 +1,59 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/meta/FileBasedClusterIdMan.h" + +#include "common/base/Base.h" + +namespace nebula { +namespace meta { + +// static +bool FileBasedClusterIdMan::persistInFile(ClusterID clusterId, const std::string& filename) { + auto dirname = fs::FileUtils::dirname(filename.c_str()); + if (!fs::FileUtils::makeDir(dirname)) { + LOG(ERROR) << "Failed mkdir " << dirname; + return false; + } + if (fs::FileUtils::remove(filename.c_str())) { + LOG(INFO) << "Remove the existed file " << filename; + } + int fd = ::open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + LOG(ERROR) << "Open file error, file " << filename << ", error " << strerror(errno); + return false; + } + int bytes = ::write(fd, reinterpret_cast(&clusterId), sizeof(ClusterID)); + if (bytes != sizeof(clusterId)) { + LOG(ERROR) << "Write clusterId failed!"; + ::close(fd); + return false; + } + LOG(INFO) << "Persist clusterId " << clusterId << " succeeded!"; + ::close(fd); + return true; +} + +// static +ClusterID FileBasedClusterIdMan::getClusterIdFromFile(const std::string& filename) { + int fd = ::open(filename.c_str(), O_RDONLY); + if (fd < 0) { + LOG(WARNING) << "Open file failed, error " << strerror(errno); + return 0; + } + ClusterID clusterId = 0; + int len = ::read(fd, reinterpret_cast(&clusterId), sizeof(ClusterID)); + if (len != sizeof(ClusterID)) { + LOG(ERROR) << "Get clusterId failed!"; + ::close(fd); + return 0; + } + LOG(INFO) << "Get clusterId: " << clusterId; + ::close(fd); + return clusterId; +} + +} // namespace meta +} // namespace nebula diff --git a/src/clients/meta/FileBasedClusterIdMan.h b/src/clients/meta/FileBasedClusterIdMan.h new file mode 100644 index 0000000..5e95f15 --- /dev/null +++ b/src/clients/meta/FileBasedClusterIdMan.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_META_FILEBASEDCLUSTERIDMAN_H_ +#define CLIENTS_META_FILEBASEDCLUSTERIDMAN_H_ + +#include "common/base/Base.h" +#include "common/fs/FileUtils.h" +#include "common/meta/ClusterIdManBase.h" + +namespace nebula { +namespace meta { + +/** + * This class manages clusterId used for meta server and storage server. + * */ +class FileBasedClusterIdMan : public ClusterIdManBase { + public: + static bool persistInFile(ClusterID clusterId, const std::string& filename); + + static ClusterID getClusterIdFromFile(const std::string& filename); +}; + +} // namespace meta +} // namespace nebula +#endif // CLIENTS_META_FILEBASEDCLUSTERIDMAN_H_ diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp new file mode 100644 index 0000000..df4e3dc --- /dev/null +++ b/src/clients/meta/MetaClient.cpp @@ -0,0 +1,3898 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/meta/MetaClient.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "clients/meta/FileBasedClusterIdMan.h" +#include "clients/meta/stats/MetaClientStats.h" +#include "common/base/Base.h" +#include "common/base/MurmurHash2.h" +#include "common/conf/Configuration.h" +#include "common/http/HttpClient.h" +#include "common/meta/NebulaSchemaProvider.h" +#include "common/network/NetworkUtils.h" +#include "common/ssl/SSLConfig.h" +#include "common/stats/StatsManager.h" +#include "common/time/TimeUtils.h" +#include "version/Version.h" +#include "webservice/Common.h" + +DEFINE_uint32(expired_time_factor, 5, "The factor of expired time based on heart beat interval"); +DEFINE_int32(heartbeat_interval_secs, 10, "Heartbeat interval in seconds"); +DEFINE_int32(meta_client_retry_times, 3, "meta client retry times, 0 means no retry"); +DEFINE_int32(meta_client_retry_interval_secs, 1, "meta client sleep interval between retry"); +DEFINE_int32(meta_client_timeout_ms, 60 * 1000, "meta client timeout"); +DEFINE_string(cluster_id_path, "cluster.id", "file path saved clusterId"); +DEFINE_int32(check_plan_killed_frequency, 8, "check plan killed every 1< indexItemVec); + +MetaClient::MetaClient(std::shared_ptr ioThreadPool, + std::vector addrs, + const MetaClientOptions& options) + : ioThreadPool_(ioThreadPool), + addrs_(std::move(addrs)), + options_(options), + metadata_(new MetaData()) { + CHECK(ioThreadPool_ != nullptr) << "IOThreadPool is required"; + CHECK(!addrs_.empty()) + << "No meta server address is specified or can be solved. Meta server is required"; + clientsMan_ = std::make_shared>( + FLAGS_enable_ssl || FLAGS_enable_meta_ssl); + updateActive(); + updateLeader(); + bgThread_ = std::make_unique(); + LOG(INFO) << "Create meta client to " << active_; + LOG(INFO) << folly::sformat( + "root path: {}, data path size: {}", options_.rootPath_, options_.dataPaths_.size()); +} + +MetaClient::~MetaClient() { + notifyStop(); + stop(); + delete metadata_.load(); + VLOG(3) << "~MetaClient"; +} + +#ifdef BUILD_STANDALONE +StatusOr MetaClient::checkLocalMachineRegistered() { + auto ret = heartbeat().get(); + if (!ret.ok()) { + if (ret.status().toString() == "Machine not existed!") { + return false; + } + LOG(ERROR) << "Check register failed: " << ret.status(); + return Status::Error("Check register failed!"); + } + return true; +} +#endif + +bool MetaClient::isMetadReady() { + // UNKNOWN is reserved for tools such as upgrader, in that case the ip/port is not set. We do + // not send heartbeat to meta to avoid writing error host info (e.g. Host("", 0)) + if (options_.role_ != cpp2::HostRole::UNKNOWN) { + auto ret = heartbeat().get(); + if (!ret.ok()) { + LOG(ERROR) << "Heartbeat failed, status:" << ret.status(); + return ready_; + } else if (options_.role_ == cpp2::HostRole::STORAGE && + metaServerVersion_ != EXPECT_META_VERSION) { + LOG(ERROR) << "Expect meta version is " << EXPECT_META_VERSION << ", but actual is " + << metaServerVersion_; + return ready_; + } + } + + // ready_ will be set in loadData + loadData(); + loadCfg(); + return ready_; +} + +bool MetaClient::waitForMetadReady(int count, int retryIntervalSecs) { + if (!options_.skipConfig_) { + std::string gflagsJsonPath; + GflagsManager::getGflagsModule(gflagsModule_); + gflagsDeclared_ = GflagsManager::declareGflags(gflagsModule_); + } + isRunning_ = true; + int tryCount = count; + while (!isMetadReady() && ((count == -1) || (tryCount > 0)) && isRunning_) { + LOG(INFO) << "Waiting for the metad to be ready!"; + --tryCount; + ::sleep(retryIntervalSecs); + } // end while + + if (!isRunning_) { + LOG(ERROR) << "Connect to the MetaServer Failed"; + return false; + } + + // Verify the graph server version + auto status = verifyVersion(); + if (!status.ok()) { + LOG(ERROR) << status; + return false; + } + + // Save graph version to meta + status = saveVersionToMeta(); + if (!status.ok()) { + LOG(ERROR) << status; + return false; + } + + CHECK(bgThread_->start()); + LOG(INFO) << "Register time task for heartbeat!"; + size_t delayMS = FLAGS_heartbeat_interval_secs * 1000 + folly::Random::rand32(900); + bgThread_->addDelayTask(delayMS, &MetaClient::heartBeatThreadFunc, this); + return ready_; +} + +void MetaClient::notifyStop() { + if (bgThread_ != nullptr) { + bgThread_->stop(); + } + isRunning_ = false; +} + +void MetaClient::stop() { + if (bgThread_ != nullptr) { + bgThread_->wait(); + bgThread_.reset(); + } +} + +void MetaClient::heartBeatThreadFunc() { + SCOPE_EXIT { + bgThread_->addDelayTask( + FLAGS_heartbeat_interval_secs * 1000, &MetaClient::heartBeatThreadFunc, this); + }; + // UNKNOWN is reserved for tools such as upgrader, in that case the ip/port is not set. We do + // not send heartbeat to meta to avoid writing error host info (e.g. Host("", 0)) + if (options_.role_ != cpp2::HostRole::UNKNOWN) { + auto ret = heartbeat().get(); + if (!ret.ok()) { + LOG(ERROR) << "Heartbeat failed, status:" << ret.status(); + return; + } + } + + // if MetaServer has some changes, refresh the localCache_ + loadData(); + loadCfg(); +} + +bool MetaClient::loadUsersAndRoles() { + auto userRoleRet = listUsers().get(); + if (!userRoleRet.ok()) { + LOG(ERROR) << "List users failed, status:" << userRoleRet.status(); + return false; + } + decltype(userRolesMap_) userRolesMap; + decltype(userPasswordMap_) userPasswordMap; + // List of username + std::unordered_set userNameList; + + for (auto& user : userRoleRet.value()) { + auto rolesRet = getUserRoles(user.first).get(); + if (!rolesRet.ok()) { + LOG(ERROR) << "List role by user failed, user : " << user.first; + return false; + } + userRolesMap[user.first] = rolesRet.value(); + userPasswordMap[user.first] = user.second; + userNameList.emplace(user.first); + } + + userRolesMap_ = std::move(userRolesMap); + userPasswordMap_ = std::move(userPasswordMap); + + // Remove expired users from cache + auto removeExpiredUser = [&](folly::ConcurrentHashMap& userMap, + const std::unordered_set& userList) { + for (auto iter = userMap.begin(); iter != userMap.end();) { + if (!userList.count(iter->first)) { + iter = userMap.erase(iter); + } else { + ++iter; + } + } + }; + removeExpiredUser(userPasswordAttemptsRemain_, userNameList); + removeExpiredUser(userLoginLockTime_, userNameList); + + // This method is called periodically by the heartbeat thread, but we don't want to reset the + // failed login attempts every time. + for (const auto& user : userNameList) { + // If the user is not in the map, insert value with the default value + // Do nothing if the account is already in the map + if (userPasswordAttemptsRemain_.find(user) == userPasswordAttemptsRemain_.end()) { + userPasswordAttemptsRemain_.insert(user, FLAGS_failed_login_attempts); + } + if (userLoginLockTime_.find(user) == userLoginLockTime_.end()) { + userLoginLockTime_.insert(user, 0); + } + } + return true; +} + +bool MetaClient::loadData() { + memory::MemoryCheckOffGuard g; + // UNKNOWN role will skip heartbeat + if (options_.role_ != cpp2::HostRole::UNKNOWN && + localDataLastUpdateTime_ == metadLastUpdateTime_) { + return true; + } + + if (ioThreadPool_->numThreads() <= 0) { + LOG(ERROR) << "The threads number in ioThreadPool should be greater than 0"; + return false; + } + + if (!loadUsersAndRoles()) { + LOG(ERROR) << "Load roles Failed"; + return false; + } + + if (!loadGlobalServiceClients()) { + LOG(ERROR) << "Load global services Failed"; + return false; + } + + if (!loadFulltextIndexes()) { + LOG(ERROR) << "Load fulltext indexes Failed"; + return false; + } + + if (!loadSessions()) { + LOG(ERROR) << "Load sessions Failed"; + return false; + } + + auto ret = listSpaces().get(); + if (!ret.ok()) { + LOG(ERROR) << "List space failed, status:" << ret.status(); + return false; + } + + decltype(localCache_) cache; + decltype(spaceIndexByName_) spaceIndexByName; + decltype(spaceTagIndexByName_) spaceTagIndexByName; + decltype(spaceEdgeIndexByName_) spaceEdgeIndexByName; + decltype(spaceNewestTagVerMap_) spaceNewestTagVerMap; + decltype(spaceNewestEdgeVerMap_) spaceNewestEdgeVerMap; + decltype(spaceEdgeIndexByType_) spaceEdgeIndexByType; + decltype(spaceTagIndexById_) spaceTagIndexById; + decltype(spaceAllEdgeMap_) spaceAllEdgeMap; + + for (auto space : ret.value()) { + auto spaceId = space.first; + MetaClient::PartTerms partTerms; + auto r = getPartsAlloc(spaceId, &partTerms).get(); + if (!r.ok()) { + LOG(ERROR) << "Get parts allocation failed for spaceId " << spaceId << ", status " + << r.status(); + return false; + } + + auto spaceCache = std::make_shared(); + auto partsAlloc = r.value(); + auto& spaceName = space.second; + spaceCache->partsOnHost_ = reverse(partsAlloc); + spaceCache->partsAlloc_ = std::move(partsAlloc); + spaceCache->termOfPartition_ = std::move(partTerms); + VLOG(2) << "Load space " << spaceId << ", parts num:" << spaceCache->partsAlloc_.size(); + + // loadSchemas + if (!loadSchemas(spaceId, + spaceCache, + spaceTagIndexByName, + spaceTagIndexById, + spaceEdgeIndexByName, + spaceEdgeIndexByType, + spaceNewestTagVerMap, + spaceNewestEdgeVerMap, + spaceAllEdgeMap)) { + LOG(ERROR) << "Load Schemas Failed"; + return false; + } + + if (!loadIndexes(spaceId, spaceCache)) { + LOG(ERROR) << "Load Indexes Failed"; + return false; + } + + if (!loadListeners(spaceId, spaceCache)) { + LOG(ERROR) << "Load Listeners Failed"; + return false; + } + + // get space properties + auto resp = getSpace(spaceName).get(); + if (!resp.ok()) { + LOG(ERROR) << "Get space properties failed for space " << spaceId; + return false; + } + auto properties = resp.value().get_properties(); + spaceCache->spaceDesc_ = std::move(properties); + + cache.emplace(spaceId, spaceCache); + spaceIndexByName.emplace(space.second, spaceId); + } + + auto hostsRet = listHosts().get(); + if (!hostsRet.ok()) { + LOG(ERROR) << "List hosts failed, status:" << hostsRet.status(); + return false; + } + + auto& hostItems = hostsRet.value(); + std::vector hosts(hostItems.size()); + std::transform(hostItems.begin(), hostItems.end(), hosts.begin(), [](auto& hostItem) -> HostAddr { + return *hostItem.hostAddr_ref(); + }); + + decltype(localCache_) oldCache; + { + oldCache = std::move(localCache_); + localCache_ = std::move(cache); + spaceIndexByName_ = std::move(spaceIndexByName); + spaceTagIndexByName_ = std::move(spaceTagIndexByName); + spaceEdgeIndexByName_ = std::move(spaceEdgeIndexByName); + spaceNewestTagVerMap_ = std::move(spaceNewestTagVerMap); + spaceNewestEdgeVerMap_ = std::move(spaceNewestEdgeVerMap); + spaceEdgeIndexByType_ = std::move(spaceEdgeIndexByType); + spaceTagIndexById_ = std::move(spaceTagIndexById); + spaceAllEdgeMap_ = std::move(spaceAllEdgeMap); + storageHosts_ = std::move(hosts); + } + + loadLeader(hostItems, spaceIndexByName_); + + localDataLastUpdateTime_.store(metadLastUpdateTime_.load()); + auto newMetaData = new MetaData(); + + for (auto& spaceInfo : localCache_) { + GraphSpaceID spaceId = spaceInfo.first; + std::shared_ptr info = spaceInfo.second; + std::shared_ptr infoDeepCopy = std::make_shared(*info); + infoDeepCopy->tagSchemas_ = buildTagSchemas(infoDeepCopy->tagItemVec_); + infoDeepCopy->edgeSchemas_ = buildEdgeSchemas(infoDeepCopy->edgeItemVec_); + infoDeepCopy->tagIndexes_ = buildIndexes(infoDeepCopy->tagIndexItemVec_); + infoDeepCopy->edgeIndexes_ = buildIndexes(infoDeepCopy->edgeIndexItemVec_); + newMetaData->localCache_[spaceId] = infoDeepCopy; + } + newMetaData->spaceIndexByName_ = spaceIndexByName_; + newMetaData->spaceTagIndexByName_ = spaceTagIndexByName_; + newMetaData->spaceEdgeIndexByName_ = spaceEdgeIndexByName_; + newMetaData->spaceEdgeIndexByType_ = spaceEdgeIndexByType_; + newMetaData->spaceNewestTagVerMap_ = spaceNewestTagVerMap_; + newMetaData->spaceNewestEdgeVerMap_ = spaceNewestEdgeVerMap_; + newMetaData->spaceTagIndexById_ = spaceTagIndexById_; + newMetaData->spaceAllEdgeMap_ = spaceAllEdgeMap_; + + newMetaData->userRolesMap_ = userRolesMap_; + newMetaData->storageHosts_ = storageHosts_; + newMetaData->fulltextIndexMap_ = fulltextIndexMap_; + newMetaData->userPasswordMap_ = userPasswordMap_; + newMetaData->sessionMap_ = std::move(sessionMap_); + newMetaData->killedPlans_ = std::move(killedPlans_); + newMetaData->serviceClientList_ = std::move(serviceClientList_); + auto oldMetaData = metadata_.load(); + metadata_.store(newMetaData); + folly::rcu_retire(oldMetaData); + diff(oldCache, localCache_); + listenerDiff(oldCache, localCache_); + loadRemoteListeners(); + ready_ = true; + return true; +} + +TagSchemas MetaClient::buildTagSchemas(std::vector tagItemVec) { + memory::MemoryCheckOffGuard g; + TagSchemas tagSchemas; + for (auto& tagIt : tagItemVec) { + // meta will return the different version from new to old + auto schema = std::make_shared(tagIt.get_version()); + for (const auto& colIt : tagIt.get_schema().get_columns()) { + addSchemaField(schema.get(), colIt); + } + // handle schema property + schema->setProp(tagIt.get_schema().get_schema_prop()); + auto& schemas = tagSchemas[tagIt.get_tag_id()]; + // Because of the byte order of schema version in meta is not same as numerical order, we have + // to check schema version + if (schemas.size() <= static_cast(schema->getVersion())) { + // since schema version is zero-based, need to add one + schemas.resize(schema->getVersion() + 1); + } + schemas[schema->getVersion()] = std::move(schema); + } + return tagSchemas; +} + +EdgeSchemas MetaClient::buildEdgeSchemas(std::vector edgeItemVec) { + memory::MemoryCheckOffGuard g; + EdgeSchemas edgeSchemas; + std::unordered_set> edges; + for (auto& edgeIt : edgeItemVec) { + // meta will return the different version from new to old + auto schema = std::make_shared(edgeIt.get_version()); + for (const auto& col : edgeIt.get_schema().get_columns()) { + MetaClient::addSchemaField(schema.get(), col); + } + // handle shcem property + schema->setProp(edgeIt.get_schema().get_schema_prop()); + auto& schemas = edgeSchemas[edgeIt.get_edge_type()]; + // Because of the byte order of schema version in meta is not same as numerical order, we have + // to check schema version + if (schemas.size() <= static_cast(schema->getVersion())) { + // since schema version is zero-based, need to add one + schemas.resize(schema->getVersion() + 1); + } + schemas[schema->getVersion()] = std::move(schema); + } + return edgeSchemas; +} + +void MetaClient::addSchemaField(NebulaSchemaProvider* schema, const cpp2::ColumnDef& col) { + memory::MemoryCheckOffGuard g; + bool hasDef = col.default_value_ref().has_value(); + auto& colType = col.get_type(); + size_t len = colType.type_length_ref().has_value() ? *colType.get_type_length() : 0; + cpp2::GeoShape geoShape = + colType.geo_shape_ref().has_value() ? *colType.get_geo_shape() : cpp2::GeoShape::ANY; + bool nullable = col.nullable_ref().has_value() ? *col.get_nullable() : false; + std::string encoded; + if (hasDef) { + encoded = *col.get_default_value(); + } + + schema->addField(col.get_name(), colType.get_type(), len, nullable, encoded, geoShape); +} + +bool MetaClient::loadSchemas(GraphSpaceID spaceId, + std::shared_ptr spaceInfoCache, + SpaceTagNameIdMap& tagNameIdMap, + SpaceTagIdNameMap& tagIdNameMap, + SpaceEdgeNameTypeMap& edgeNameTypeMap, + SpaceEdgeTypeNameMap& edgeTypeNameMap, + SpaceNewestTagVerMap& newestTagVerMap, + SpaceNewestEdgeVerMap& newestEdgeVerMap, + SpaceAllEdgeMap& allEdgeMap) { + memory::MemoryCheckOffGuard g; + auto tagRet = listTagSchemas(spaceId).get(); + if (!tagRet.ok()) { + LOG(ERROR) << "Get tag schemas failed for spaceId " << spaceId << ", " << tagRet.status(); + return false; + } + + auto edgeRet = listEdgeSchemas(spaceId).get(); + if (!edgeRet.ok()) { + LOG(ERROR) << "Get edge schemas failed for spaceId " << spaceId << ", " << edgeRet.status(); + return false; + } + + auto tagItemVec = tagRet.value(); + auto edgeItemVec = edgeRet.value(); + allEdgeMap[spaceId] = {}; + spaceInfoCache->tagItemVec_ = tagItemVec; + spaceInfoCache->tagSchemas_ = buildTagSchemas(tagItemVec); + spaceInfoCache->edgeItemVec_ = edgeItemVec; + spaceInfoCache->edgeSchemas_ = buildEdgeSchemas(edgeItemVec); + + for (auto& tagIt : tagItemVec) { + tagNameIdMap.emplace(std::make_pair(spaceId, tagIt.get_tag_name()), tagIt.get_tag_id()); + tagIdNameMap.emplace(std::make_pair(spaceId, tagIt.get_tag_id()), tagIt.get_tag_name()); + // get the latest tag version + auto it = newestTagVerMap.find(std::make_pair(spaceId, tagIt.get_tag_id())); + if (it != newestTagVerMap.end()) { + if (it->second < tagIt.get_version()) { + it->second = tagIt.get_version(); + } + } else { + newestTagVerMap.emplace(std::make_pair(spaceId, tagIt.get_tag_id()), tagIt.get_version()); + } + VLOG(3) << "Load Tag Schema Space " << spaceId << ", ID " << tagIt.get_tag_id() << ", Name " + << tagIt.get_tag_name() << ", Version " << tagIt.get_version() << " Successfully!"; + } + + std::unordered_set> edges; + for (auto& edgeIt : edgeItemVec) { + edgeNameTypeMap.emplace(std::make_pair(spaceId, edgeIt.get_edge_name()), + edgeIt.get_edge_type()); + edgeTypeNameMap.emplace(std::make_pair(spaceId, edgeIt.get_edge_type()), + edgeIt.get_edge_name()); + if (edges.find({spaceId, edgeIt.get_edge_type()}) != edges.cend()) { + continue; + } + edges.emplace(spaceId, edgeIt.get_edge_type()); + allEdgeMap[spaceId].emplace_back(edgeIt.get_edge_name()); + // get the latest edge version + auto it2 = newestEdgeVerMap.find(std::make_pair(spaceId, edgeIt.get_edge_type())); + if (it2 != newestEdgeVerMap.end()) { + if (it2->second < edgeIt.get_version()) { + it2->second = edgeIt.get_version(); + } + } else { + newestEdgeVerMap.emplace(std::make_pair(spaceId, edgeIt.get_edge_type()), + edgeIt.get_version()); + } + VLOG(3) << "Load Edge Schema Space " << spaceId << ", Type " << edgeIt.get_edge_type() + << ", Name " << edgeIt.get_edge_name() << ", Version " << edgeIt.get_version() + << " Successfully!"; + } + + return true; +} + +Indexes buildIndexes(std::vector indexItemVec) { + memory::MemoryCheckOffGuard g; + Indexes indexes; + for (auto index : indexItemVec) { + auto indexName = index.get_index_name(); + auto indexID = index.get_index_id(); + auto indexPtr = std::make_shared(index); + indexes.emplace(indexID, indexPtr); + } + return indexes; +} + +bool MetaClient::loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; + auto tagIndexesRet = listTagIndexes(spaceId).get(); + if (!tagIndexesRet.ok()) { + LOG(ERROR) << "Get tag indexes failed for spaceId " << spaceId << ", " + << tagIndexesRet.status(); + return false; + } + + auto edgeIndexesRet = listEdgeIndexes(spaceId).get(); + if (!edgeIndexesRet.ok()) { + LOG(ERROR) << "Get edge indexes failed for spaceId " << spaceId << ", " + << edgeIndexesRet.status(); + return false; + } + + auto tagIndexItemVec = tagIndexesRet.value(); + cache->tagIndexItemVec_ = tagIndexItemVec; + cache->tagIndexes_ = buildIndexes(tagIndexItemVec); + for (const auto& tagIndex : tagIndexItemVec) { + auto indexName = tagIndex.get_index_name(); + auto indexID = tagIndex.get_index_id(); + std::pair pair(spaceId, indexName); + tagNameIndexMap_[pair] = indexID; + } + + auto edgeIndexItemVec = edgeIndexesRet.value(); + cache->edgeIndexItemVec_ = edgeIndexItemVec; + cache->edgeIndexes_ = buildIndexes(edgeIndexItemVec); + for (auto& edgeIndex : edgeIndexItemVec) { + auto indexName = edgeIndex.get_index_name(); + auto indexID = edgeIndex.get_index_id(); + std::pair pair(spaceId, indexName); + edgeNameIndexMap_[pair] = indexID; + } + return true; +} + +bool MetaClient::loadListeners(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; + auto listenerRet = listListener(spaceId).get(); + if (!listenerRet.ok()) { + LOG(ERROR) << "Get listeners failed for spaceId " << spaceId << ", " << listenerRet.status(); + return false; + } + Listeners listeners; + for (auto& listener : listenerRet.value()) { + listeners[listener.get_host()].emplace_back( + std::make_pair(listener.get_part_id(), listener.get_type())); + } + cache->listeners_ = std::move(listeners); + return true; +} + +bool MetaClient::loadGlobalServiceClients() { + memory::MemoryCheckOffGuard g; + auto ret = listServiceClients(cpp2::ExternalServiceType::ELASTICSEARCH).get(); + if (!ret.ok()) { + LOG(ERROR) << "List services failed, status:" << ret.status(); + return false; + } + serviceClientList_ = std::move(ret).value(); + return true; +} + +bool MetaClient::loadFulltextIndexes() { + memory::MemoryCheckOffGuard g; + auto ftRet = listFTIndexes().get(); + if (!ftRet.ok()) { + LOG(ERROR) << "List fulltext indexes failed, status:" << ftRet.status(); + return false; + } + fulltextIndexMap_ = std::move(ftRet).value(); + return true; +} + +Status MetaClient::checkTagIndexed(GraphSpaceID spaceId, IndexID indexID) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { + auto indexIt = it->second->tagIndexes_.find(indexID); + if (indexIt != it->second->tagIndexes_.end()) { + return Status::OK(); + } else { + return Status::IndexNotFound(); + } + } + return Status::SpaceNotFound(); +} + +Status MetaClient::checkEdgeIndexed(GraphSpaceID space, IndexID indexID) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(space); + if (it != metadata.localCache_.end()) { + auto indexIt = it->second->edgeIndexes_.find(indexID); + if (indexIt != it->second->edgeIndexes_.end()) { + return Status::OK(); + } else { + return Status::IndexNotFound(); + } + } + return Status::SpaceNotFound(); +} + +std::unordered_map> MetaClient::reverse( + const PartsAlloc& parts) { + memory::MemoryCheckOffGuard g; + std::unordered_map> hosts; + for (auto& partHost : parts) { + for (auto& h : partHost.second) { + hosts[h].emplace_back(partHost.first); + } + } + return hosts; +} + +template +void MetaClient::getResponse(Request req, + RemoteFunc remoteFunc, + RespGenerator respGen, + folly::Promise> pro, + bool toLeader, + int32_t retry, + int32_t retryLimit) { + memory::MemoryCheckOffGuard g; + stats::StatsManager::addValue(kNumRpcSentToMetad); + auto* evb = ioThreadPool_->getEventBase(); + HostAddr host; + { + folly::SharedMutex::ReadHolder holder(&hostLock_); + host = toLeader ? leader_ : active_; + } + folly::via(evb, + [host, + evb, + req = std::move(req), + remoteFunc = std::move(remoteFunc), + respGen = std::move(respGen), + pro = std::move(pro), + toLeader, + retry, + retryLimit, + this]() mutable { + auto client = clientsMan_->client(host, evb, false, FLAGS_meta_client_timeout_ms); + VLOG(1) << "Send request to meta " << host; + remoteFunc(client, req) + .via(evb) + .then([host, + req = std::move(req), + remoteFunc = std::move(remoteFunc), + respGen = std::move(respGen), + pro = std::move(pro), + toLeader, + retry, + retryLimit, + evb, + this](folly::Try&& t) mutable { + // exception occurred during RPC + if (t.hasException()) { + stats::StatsManager::addValue(kNumRpcSentToMetadFailed); + if (toLeader) { + updateLeader(); + } else { + updateActive(); + } + if (retry < retryLimit) { + evb->runAfterDelay( + [req = std::move(req), + remoteFunc = std::move(remoteFunc), + respGen = std::move(respGen), + pro = std::move(pro), + toLeader, + retry, + retryLimit, + this]() mutable { + getResponse(std::move(req), + std::move(remoteFunc), + std::move(respGen), + std::move(pro), + toLeader, + retry + 1, + retryLimit); + }, + FLAGS_meta_client_retry_interval_secs * 1000); + return; + } else { + LOG(ERROR) << "Send request to " << host << ", exceed retry limit"; + LOG(ERROR) << "RpcResponse exception: " << t.exception().what().c_str(); + pro.setValue(Status::Error("RPC failure in MetaClient: %s", + t.exception().what().c_str())); + } + return; + } + + auto&& resp = t.value(); + auto code = resp.get_code(); + if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { + // succeeded + pro.setValue(respGen(std::move(resp))); + return; + } else if (code == nebula::cpp2::ErrorCode::E_LEADER_CHANGED || + code == nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND) { + updateLeader(resp.get_leader()); + if (retry < retryLimit) { + evb->runAfterDelay( + [req = std::move(req), + remoteFunc = std::move(remoteFunc), + respGen = std::move(respGen), + pro = std::move(pro), + toLeader, + retry, + retryLimit, + this]() mutable { + getResponse(std::move(req), + std::move(remoteFunc), + std::move(respGen), + std::move(pro), + toLeader, + retry + 1, + retryLimit); + }, + FLAGS_meta_client_retry_interval_secs * 1000); + return; + } + } else if (code == nebula::cpp2::ErrorCode::E_CLIENT_SERVER_INCOMPATIBLE) { + pro.setValue(respGen(std::move(resp))); + return; + } + pro.setValue(this->handleResponse(resp)); + }); // then + }); // via +} + +std::vector MetaClient::toSpaceIdName(const std::vector& tIdNames) { + memory::MemoryCheckOffGuard g; + std::vector idNames; + idNames.resize(tIdNames.size()); + std::transform(tIdNames.begin(), tIdNames.end(), idNames.begin(), [](const auto& tin) { + return SpaceIdName(tin.get_id().get_space_id(), tin.get_name()); + }); + return idNames; +} + +template +Status MetaClient::handleResponse(const RESP& resp) { + memory::MemoryCheckOffGuard g; + switch (resp.get_code()) { + case nebula::cpp2::ErrorCode::SUCCEEDED: + return Status::OK(); + case nebula::cpp2::ErrorCode::E_DISCONNECTED: + return Status::Error("Disconnected!"); + case nebula::cpp2::ErrorCode::E_FAIL_TO_CONNECT: + return Status::Error("Fail to connect!"); + case nebula::cpp2::ErrorCode::E_RPC_FAILURE: + return Status::Error("Rpc failure, probably timeout!"); + case nebula::cpp2::ErrorCode::E_LEADER_CHANGED: + return Status::LeaderChanged("Leader changed!"); + case nebula::cpp2::ErrorCode::E_NO_HOSTS: + return Status::Error("No hosts!"); + case nebula::cpp2::ErrorCode::E_EXISTED: + return Status::Error("Existed!"); + case nebula::cpp2::ErrorCode::E_HISTORY_CONFLICT: + return Status::Error("Schema exisited before!"); + case nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND: + return Status::SpaceNotFound("Space not existed!"); + case nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND: + return Status::TagNotFound("Tag not existed!"); + case nebula::cpp2::ErrorCode::E_EDGE_NOT_FOUND: + return Status::EdgeNotFound("Edge not existed!"); + case nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND: + return Status::IndexNotFound("Index not existed!"); + case nebula::cpp2::ErrorCode::E_STATS_NOT_FOUND: + return Status::Error( + "There is no any stats info to show, please execute " + "`submit job stats' firstly!"); + case nebula::cpp2::ErrorCode::E_EDGE_PROP_NOT_FOUND: + return Status::Error("Edge prop not existed!"); + case nebula::cpp2::ErrorCode::E_TAG_PROP_NOT_FOUND: + return Status::Error("Tag prop not existed!"); + case nebula::cpp2::ErrorCode::E_ROLE_NOT_FOUND: + return Status::Error("Role not existed!"); + case nebula::cpp2::ErrorCode::E_CONFIG_NOT_FOUND: + return Status::Error("Conf not existed!"); + case nebula::cpp2::ErrorCode::E_PART_NOT_FOUND: + return Status::Error("Part not existed!"); + case nebula::cpp2::ErrorCode::E_USER_NOT_FOUND: + return Status::Error("User not existed!"); + case nebula::cpp2::ErrorCode::E_MACHINE_NOT_FOUND: + return Status::Error("Machine not existed!"); + case nebula::cpp2::ErrorCode::E_ZONE_NOT_FOUND: + return Status::Error("Zone not existed!"); + case nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND: + return Status::Error("Key not existed!"); + case nebula::cpp2::ErrorCode::E_INVALID_HOST: + return Status::Error("Invalid host!"); + case nebula::cpp2::ErrorCode::E_UNSUPPORTED: + return Status::Error("Unsupported!"); + case nebula::cpp2::ErrorCode::E_NOT_DROP: + return Status::Error("Not allowed to drop!"); + case nebula::cpp2::ErrorCode::E_BALANCER_RUNNING: + return Status::Error("The balancer is running!"); + case nebula::cpp2::ErrorCode::E_CONFIG_IMMUTABLE: + return Status::Error("Config immutable!"); + case nebula::cpp2::ErrorCode::E_INVALID_PARM: + return Status::Error("Invalid param!"); + case nebula::cpp2::ErrorCode::E_WRONGCLUSTER: + return Status::Error("Wrong cluster!"); + case nebula::cpp2::ErrorCode::E_ZONE_NOT_ENOUGH: + return Status::Error("Host not enough!"); + case nebula::cpp2::ErrorCode::E_ZONE_IS_EMPTY: + return Status::Error("Host not exist!"); + case nebula::cpp2::ErrorCode::E_STORE_FAILURE: + return Status::Error("Store failure!"); + case nebula::cpp2::ErrorCode::E_BAD_BALANCE_PLAN: + return Status::Error("Bad balance plan!"); + case nebula::cpp2::ErrorCode::E_BALANCED: + return Status::Error("The cluster is balanced!"); + case nebula::cpp2::ErrorCode::E_NO_RUNNING_BALANCE_PLAN: + return Status::Error("No running balance plan!"); + case nebula::cpp2::ErrorCode::E_NO_VALID_HOST: + return Status::Error("No valid host hold the partition!"); + case nebula::cpp2::ErrorCode::E_CORRUPTED_BALANCE_PLAN: + return Status::Error("No corrupted balance plan!"); + case nebula::cpp2::ErrorCode::E_INVALID_PASSWORD: + return Status::Error("Invalid password!"); + case nebula::cpp2::ErrorCode::E_IMPROPER_ROLE: + return Status::Error("Improper role!"); + case nebula::cpp2::ErrorCode::E_INVALID_PARTITION_NUM: + return Status::Error("No valid partition_num!"); + case nebula::cpp2::ErrorCode::E_INVALID_REPLICA_FACTOR: + return Status::Error("No valid replica_factor!"); + case nebula::cpp2::ErrorCode::E_INVALID_CHARSET: + return Status::Error("No valid charset!"); + case nebula::cpp2::ErrorCode::E_INVALID_COLLATE: + return Status::Error("No valid collate!"); + case nebula::cpp2::ErrorCode::E_CHARSET_COLLATE_NOT_MATCH: + return Status::Error("Charset and collate not match!"); + case nebula::cpp2::ErrorCode::E_SNAPSHOT_FAILURE: + return Status::Error("Snapshot failure!"); + case nebula::cpp2::ErrorCode::E_SNAPSHOT_RUNNING_JOBS: + return Status::Error("Snapshot failed encounter running jobs!"); + case nebula::cpp2::ErrorCode::E_SNAPSHOT_NOT_FOUND: + return Status::Error("Snapshot not found!"); + case nebula::cpp2::ErrorCode::E_BLOCK_WRITE_FAILURE: + return Status::Error("Block write failure!"); + case nebula::cpp2::ErrorCode::E_REBUILD_INDEX_FAILED: + return Status::Error("Rebuild index failed!"); + case nebula::cpp2::ErrorCode::E_INDEX_WITH_TTL: + return Status::Error("Index with ttl!"); + case nebula::cpp2::ErrorCode::E_ADD_JOB_FAILURE: + return Status::Error("Add job failure!"); + case nebula::cpp2::ErrorCode::E_STOP_JOB_FAILURE: + return Status::Error("Stop job failure!"); + case nebula::cpp2::ErrorCode::E_SAVE_JOB_FAILURE: + return Status::Error("Save job failure!"); + case nebula::cpp2::ErrorCode::E_JOB_ALREADY_FINISH: + return Status::Error( + "Finished job or failed job can not be stopped, please start another job instead"); + case nebula::cpp2::ErrorCode::E_JOB_NOT_STOPPABLE: + return Status::Error( + "The job type do not support stopping, either wait previous job done or restart the " + "cluster to start another job"); + case nebula::cpp2::ErrorCode::E_BALANCER_FAILURE: + return Status::Error("Balance failure!"); + case nebula::cpp2::ErrorCode::E_NO_INVALID_BALANCE_PLAN: + return Status::Error("No invalid balance plan!"); + case nebula::cpp2::ErrorCode::E_JOB_NOT_FINISHED: + return Status::Error("Job is not finished!"); + case nebula::cpp2::ErrorCode::E_TASK_REPORT_OUT_DATE: + return Status::Error("Task report is out of date!"); + case nebula::cpp2::ErrorCode::E_BACKUP_FAILED: + return Status::Error("Backup failure!"); + case nebula::cpp2::ErrorCode::E_BACKUP_RUNNING_JOBS: + return Status::Error("Backup encounter running or queued jobs!"); + case nebula::cpp2::ErrorCode::E_BACKUP_SPACE_NOT_FOUND: + return Status::Error("The space is not found when backup!"); + case nebula::cpp2::ErrorCode::E_RESTORE_FAILURE: + return Status::Error("Restore failure!"); + case nebula::cpp2::ErrorCode::E_LIST_CLUSTER_FAILURE: + return Status::Error("list cluster failure!"); + case nebula::cpp2::ErrorCode::E_LIST_CLUSTER_GET_ABS_PATH_FAILURE: + return Status::Error("Failed to get the absolute path!"); + case nebula::cpp2::ErrorCode::E_LIST_CLUSTER_NO_AGENT_FAILURE: + return Status::Error("There is no agent!"); + case nebula::cpp2::ErrorCode::E_INVALID_JOB: + return Status::Error("No valid job!"); + case nebula::cpp2::ErrorCode::E_JOB_NOT_IN_SPACE: + return Status::Error("Job not existed in chosen space!"); + case nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER: + return Status::Error("Need to recover or stop failed data balance job firstly!"); + case nebula::cpp2::ErrorCode::E_BACKUP_EMPTY_TABLE: + return Status::Error("Backup empty table!"); + case nebula::cpp2::ErrorCode::E_BACKUP_TABLE_FAILED: + return Status::Error("Backup table failure!"); + case nebula::cpp2::ErrorCode::E_SESSION_NOT_FOUND: + return Status::Error("Session not existed!"); + case nebula::cpp2::ErrorCode::E_SCHEMA_NAME_EXISTS: + return Status::Error("Schema with same name exists"); + case nebula::cpp2::ErrorCode::E_RELATED_INDEX_EXISTS: + return Status::Error("Related index exists, please drop index first"); + case nebula::cpp2::ErrorCode::E_RELATED_SPACE_EXISTS: + return Status::Error("There are still space on the host"); + case nebula::cpp2::ErrorCode::E_RELATED_FULLTEXT_INDEX_EXISTS: + return Status::Error("Related fulltext index exists, please drop it first"); + case nebula::cpp2::ErrorCode::E_HOST_CAN_NOT_BE_ADDED: + return Status::Error("Could not add a host, which is not a storage and not expired either"); + case nebula::cpp2::ErrorCode::E_ACCESS_ES_FAILURE: + return Status::Error("Access elasticsearch failed"); + default: + return Status::Error("Unknown error %d!", static_cast(resp.get_code())); + } +} + +PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; + PartsMap partMap; + for (const auto& it : localCache) { + auto spaceId = it.first; + auto& cache = it.second; + auto partsIt = cache->partsOnHost_.find(host); + if (partsIt != cache->partsOnHost_.end()) { + for (auto& partId : partsIt->second) { + auto partAllocIter = cache->partsAlloc_.find(partId); + CHECK(partAllocIter != cache->partsAlloc_.end()); + auto& partM = partMap[spaceId][partId]; + partM.spaceId_ = spaceId; + partM.partId_ = partId; + partM.hosts_ = partAllocIter->second; + } + } + } + return partMap; +} + +void MetaClient::diff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; + folly::SharedMutex::WriteHolder holder(listenerLock_); + if (listener_ == nullptr) { + VLOG(3) << "Listener is null!"; + return; + } + auto newPartsMap = doGetPartsMap(options_.localHost_, newCache); + auto oldPartsMap = doGetPartsMap(options_.localHost_, oldCache); + VLOG(1) << "Let's check if any new parts added/updated for " << options_.localHost_; + for (auto& it : newPartsMap) { + auto spaceId = it.first; + const auto& newParts = it.second; + auto oldIt = oldPartsMap.find(spaceId); + if (oldIt == oldPartsMap.end()) { + VLOG(1) << "SpaceId " << spaceId << " was added!"; + listener_->onSpaceAdded(spaceId); + for (const auto& newPart : newParts) { + listener_->onPartAdded(newPart.second); + } + } else { + const auto& oldParts = oldIt->second; + for (const auto& newPart : newParts) { + auto oldPartIt = oldParts.find(newPart.first); + if (oldPartIt == oldParts.end()) { + VLOG(1) << "SpaceId " << spaceId << ", partId " << newPart.first << " was added!"; + listener_->onPartAdded(newPart.second); + } else { + const auto& oldPartHosts = oldPartIt->second; + const auto& newPartHosts = newPart.second; + if (oldPartHosts != newPartHosts) { + VLOG(1) << "SpaceId " << spaceId << ", partId " << newPart.first << " was updated!"; + listener_->onPartUpdated(newPartHosts); + } + } + } + } + } + VLOG(1) << "Let's check if any old parts removed...."; + for (auto& it : oldPartsMap) { + auto spaceId = it.first; + const auto& oldParts = it.second; + auto newIt = newPartsMap.find(spaceId); + if (newIt == newPartsMap.end()) { + VLOG(1) << "SpaceId " << spaceId << " was removed!"; + for (const auto& oldPart : oldParts) { + listener_->onPartRemoved(spaceId, oldPart.first); + } + listener_->onSpaceRemoved(spaceId); + } else { + const auto& newParts = newIt->second; + for (const auto& oldPart : oldParts) { + auto newPartIt = newParts.find(oldPart.first); + if (newPartIt == newParts.end()) { + VLOG(1) << "SpaceId " << spaceId << ", partId " << oldPart.first << " was removed!"; + listener_->onPartRemoved(spaceId, oldPart.first); + } + } + } + } +} + +void MetaClient::listenerDiff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; + folly::SharedMutex::WriteHolder holder(listenerLock_); + if (listener_ == nullptr) { + VLOG(3) << "Listener is null!"; + return; + } + auto newMap = doGetListenersMap(options_.localHost_, newCache); + auto oldMap = doGetListenersMap(options_.localHost_, oldCache); + if (newMap == oldMap) { + return; + } + + VLOG(1) << "Let's check if any listeners is updated for " << options_.localHost_; + for (auto& [spaceId, typeMap] : newMap) { + auto oldSpaceIter = oldMap.find(spaceId); + if (oldSpaceIter == oldMap.end()) { + // create all type of listener when new space listener added + for (const auto& [type, listenerParts] : typeMap) { + VLOG(1) << "[Listener] SpaceId " << spaceId << " was added, type is " + << apache::thrift::util::enumNameSafe(type); + listener_->onListenerSpaceAdded(spaceId, type); + for (const auto& newListener : listenerParts) { + VLOG(1) << "[Listener] SpaceId " << spaceId << ", partId " << newListener.partId_ + << " was added, type is " << apache::thrift::util::enumNameSafe(type); + listener_->onListenerPartAdded(spaceId, newListener.partId_, type, newListener.peers_); + } + } + } else { + for (auto& [type, listenerParts] : typeMap) { + auto oldTypeIter = oldSpaceIter->second.find(type); + // create missing type of listener when new type of listener added + if (oldTypeIter == oldSpaceIter->second.end()) { + VLOG(1) << "[Listener] SpaceId " << spaceId << " was added, type is " + << apache::thrift::util::enumNameSafe(type); + listener_->onListenerSpaceAdded(spaceId, type); + for (const auto& newListener : listenerParts) { + VLOG(1) << "[Listener] SpaceId " << spaceId << ", partId " << newListener.partId_ + << " was added, type is " << apache::thrift::util::enumNameSafe(type); + listener_->onListenerPartAdded(spaceId, newListener.partId_, type, newListener.peers_); + } + } else { + // create missing part of listener of specified type + std::sort(listenerParts.begin(), listenerParts.end()); + std::sort(oldTypeIter->second.begin(), oldTypeIter->second.end()); + std::vector diff; + std::set_difference(listenerParts.begin(), + listenerParts.end(), + oldTypeIter->second.begin(), + oldTypeIter->second.end(), + std::back_inserter(diff)); + for (const auto& newListener : diff) { + VLOG(1) << "[Listener] SpaceId " << spaceId << ", partId " << newListener.partId_ + << " was added, type is " << apache::thrift::util::enumNameSafe(type); + listener_->onListenerPartAdded(spaceId, newListener.partId_, type, newListener.peers_); + } + } + } + } + } + + VLOG(1) << "Let's check if any listeners is removed from " << options_.localHost_; + for (auto& [spaceId, typeMap] : oldMap) { + auto newSpaceIter = newMap.find(spaceId); + if (newSpaceIter == newMap.end()) { + // remove all type of listener when space listener removed + for (const auto& [type, listenerParts] : typeMap) { + for (const auto& outdateListener : listenerParts) { + VLOG(1) << "SpaceId " << spaceId << ", partId " << outdateListener.partId_ + << " was removed, type is " << apache::thrift::util::enumNameSafe(type); + listener_->onListenerPartRemoved(spaceId, outdateListener.partId_, type); + } + listener_->onListenerSpaceRemoved(spaceId, type); + VLOG(1) << "[Listener] SpaceId " << spaceId << " was removed, type is " + << apache::thrift::util::enumNameSafe(type); + } + } else { + for (auto& [type, listenerParts] : typeMap) { + auto newTypeIter = newSpaceIter->second.find(type); + // remove specified type of listener + if (newTypeIter == newSpaceIter->second.end()) { + for (const auto& outdateListener : listenerParts) { + VLOG(1) << "[Listener] SpaceId " << spaceId << ", partId " << outdateListener.partId_ + << " was removed!"; + listener_->onListenerPartRemoved(spaceId, outdateListener.partId_, type); + } + listener_->onListenerSpaceRemoved(spaceId, type); + VLOG(1) << "[Listener] SpaceId " << spaceId << " was removed, type is " + << apache::thrift::util::enumNameSafe(type); + } else { + // remove outdate part of listener of specified type + std::sort(listenerParts.begin(), listenerParts.end()); + std::sort(newTypeIter->second.begin(), newTypeIter->second.end()); + std::vector diff; + std::set_difference(listenerParts.begin(), + listenerParts.end(), + newTypeIter->second.begin(), + newTypeIter->second.end(), + std::back_inserter(diff)); + for (const auto& outdateListener : diff) { + VLOG(1) << "[Listener] SpaceId " << spaceId << ", partId " << outdateListener.partId_ + << " was removed!"; + listener_->onListenerPartRemoved(spaceId, outdateListener.partId_, type); + } + } + } + } + } +} + +void MetaClient::loadRemoteListeners() { + memory::MemoryCheckOffGuard g; + folly::SharedMutex::WriteHolder holder(listenerLock_); + if (listener_ == nullptr) { + VLOG(3) << "Listener is null!"; + return; + } + auto partsMap = getPartsMapFromCache(options_.localHost_); + for (const auto& spaceEntry : partsMap) { + auto spaceId = spaceEntry.first; + for (const auto& partEntry : spaceEntry.second) { + auto partId = partEntry.first; + auto listeners = getListenerHostTypeBySpacePartType(spaceId, partId); + std::vector remoteListeners; + if (listeners.ok()) { + for (const auto& listener : listeners.value()) { + remoteListeners.emplace_back(listener.first); + } + } + listener_->onCheckRemoteListeners(spaceId, partId, remoteListeners); + } + } +} + +/// ================================== public methods ================================= + +PartitionID MetaClient::partId(int32_t numParts, const VertexID id) const { + memory::MemoryCheckOffGuard g; + // If the length of the id is 8, we will treat it as int64_t to be compatible + // with the version 1.0 + uint64_t vid = 0; + if (id.size() == 8) { + memcpy(static_cast(&vid), id.data(), 8); + } else { + MurmurHash2 hash; + vid = hash(id.data()); + } + PartitionID pId = vid % numParts + 1; + CHECK_GT(pId, 0U); + return pId; +} + +folly::Future> MetaClient::submitJob( + GraphSpaceID spaceId, cpp2::JobOp op, cpp2::JobType type, std::vector paras) { + memory::MemoryCheckOffGuard g; + cpp2::AdminJobReq req; + req.space_id_ref() = spaceId; + req.op_ref() = op; + req.type_ref() = type; + req.paras_ref() = std::move(paras); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_runAdminJob(request); }, + [](cpp2::AdminJobResp&& resp) -> decltype(auto) { return resp.get_result(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createSpace(meta::cpp2::SpaceDesc spaceDesc, + bool ifNotExists) { + memory::MemoryCheckOffGuard g; + cpp2::CreateSpaceReq req; + req.properties_ref() = std::move(spaceDesc); + req.if_not_exists_ref() = ifNotExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createSpace(request); }, + [](cpp2::ExecResp&& resp) -> GraphSpaceID { return resp.get_id().get_space_id(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createSpaceAs(const std::string& oldSpaceName, + const std::string& newSpaceName, + bool ifNotExists) { + memory::MemoryCheckOffGuard g; + cpp2::CreateSpaceAsReq req; + req.old_space_name_ref() = oldSpaceName; + req.new_space_name_ref() = newSpaceName; + req.if_not_exists_ref() = ifNotExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createSpaceAs(request); }, + [](cpp2::ExecResp&& resp) -> GraphSpaceID { return resp.get_id().get_space_id(); }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listSpaces() { + memory::MemoryCheckOffGuard g; + cpp2::ListSpacesReq req; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listSpaces(request); }, + [this](cpp2::ListSpacesResp&& resp) -> decltype(auto) { + return this->toSpaceIdName(resp.get_spaces()); + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getSpace(std::string name) { + memory::MemoryCheckOffGuard g; + cpp2::GetSpaceReq req; + req.space_name_ref() = std::move(name); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getSpace(request); }, + [](cpp2::GetSpaceResp&& resp) -> decltype(auto) { return std::move(resp).get_item(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropSpaceReq req; + req.space_name_ref() = std::move(name); + req.if_exists_ref() = ifExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropSpace(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::clearSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::ClearSpaceReq req; + req.space_name_ref() = std::move(name); + req.if_exists_ref() = ifExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_clearSpace(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listHosts(cpp2::ListHostType tp) { + memory::MemoryCheckOffGuard g; + cpp2::ListHostsReq req; + req.type_ref() = tp; + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listHosts(request); }, + [](cpp2::ListHostsResp&& resp) -> decltype(auto) { return resp.get_hosts(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::alterSpace(const std::string& spaceName, + meta::cpp2::AlterSpaceOp op, + const std::vector& paras) { + memory::MemoryCheckOffGuard g; + cpp2::AlterSpaceReq req; + req.op_ref() = op; + req.space_name_ref() = spaceName; + req.paras_ref() = paras; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_alterSpace(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listParts( + GraphSpaceID spaceId, std::vector partIds) { + memory::MemoryCheckOffGuard g; + cpp2::ListPartsReq req; + req.space_id_ref() = spaceId; + req.part_ids_ref() = std::move(partIds); + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listParts(request); }, + [](cpp2::ListPartsResp&& resp) -> decltype(auto) { return resp.get_parts(); }, + std::move(promise)); + return future; +} + +folly::Future>>> +MetaClient::getPartsAlloc(GraphSpaceID spaceId, PartTerms* partTerms) { + memory::MemoryCheckOffGuard g; + cpp2::GetPartsAllocReq req; + req.space_id_ref() = spaceId; + folly::Promise>>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getPartsAlloc(request); }, + [=](cpp2::GetPartsAllocResp&& resp) -> decltype(auto) { + std::unordered_map> parts; + for (const auto& it : resp.get_parts()) { + parts.emplace(it.first, it.second); + } + if (partTerms && resp.terms_ref().has_value()) { + for (auto& termOfPart : resp.terms_ref().value()) { + (*partTerms)[termOfPart.first] = termOfPart.second; + } + } + return parts; + }, + std::move(promise)); + return future; +} + +StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceIndexByName_.find(name); + if (it != metadata.spaceIndexByName_.end()) { + return it->second; + } + return Status::SpaceNotFound(fmt::format("SpaceName `{}`", name)); +} + +StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + LOG(ERROR) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + return spaceIt->second->spaceDesc_.get_space_name(); +} + +StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, + const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceTagIndexByName_.find(std::make_pair(space, name)); + if (it == metadata.spaceTagIndexByName_.end()) { + return Status::TagNotFound(fmt::format("TagName `{}`", name)); + } + return it->second; +} + +StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& space, + const TagID& tagId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceTagIndexById_.find(std::make_pair(space, tagId)); + if (it == metadata.spaceTagIndexById_.end()) { + return Status::TagNotFound(fmt::format("TagID `{}`", tagId)); + } + return it->second; +} + +StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& space, + const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceEdgeIndexByName_.find(std::make_pair(space, name)); + if (it == metadata.spaceEdgeIndexByName_.end()) { + return Status::EdgeNotFound(fmt::format("EdgeName `{}`", name)); + } + return it->second; +} + +StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& space, + const EdgeType edgeType) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceEdgeIndexByType_.find(std::make_pair(space, edgeType)); + if (it == metadata.spaceEdgeIndexByType_.end()) { + return Status::EdgeNotFound(fmt::format("EdgeType `{}`", edgeType)); + } + return it->second; +} + +StatusOr> MetaClient::getAllEdgeFromCache(const GraphSpaceID& space) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceAllEdgeMap_.find(space); + if (it == metadata.spaceAllEdgeMap_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", space)); + } + return it->second; +} + +PartsMap MetaClient::getPartsMapFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return doGetPartsMap(host, metadata.localCache_); +} + +StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it == metadata.localCache_.end()) { + return Status::Error("Space not found, spaceid: %d", spaceId); + } + auto& cache = it->second; + auto partAllocIter = cache->partsAlloc_.find(partId); + if (partAllocIter == cache->partsAlloc_.end()) { + return Status::Error("Part not found in cache, spaceid: %d, partid: %d", spaceId, partId); + } + PartHosts ph; + ph.spaceId_ = spaceId; + ph.partId_ = partId; + ph.hosts_ = partAllocIter->second; + return ph; +} + +Status MetaClient::checkPartExistInCache(const HostAddr& host, + GraphSpaceID spaceId, + PartitionID partId) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { + auto partsIt = it->second->partsOnHost_.find(host); + if (partsIt != it->second->partsOnHost_.end()) { + for (auto& pId : partsIt->second) { + if (pId == partId) { + return Status::OK(); + } + } + return Status::PartNotFound(); + } else { + return Status::HostNotFound(); + } + } + return Status::SpaceNotFound(); +} + +Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it != metadata.localCache_.end()) { + auto partsIt = it->second->partsOnHost_.find(host); + if (partsIt != it->second->partsOnHost_.end() && !partsIt->second.empty()) { + return Status::OK(); + } else { + return Status::PartNotFound(); + } + } + return Status::SpaceNotFound(); +} + +StatusOr MetaClient::partsNum(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.localCache_.find(spaceId); + if (it == metadata.localCache_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + return it->second->partsAlloc_.size(); +} + +folly::Future> MetaClient::createTagSchema(GraphSpaceID spaceId, + std::string name, + cpp2::Schema schema, + bool ifNotExists) { + memory::MemoryCheckOffGuard g; + cpp2::CreateTagReq req; + req.space_id_ref() = spaceId; + req.tag_name_ref() = std::move(name); + req.schema_ref() = std::move(schema); + req.if_not_exists_ref() = ifNotExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createTag(request); }, + [](cpp2::ExecResp&& resp) -> TagID { return resp.get_id().get_tag_id(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, + std::string name, + std::vector items, + cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; + cpp2::AlterTagReq req; + req.space_id_ref() = spaceId; + req.tag_name_ref() = std::move(name); + req.tag_items_ref() = std::move(items); + req.schema_prop_ref() = std::move(schemaProp); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_alterTag(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listTagSchemas( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::ListTagsReq req; + req.space_id_ref() = spaceId; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listTags(request); }, + [](cpp2::ListTagsResp&& resp) -> decltype(auto) { return std::move(resp).get_tags(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropTagSchema(GraphSpaceID spaceId, + std::string tagName, + const bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropTagReq req; + req.space_id_ref() = spaceId; + req.tag_name_ref() = std::move(tagName); + req.if_exists_ref() = ifExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropTag(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getTagSchema(GraphSpaceID spaceId, + std::string name, + int64_t version) { + memory::MemoryCheckOffGuard g; + cpp2::GetTagReq req; + req.space_id_ref() = spaceId; + req.tag_name_ref() = std::move(name); + req.version_ref() = version; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getTag(request); }, + [](cpp2::GetTagResp&& resp) -> cpp2::Schema { return std::move(resp).get_schema(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createEdgeSchema(GraphSpaceID spaceId, + std::string name, + cpp2::Schema schema, + bool ifNotExists) { + memory::MemoryCheckOffGuard g; + cpp2::CreateEdgeReq req; + req.space_id_ref() = spaceId; + req.edge_name_ref() = std::move(name); + req.schema_ref() = schema; + req.if_not_exists_ref() = ifNotExists; + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createEdge(request); }, + [](cpp2::ExecResp&& resp) -> EdgeType { return resp.get_id().get_edge_type(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, + std::string name, + std::vector items, + cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; + cpp2::AlterEdgeReq req; + req.space_id_ref() = spaceId; + req.edge_name_ref() = std::move(name); + req.edge_items_ref() = std::move(items); + req.schema_prop_ref() = std::move(schemaProp); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_alterEdge(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listEdgeSchemas( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::ListEdgesReq req; + req.space_id_ref() = spaceId; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listEdges(request); }, + [](cpp2::ListEdgesResp&& resp) -> decltype(auto) { return std::move(resp).get_edges(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spaceId, + std::string name, + SchemaVer version) { + memory::MemoryCheckOffGuard g; + cpp2::GetEdgeReq req; + req.space_id_ref() = spaceId; + req.edge_name_ref() = std::move(name); + req.version_ref() = version; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getEdge(request); }, + [](cpp2::GetEdgeResp&& resp) -> cpp2::Schema { return std::move(resp).get_schema(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropEdgeSchema(GraphSpaceID spaceId, + std::string name, + const bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropEdgeReq req; + req.space_id_ref() = spaceId; + req.edge_name_ref() = std::move(name); + req.if_exists_ref() = ifExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropEdge(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID, + std::string indexName, + std::string tagName, + std::vector fields, + bool ifNotExists, + const cpp2::IndexParams* indexParams, + const std::string* comment) { + memory::MemoryCheckOffGuard g; + cpp2::CreateTagIndexReq req; + req.space_id_ref() = spaceID; + req.index_name_ref() = std::move(indexName); + req.tag_name_ref() = std::move(tagName); + req.fields_ref() = std::move(fields); + req.if_not_exists_ref() = ifNotExists; + if (indexParams != nullptr) { + req.index_params_ref() = *indexParams; + } + if (comment != nullptr) { + req.comment_ref() = *comment; + } + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createTagIndex(request); }, + [](cpp2::ExecResp&& resp) -> IndexID { return resp.get_id().get_index_id(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropTagIndex(GraphSpaceID spaceID, + std::string name, + bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropTagIndexReq req; + req.space_id_ref() = (spaceID); + req.index_name_ref() = (std::move(name)); + req.if_exists_ref() = (ifExists); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropTagIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getTagIndex(GraphSpaceID spaceID, + std::string name) { + memory::MemoryCheckOffGuard g; + cpp2::GetTagIndexReq req; + req.space_id_ref() = spaceID; + req.index_name_ref() = std::move(name); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getTagIndex(request); }, + [](cpp2::GetTagIndexResp&& resp) -> cpp2::IndexItem { return std::move(resp).get_item(); }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listTagIndexes( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::ListTagIndexesReq req; + req.space_id_ref() = spaceId; + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listTagIndexes(request); }, + [](cpp2::ListTagIndexesResp&& resp) -> decltype(auto) { return std::move(resp).get_items(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::rebuildTagIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; + cpp2::RebuildIndexReq req; + req.space_id_ref() = spaceID; + req.index_name_ref() = std::move(name); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_rebuildTagIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listTagIndexStatus( + GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; + cpp2::ListIndexStatusReq req; + req.space_id_ref() = spaceID; + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listTagIndexStatus(request); }, + [](cpp2::ListIndexStatusResp&& resp) -> decltype(auto) { + return std::move(resp).get_statuses(); + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createEdgeIndex( + GraphSpaceID spaceID, + std::string indexName, + std::string edgeName, + std::vector fields, + bool ifNotExists, + const cpp2::IndexParams* indexParams, + const std::string* comment) { + memory::MemoryCheckOffGuard g; + cpp2::CreateEdgeIndexReq req; + req.space_id_ref() = spaceID; + req.index_name_ref() = std::move(indexName); + req.edge_name_ref() = std::move(edgeName); + req.fields_ref() = std::move(fields); + req.if_not_exists_ref() = ifNotExists; + if (indexParams != nullptr) { + req.index_params_ref() = *indexParams; + } + if (comment != nullptr) { + req.comment_ref() = *comment; + } + + folly::Promise> promise; + auto future = promise.getFuture(); + + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createEdgeIndex(request); }, + [](cpp2::ExecResp&& resp) -> IndexID { return resp.get_id().get_index_id(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropEdgeIndex(GraphSpaceID spaceId, + std::string name, + bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropEdgeIndexReq req; + req.space_id_ref() = spaceId; + req.index_name_ref() = std::move(name); + req.if_exists_ref() = ifExists; + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropEdgeIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getEdgeIndex(GraphSpaceID spaceId, + std::string name) { + memory::MemoryCheckOffGuard g; + cpp2::GetEdgeIndexReq req; + req.space_id_ref() = spaceId; + req.index_name_ref() = std::move(name); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getEdgeIndex(request); }, + [](cpp2::GetEdgeIndexResp&& resp) -> cpp2::IndexItem { return std::move(resp).get_item(); }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listEdgeIndexes( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::ListEdgeIndexesReq req; + req.space_id_ref() = spaceId; + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listEdgeIndexes(request); }, + [](cpp2::ListEdgeIndexesResp&& resp) -> decltype(auto) { + return std::move(resp).get_items(); + }, + std::move(promise)); + return future; +} + +StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + LOG(ERROR) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + auto& vidType = spaceIt->second->spaceDesc_.get_vid_type(); + auto vIdLen = vidType.type_length_ref().has_value() ? *vidType.get_type_length() : 0; + if (vIdLen <= 0) { + return Status::Error("Space %d vertexId length invalid", spaceId); + } + return vIdLen; +} + +StatusOr MetaClient::getSpaceVidType(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + LOG(ERROR) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + auto vIdType = spaceIt->second->spaceDesc_.get_vid_type().get_type(); + if (vIdType != nebula::cpp2::PropertyType::INT64 && + vIdType != nebula::cpp2::PropertyType::FIXED_STRING) { + std::stringstream ss; + ss << "Space " << spaceId + << ", vertexId type invalid: " << apache::thrift::util::enumNameSafe(vIdType); + return Status::Error(ss.str()); + } + return vIdType; +} + +StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + LOG(ERROR) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + return spaceIt->second->spaceDesc_; +} + +StatusOr MetaClient::getIsolationLevel(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + auto spaceDescStatus = getSpaceDesc(spaceId); + if (!spaceDescStatus.ok()) { + return spaceDescStatus.status(); + } + using IsolationLevel = meta::cpp2::IsolationLevel; + return spaceDescStatus.value().isolation_level_ref().value_or(IsolationLevel::DEFAULT); +} + +StatusOr> MetaClient::getTagSchemaFromCache( + GraphSpaceID spaceId, TagID tagID, SchemaVer ver) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt != metadata.localCache_.end()) { + auto tagIt = spaceIt->second->tagSchemas_.find(tagID); + if (tagIt != spaceIt->second->tagSchemas_.end() && !tagIt->second.empty()) { + size_t vNum = tagIt->second.size(); + if (static_cast(vNum) > ver) { + return ver < 0 ? tagIt->second.back() : tagIt->second[ver]; + } + } + } + LOG(ERROR) << "The tag schema " << tagID << " not found in space " << spaceId; + return std::shared_ptr(); +} + +StatusOr> MetaClient::getEdgeSchemaFromCache( + GraphSpaceID spaceId, EdgeType edgeType, SchemaVer ver) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt != metadata.localCache_.end()) { + auto edgeIt = spaceIt->second->edgeSchemas_.find(edgeType); + if (edgeIt != spaceIt->second->edgeSchemas_.end() && !edgeIt->second.empty()) { + size_t vNum = edgeIt->second.size(); + if (static_cast(vNum) > ver) { + return ver < 0 ? edgeIt->second.back() : edgeIt->second[ver]; + } + } + } + LOG(ERROR) << "The edge schema " << edgeType << " not found in space " << spaceId; + return std::shared_ptr(); +} + +StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + return iter->second->tagSchemas_; +} + +StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + TagSchema tagsSchema; + tagsSchema.reserve(iter->second->tagSchemas_.size()); + // fetch all tagIds + for (const auto& tagSchema : iter->second->tagSchemas_) { + tagsSchema.emplace(tagSchema.first, tagSchema.second.back()); + } + return tagsSchema; +} + +StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + return iter->second->edgeSchemas_; +} + +StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.localCache_.find(spaceId); + if (iter == metadata.localCache_.end()) { + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } + EdgeSchema edgesSchema; + edgesSchema.reserve(iter->second->edgeSchemas_.size()); + // fetch all edgeTypes + for (const auto& edgeSchema : iter->second->edgeSchemas_) { + edgesSchema.emplace(edgeSchema.first, edgeSchema.second.back()); + } + return edgesSchema; +} + +folly::Future> MetaClient::rebuildEdgeIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; + cpp2::RebuildIndexReq req; + req.space_id_ref() = spaceID; + req.index_name_ref() = std::move(name); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_rebuildEdgeIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listEdgeIndexStatus( + GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; + cpp2::ListIndexStatusReq req; + req.space_id_ref() = spaceID; + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listEdgeIndexStatus(request); }, + [](cpp2::ListIndexStatusResp&& resp) -> decltype(auto) { + return std::move(resp).get_statuses(); + }, + std::move(promise)); + return future; +} + +StatusOr> MetaClient::getTagIndexByNameFromCache( + const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + std::pair key(space, name); + auto iter = tagNameIndexMap_.find(key); + if (iter == tagNameIndexMap_.end()) { + return Status::IndexNotFound(fmt::format("Index: {}:{}", space, name)); + } + auto indexID = iter->second; + auto itemStatus = getTagIndexFromCache(space, indexID); + if (!itemStatus.ok()) { + return itemStatus.status(); + } + return itemStatus.value(); +} + +StatusOr> MetaClient::getEdgeIndexByNameFromCache( + const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + std::pair key(space, name); + auto iter = edgeNameIndexMap_.find(key); + if (iter == edgeNameIndexMap_.end()) { + return Status::IndexNotFound(fmt::format("Index: {}:{}", space, name)); + } + auto indexID = iter->second; + auto itemStatus = getEdgeIndexFromCache(space, indexID); + if (!itemStatus.ok()) { + return itemStatus.status(); + } + return itemStatus.value(); +} + +StatusOr> MetaClient::getTagIndexFromCache(GraphSpaceID spaceId, + IndexID indexID) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } else { + auto iter = spaceIt->second->tagIndexes_.find(indexID); + if (iter == spaceIt->second->tagIndexes_.end()) { + VLOG(3) << "Space " << spaceId << ", Tag Index " << indexID << " not found!"; + return Status::IndexNotFound(); + } else { + return iter->second; + } + } +} + +StatusOr MetaClient::getRelatedTagIDByIndexNameFromCache(const GraphSpaceID space, + const std::string& indexName) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + auto indexRet = getTagIndexByNameFromCache(space, indexName); + if (!indexRet.ok()) { + LOG(ERROR) << "Index " << indexName << " Not Found"; + return indexRet.status(); + } + + return indexRet.value()->get_schema_id().get_tag_id(); +} + +StatusOr> MetaClient::getEdgeIndexFromCache(GraphSpaceID spaceId, + IndexID indexId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } else { + auto iter = spaceIt->second->edgeIndexes_.find(indexId); + if (iter == spaceIt->second->edgeIndexes_.end()) { + VLOG(3) << "Space " << spaceId << ", Edge Index " << indexId << " not found!"; + return Status::IndexNotFound(fmt::format("Index: {}:{}", spaceId, indexId)); + } else { + return iter->second; + } + } +} + +StatusOr MetaClient::getRelatedEdgeTypeByIndexNameFromCache( + const GraphSpaceID space, const std::string& indexName) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + auto indexRet = getEdgeIndexByNameFromCache(space, indexName); + if (!indexRet.ok()) { + LOG(ERROR) << "Index " << indexName << " Not Found"; + return indexRet.status(); + } + + return indexRet.value()->get_schema_id().get_edge_type(); +} + +StatusOr>> MetaClient::getTagIndexesFromCache( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } else { + auto tagIndexes = spaceIt->second->tagIndexes_; + auto iter = tagIndexes.begin(); + std::vector> items; + while (iter != tagIndexes.end()) { + items.emplace_back(iter->second); + iter++; + } + return items; + } +} + +StatusOr>> MetaClient::getEdgeIndexesFromCache( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(fmt::format("SpaceId `{}`", spaceId)); + } else { + auto edgeIndexes = spaceIt->second->edgeIndexes_; + auto iter = edgeIndexes.begin(); + std::vector> items; + while (iter != edgeIndexes.end()) { + items.emplace_back(iter->second); + iter++; + } + return items; + } +} + +StatusOr MetaClient::getStorageLeaderFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + { + folly::SharedMutex::ReadHolder holder(leadersLock_); + auto iter = leadersInfo_.leaderMap_.find({spaceId, partId}); + if (iter != leadersInfo_.leaderMap_.end()) { + return iter->second; + } + } + { + // no leader found, pick one in round-robin + auto partHostsRet = getPartHostsFromCache(spaceId, partId); + if (!partHostsRet.ok()) { + return partHostsRet.status(); + } + auto partHosts = partHostsRet.value(); + folly::SharedMutex::WriteHolder wh(leadersLock_); + VLOG(1) << "No leader exists. Choose one in round-robin."; + auto index = (leadersInfo_.pickedIndex_[{spaceId, partId}] + 1) % partHosts.hosts_.size(); + auto picked = partHosts.hosts_[index]; + leadersInfo_.leaderMap_[{spaceId, partId}] = picked; + leadersInfo_.pickedIndex_[{spaceId, partId}] = index; + return picked; + } +} + +void MetaClient::updateStorageLeader(GraphSpaceID spaceId, + PartitionID partId, + const HostAddr& leader) { + memory::MemoryCheckOffGuard g; + VLOG(1) << "Update the leader for [" << spaceId << ", " << partId << "] to " << leader; + folly::SharedMutex::WriteHolder holder(leadersLock_); + leadersInfo_.leaderMap_[{spaceId, partId}] = leader; +} + +void MetaClient::invalidStorageLeader(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; + VLOG(1) << "Invalidate the leader for [" << spaceId << ", " << partId << "]"; + folly::SharedMutex::WriteHolder holder(leadersLock_); + leadersInfo_.leaderMap_.erase({spaceId, partId}); +} + +StatusOr MetaClient::getLeaderInfo() { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::SharedMutex::ReadHolder holder(leadersLock_); + return leadersInfo_; +} + +const std::vector& MetaClient::getAddresses() { + return addrs_; +} + +std::vector MetaClient::getRolesByUserFromCache(const std::string& user) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return std::vector(0); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userRolesMap_.find(user); + if (iter == metadata.userRolesMap_.end()) { + return std::vector(0); + } + return iter->second; +} + +Status MetaClient::authCheckFromCache(const std::string& account, const std::string& password) { + memory::MemoryCheckOffGuard g; + // Check meta service status + if (!ready_) { + return Status::Error("Meta Service not ready"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userPasswordMap_.find(account); + if (iter == metadata.userPasswordMap_.end()) { + return Status::Error("User not exist"); + } + auto lockedSince = userLoginLockTime_[account]; + auto passwordAttemptRemain = userPasswordAttemptsRemain_[account]; + + // If lockedSince is non-zero, it means the account has been locked + if (lockedSince != 0) { + auto remainingLockTime = + (lockedSince + FLAGS_password_lock_time_in_secs) - time::WallClock::fastNowInSec(); + // If the account is still locked, there is no need to check the password + if (remainingLockTime > 0) { + return Status::Error( + "%d times consecutive incorrect passwords has been input, user name: %s has been " + "locked, try again in %ld seconds", + FLAGS_failed_login_attempts, + account.c_str(), + remainingLockTime); + } + // Clear lock state and reset attempts + userLoginLockTime_.assign_if_equal(account, lockedSince, 0); + userPasswordAttemptsRemain_.assign_if_equal( + account, passwordAttemptRemain, FLAGS_failed_login_attempts); + } + + if (iter->second != password) { + // By default there is no limit of login attempts if any of these 2 flags is unset + if (FLAGS_failed_login_attempts == 0 || FLAGS_password_lock_time_in_secs == 0) { + return Status::Error("Invalid password"); + } + + // If the password is not correct and passwordAttemptRemain > 0, + // Allow another attempt + passwordAttemptRemain = userPasswordAttemptsRemain_[account]; + if (passwordAttemptRemain > 0) { + auto newAttemptRemain = passwordAttemptRemain - 1; + userPasswordAttemptsRemain_.assign_if_equal(account, passwordAttemptRemain, newAttemptRemain); + if (newAttemptRemain == 0) { + // If the remaining attempts is 0, failed to authenticate + // Block user login + userLoginLockTime_.assign_if_equal(account, 0, time::WallClock::fastNowInSec()); + return Status::Error( + "%d times consecutive incorrect passwords has been input, user name: %s has been " + "locked, try again in %d seconds", + FLAGS_failed_login_attempts, + account.c_str(), + FLAGS_password_lock_time_in_secs); + } + LOG(ERROR) << "Invalid password, remaining attempts: " << newAttemptRemain; + return Status::Error("Invalid password, remaining attempts: %d", newAttemptRemain); + } + } + + // Authentication succeed, reset password attempts + userPasswordAttemptsRemain_.assign(account, FLAGS_failed_login_attempts); + userLoginLockTime_.assign(account, 0); + return Status::OK(); +} + +bool MetaClient::checkShadowAccountFromCache(const std::string& account) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return false; + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto iter = metadata.userPasswordMap_.find(account); + if (iter != metadata.userPasswordMap_.end()) { + return true; + } + return false; +} + +StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceInfo = metadata.localCache_.find(spaceId); + if (spaceInfo == metadata.localCache_.end()) { + return Status::Error("Term not found!"); + } + + auto termInfo = spaceInfo->second->termOfPartition_.find(partId); + if (termInfo == spaceInfo->second->termOfPartition_.end()) { + return Status::Error("Term not found!"); + } + + return termInfo->second; +} + +StatusOr> MetaClient::getStorageHosts() { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return metadata.storageHosts_; +} + +StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& space, + const TagID& tagId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceNewestTagVerMap_.find(std::make_pair(space, tagId)); + if (it == metadata.spaceNewestTagVerMap_.end()) { + return Status::TagNotFound(); + } + return it->second; +} + +StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID& space, + const EdgeType& edgeType) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto it = metadata.spaceNewestEdgeVerMap_.find(std::make_pair(space, edgeType)); + if (it == metadata.spaceNewestEdgeVerMap_.end()) { + return Status::EdgeNotFound(fmt::format("EdgeType `{}`", edgeType)); + } + return it->second; +} + +folly::Future> MetaClient::heartbeat() { + memory::MemoryCheckOffGuard g; + cpp2::HBReq req; + req.host_ref() = options_.localHost_; + req.role_ref() = options_.role_; + req.git_info_sha_ref() = options_.gitInfoSHA_; + if (options_.role_ == cpp2::HostRole::STORAGE || + options_.role_ == cpp2::HostRole::STORAGE_LISTENER) { + if (options_.clusterId_.load() == 0) { + options_.clusterId_ = FileBasedClusterIdMan::getClusterIdFromFile(FLAGS_cluster_id_path); + } + req.cluster_id_ref() = options_.clusterId_.load(); + std::unordered_map> leaderIds; + if (listener_ != nullptr) { + listener_->fetchLeaderInfo(leaderIds); + if (leaderIds_ != leaderIds) { + { + folly::SharedMutex::WriteHolder holder(leaderIdsLock_); + leaderIds_.clear(); + leaderIds_ = leaderIds; + } + } + req.leader_partIds_ref() = std::move(leaderIds); + } else { + req.leader_partIds_ref() = std::move(leaderIds); + } + + kvstore::SpaceDiskPartsMap diskParts; + if (listener_ != nullptr) { + listener_->fetchDiskParts(diskParts); + if (diskParts_ != diskParts) { + { + folly::SharedMutex::WriteHolder holder(&diskPartsLock_); + diskParts_.clear(); + diskParts_ = diskParts; + } + req.disk_parts_ref() = diskParts; + } + } else { + req.disk_parts_ref() = diskParts; + } + } + + // info used in the agent, only set once + // TODO(spw): if we could add data path(disk) dynamically in the future, it should be + // reported every time it changes + if (!dirInfoReported_) { + nebula::cpp2::DirInfo dirInfo; + if (options_.role_ == cpp2::HostRole::GRAPH) { + dirInfo.root_ref() = options_.rootPath_; + } else if (options_.role_ == cpp2::HostRole::STORAGE) { + dirInfo.root_ref() = options_.rootPath_; + dirInfo.data_ref() = options_.dataPaths_; + } + req.dir_ref() = dirInfo; + } + + folly::Promise> promise; + auto future = promise.getFuture(); + VLOG(1) << "Send heartbeat to " << leader_ << ", clusterId " << req.get_cluster_id(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_heartBeat(request); }, + [this](cpp2::HBResp&& resp) -> bool { + if ((options_.role_ == cpp2::HostRole::STORAGE || + options_.role_ == cpp2::HostRole::STORAGE_LISTENER) && + options_.clusterId_.load() == 0) { + LOG(INFO) << "Persist the cluster Id from metad " << resp.get_cluster_id(); + if (FileBasedClusterIdMan::persistInFile(resp.get_cluster_id(), FLAGS_cluster_id_path)) { + options_.clusterId_.store(resp.get_cluster_id()); + } else { + DLOG(FATAL) << "Can't persist the clusterId in file " << FLAGS_cluster_id_path; + return false; + } + } + heartbeatTime_ = time::WallClock::fastNowInMilliSec(); + metadLastUpdateTime_ = resp.get_last_update_time_in_ms(); + VLOG(1) << "Metad last update time: " << metadLastUpdateTime_; + metaServerVersion_ = resp.get_meta_version(); + + bool succeeded = resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + if (succeeded) { + dirInfoReported_ = true; + } + return succeeded; + }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::createUser(std::string account, + std::string password, + bool ifNotExists) { + memory::MemoryCheckOffGuard g; + cpp2::CreateUserReq req; + req.account_ref() = std::move(account); + req.encoded_pwd_ref() = std::move(password); + req.if_not_exists_ref() = ifNotExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createUser(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropUser(std::string account, bool ifExists) { + memory::MemoryCheckOffGuard g; + cpp2::DropUserReq req; + req.account_ref() = std::move(account); + req.if_exists_ref() = ifExists; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropUser(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::alterUser(std::string account, std::string password) { + memory::MemoryCheckOffGuard g; + cpp2::AlterUserReq req; + req.account_ref() = std::move(account); + req.encoded_pwd_ref() = std::move(password); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_alterUser(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::grantToUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; + cpp2::GrantRoleReq req; + req.role_item_ref() = std::move(roleItem); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_grantRole(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::revokeFromUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; + cpp2::RevokeRoleReq req; + req.role_item_ref() = std::move(roleItem); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_revokeRole(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listUsers() { + memory::MemoryCheckOffGuard g; + cpp2::ListUsersReq req; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listUsers(request); }, + [](cpp2::ListUsersResp&& resp) -> decltype(auto) { return std::move(resp).get_users(); }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listRoles(GraphSpaceID space) { + memory::MemoryCheckOffGuard g; + cpp2::ListRolesReq req; + req.space_id_ref() = std::move(space); + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listRoles(request); }, + [](cpp2::ListRolesResp&& resp) -> decltype(auto) { return std::move(resp).get_roles(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::changePassword(std::string account, + std::string newPwd, + std::string oldPwd) { + memory::MemoryCheckOffGuard g; + cpp2::ChangePasswordReq req; + req.account_ref() = std::move(account); + req.new_encoded_pwd_ref() = std::move(newPwd); + req.old_encoded_pwd_ref() = std::move(oldPwd); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_changePassword(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::getUserRoles(std::string account) { + memory::MemoryCheckOffGuard g; + cpp2::GetUserRolesReq req; + req.account_ref() = std::move(account); + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getUserRoles(request); }, + [](cpp2::ListRolesResp&& resp) -> decltype(auto) { return std::move(resp).get_roles(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::regConfig(const std::vector& items) { + memory::MemoryCheckOffGuard g; + cpp2::RegConfigReq req; + req.items_ref() = items; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_regConfig(request); }, + [](cpp2::ExecResp&& resp) -> decltype(auto) { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::getConfig( + const cpp2::ConfigModule& module, const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!configReady_) { + return Status::Error("Not ready!"); + } + cpp2::ConfigItem item; + item.module_ref() = module; + item.name_ref() = name; + cpp2::GetConfigReq req; + req.item_ref() = item; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getConfig(request); }, + [](cpp2::GetConfigResp&& resp) -> decltype(auto) { return std::move(resp).get_items(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& module, + const std::string& name, + const Value& value) { + memory::MemoryCheckOffGuard g; + cpp2::ConfigItem item; + item.module_ref() = module; + item.name_ref() = name; + item.value_ref() = value; + + cpp2::SetConfigReq req; + req.item_ref() = item; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_setConfig(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listConfigs( + const cpp2::ConfigModule& module) { + memory::MemoryCheckOffGuard g; + cpp2::ListConfigsReq req; + req.module_ref() = module; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listConfigs(request); }, + [](cpp2::ListConfigsResp&& resp) -> decltype(auto) { return std::move(resp).get_items(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::createSnapshot() { + memory::MemoryCheckOffGuard g; + cpp2::CreateSnapshotReq req; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createSnapshot(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise), + true, + 0, + 1); + return future; +} + +folly::Future> MetaClient::dropSnapshot(const std::string& name) { + memory::MemoryCheckOffGuard g; + cpp2::DropSnapshotReq req; + std::vector names{name}; + req.names_ref() = names; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropSnapshot(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listSnapshots() { + memory::MemoryCheckOffGuard g; + cpp2::ListSnapshotsReq req; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listSnapshots(request); }, + [](cpp2::ListSnapshotsResp&& resp) -> decltype(auto) { + return std::move(resp).get_snapshots(); + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::addListener(GraphSpaceID spaceId, + cpp2::ListenerType type, + std::vector hosts) { + memory::MemoryCheckOffGuard g; + cpp2::AddListenerReq req; + req.space_id_ref() = spaceId; + req.type_ref() = type; + req.hosts_ref() = std::move(hosts); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_addListener(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::removeListener(GraphSpaceID spaceId, + cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; + cpp2::RemoveListenerReq req; + req.space_id_ref() = spaceId; + req.type_ref() = type; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_removeListener(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listListener( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::ListListenerReq req; + req.space_id_ref() = spaceId; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listListener(request); }, + [](cpp2::ListListenerResp&& resp) -> decltype(auto) { + return std::move(resp).get_listeners(); + }, + std::move(promise)); + return future; +} + +bool MetaClient::registerCfg() { + memory::MemoryCheckOffGuard g; + auto ret = regConfig(gflagsDeclared_).get(); + if (ret.ok()) { + LOG(INFO) << "Register gflags ok " << gflagsDeclared_.size(); + configReady_ = true; + } + return configReady_; +} + +StatusOr>> +MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAddr& host) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(); + } + auto iter = spaceIt->second->listeners_.find(host); + if (iter == spaceIt->second->listeners_.end()) { + VLOG(3) << "Space " << spaceId << ", Listener on host " << host.toString() << " not found!"; + return Status::ListenerNotFound(); + } else { + return iter->second; + } +} + +StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return doGetListenersMap(host, metadata.localCache_); +} + +ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; + ListenersMap listenersMap; + for (const auto& space : localCache) { + auto spaceId = space.first; + for (const auto& listener : space.second->listeners_) { + if (host != listener.first) { + continue; + } + for (const auto& part : listener.second) { + auto partId = part.first; + auto type = part.second; + auto partIter = space.second->partsAlloc_.find(partId); + if (partIter != space.second->partsAlloc_.end()) { + auto peers = partIter->second; + listenersMap[spaceId][type].emplace_back(partId, std::move(peers)); + } else { + FLOG_WARN("%s has listener of [%d, %d], but can't find part peers", + host.toString().c_str(), + spaceId, + partId); + } + } + } + } + return listenersMap; +} + +StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spaceId, + PartitionID partId, + cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(); + } + for (const auto& host : spaceIt->second->listeners_) { + for (const auto& l : host.second) { + if (l.first == partId && l.second == type) { + return host.first; + } + } + } + return Status::ListenerNotFound(); +} + +StatusOr> MetaClient::getListenerHostTypeBySpacePartType( + GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + auto spaceIt = metadata.localCache_.find(spaceId); + if (spaceIt == metadata.localCache_.end()) { + VLOG(3) << "Space " << spaceId << " not found!"; + return Status::SpaceNotFound(); + } + std::vector items; + for (const auto& host : spaceIt->second->listeners_) { + for (const auto& l : host.second) { + if (l.first == partId) { + items.emplace_back(std::make_pair(host.first, l.second)); + } + } + } + if (items.empty()) { + return Status::ListenerNotFound(); + } + return items; +} + +bool MetaClient::loadCfg() { + memory::MemoryCheckOffGuard g; + // UNKNOWN role will skip heartbeat + if (options_.skipConfig_ || (options_.role_ != cpp2::HostRole::UNKNOWN && + localCfgLastUpdateTime_ == metadLastUpdateTime_)) { + return true; + } + if (!configReady_ && !registerCfg()) { + return false; + } + // only load current module's config is enough + auto ret = listConfigs(gflagsModule_).get(); + if (ret.ok()) { + // if we load config from meta server successfully, update gflags and set configReady_ + auto items = ret.value(); + MetaConfigMap metaConfigMap; + for (auto& item : items) { + std::pair key = {item.get_module(), item.get_name()}; + metaConfigMap.emplace(std::move(key), std::move(item)); + } + { + // For any configurations that is in meta, update in cache to replace previous value + folly::SharedMutex::WriteHolder holder(configCacheLock_); + for (const auto& entry : metaConfigMap) { + auto& key = entry.first; + auto it = metaConfigMap_.find(key); + if (it == metaConfigMap_.end() || + metaConfigMap[key].get_value() != it->second.get_value()) { + updateGflagsValue(entry.second); + metaConfigMap_[key] = entry.second; + } + } + } + } else { + LOG(ERROR) << "Load configs failed: " << ret.status(); + return false; + } + localCfgLastUpdateTime_.store(metadLastUpdateTime_.load()); + return true; +} + +void MetaClient::updateGflagsValue(const cpp2::ConfigItem& item) { + memory::MemoryCheckOffGuard g; + if (item.get_mode() != cpp2::ConfigMode::MUTABLE) { + return; + } + const auto& value = item.get_value(); + std::string curValue; + if (!gflags::GetCommandLineOption(item.get_name().c_str(), &curValue)) { + return; + } else { + auto gflagValue = GflagsManager::gflagsValueToValue(value.typeName(), curValue); + if (gflagValue != value) { + auto valueStr = GflagsManager::ValueToGflagString(value); + if (value.isMap() && value.getMap().kvs.empty()) { + // Be compatible with previous configuration + valueStr = "{}"; + } + gflags::SetCommandLineOption(item.get_name().c_str(), valueStr.c_str()); + GflagsManager::onModified(item.get_name()); + // TODO: we simply judge the rocksdb by nested type for now + if (listener_ != nullptr && value.isMap()) { + updateNestedGflags(value.getMap().kvs); + } + LOG(INFO) << "Update config " << item.get_name() << " from " << curValue << " to " + << valueStr; + } + } +} + +void MetaClient::updateNestedGflags(const std::unordered_map& nameValues) { + memory::MemoryCheckOffGuard g; + std::unordered_map optionMap; + for (const auto& value : nameValues) { + optionMap.emplace(value.first, value.second.toString()); + } + + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + for (const auto& spaceEntry : metadata.localCache_) { + listener_->onSpaceOptionUpdated(spaceEntry.first, optionMap); + } +} + +Status MetaClient::refreshCache() { + memory::MemoryCheckOffGuard g; + auto ret = bgThread_->addTask(&MetaClient::loadData, this).get(); + return ret ? Status::OK() : Status::Error("Load data failed"); +} + +void MetaClient::loadLeader(const std::vector& hostItems, + const SpaceNameIdMap& spaceIndexByName) { + memory::MemoryCheckOffGuard g; + LeaderInfo leaderInfo; + for (auto& item : hostItems) { + for (auto& spaceEntry : item.get_leader_parts()) { + auto spaceName = spaceEntry.first; + // ready_ is still false, can't read from cache + auto iter = spaceIndexByName.find(spaceName); + if (iter == spaceIndexByName.end()) { + continue; + } + auto spaceId = iter->second; + for (const auto& partId : spaceEntry.second) { + leaderInfo.leaderMap_[{spaceId, partId}] = item.get_hostAddr(); + auto partHosts = getPartHostsFromCache(spaceId, partId); + size_t leaderIndex = 0; + if (partHosts.ok()) { + const auto& peers = partHosts.value().hosts_; + for (size_t i = 0; i < peers.size(); i++) { + if (peers[i] == item.get_hostAddr()) { + leaderIndex = i; + break; + } + } + } + leaderInfo.pickedIndex_[{spaceId, partId}] = leaderIndex; + } + } + LOG(INFO) << "Load leader of " << item.get_hostAddr() << " in " + << item.get_leader_parts().size() << " space"; + } + { + // todo(doodle): in worst case, storage and meta isolated, so graph may get a outdate + // leader info. The problem could be solved if leader term are cached as well. + LOG(INFO) << "Load leader ok"; + folly::SharedMutex::WriteHolder wh(leadersLock_); + leadersInfo_ = std::move(leaderInfo); + } +} + +folly::Future> MetaClient::addHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; + cpp2::AddHostsReq req; + req.hosts_ref() = std::move(hosts); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_addHosts(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; + cpp2::DropHostsReq req; + req.hosts_ref() = std::move(hosts); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropHosts(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::mergeZone(std::vector zones, + std::string zoneName) { + memory::MemoryCheckOffGuard g; + cpp2::MergeZoneReq req; + req.zone_name_ref() = std::move(zoneName); + req.zones_ref() = std::move(zones); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_mergeZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::divideZone( + std::string zoneName, std::unordered_map> zoneItems) { + memory::MemoryCheckOffGuard g; + cpp2::DivideZoneReq req; + req.zone_name_ref() = std::move(zoneName); + req.zone_items_ref() = std::move(zoneItems); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_divideZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::renameZone(std::string originalZoneName, + std::string zoneName) { + memory::MemoryCheckOffGuard g; + cpp2::RenameZoneReq req; + req.original_zone_name_ref() = std::move(originalZoneName); + req.zone_name_ref() = std::move(zoneName); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_renameZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::dropZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; + cpp2::DropZoneReq req; + req.zone_name_ref() = std::move(zoneName); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::addHostsIntoZone(std::vector hosts, + std::string zoneName, + bool isNew) { + memory::MemoryCheckOffGuard g; + cpp2::AddHostsIntoZoneReq req; + req.hosts_ref() = hosts; + req.zone_name_ref() = zoneName; + req.is_new_ref() = isNew; + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_addHostsIntoZone(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::getZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; + cpp2::GetZoneReq req; + req.zone_name_ref() = std::move(zoneName); + + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getZone(request); }, + [](cpp2::GetZoneResp&& resp) -> decltype(auto) { return resp.get_hosts(); }, + std::move(promise)); + return future; +} + +folly::Future>> MetaClient::listZones() { + memory::MemoryCheckOffGuard g; + cpp2::ListZonesReq req; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listZones(request); }, + [](cpp2::ListZonesResp&& resp) -> decltype(auto) { return resp.get_zones(); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getStats(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + cpp2::GetStatsReq req; + req.space_id_ref() = (spaceId); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getStats(request); }, + [](cpp2::GetStatsResp&& resp) -> cpp2::StatsItem { return std::move(resp).get_stats(); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::reportTaskFinish( + GraphSpaceID spaceId, + int32_t jobId, + int32_t taskId, + nebula::cpp2::ErrorCode taskErrCode, + cpp2::StatsItem* statisticItem) { + memory::MemoryCheckOffGuard g; + cpp2::ReportTaskReq req; + req.code_ref() = taskErrCode; + req.space_id_ref() = spaceId; + req.job_id_ref() = jobId; + req.task_id_ref() = taskId; + if (statisticItem) { + req.stats_ref() = *statisticItem; + } + folly::Promise> pro; + auto fut = pro.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_reportTaskFinish(request); }, + [](cpp2::ExecResp&& resp) -> nebula::cpp2::ErrorCode { return resp.get_code(); }, + std::move(pro), + true); + return fut; +} + +folly::Future> MetaClient::signInService( + const cpp2::ExternalServiceType& type, const std::vector& clients) { + memory::MemoryCheckOffGuard g; + cpp2::SignInServiceReq req; + req.type_ref() = type; + req.clients_ref() = clients; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_signInService(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::signOutService(const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; + cpp2::SignOutServiceReq req; + req.type_ref() = type; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_signOutService(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::listServiceClients( + const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; + cpp2::ListServiceClientsReq req; + req.type_ref() = type; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listServiceClients(request); }, + [](cpp2::ListServiceClientsResp&& resp) -> decltype(auto) { + return std::move(resp).get_clients(); + }, + std::move(promise)); + return future; +} + +StatusOr> MetaClient::getServiceClientsFromCache( + const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + auto& metadata = *metadata_.load(); + if (type == cpp2::ExternalServiceType::ELASTICSEARCH) { + auto sIter = metadata.serviceClientList_.find(type); + if (sIter != metadata.serviceClientList_.end()) { + return sIter->second; + } + } + return Status::Error("Service not found!"); +} + +folly::Future> MetaClient::createFTIndex(const std::string& name, + const cpp2::FTIndex& index) { + memory::MemoryCheckOffGuard g; + cpp2::CreateFTIndexReq req; + req.fulltext_index_name_ref() = name; + req.index_ref() = index; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createFTIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::dropFTIndex(GraphSpaceID spaceId, + const std::string& name) { + memory::MemoryCheckOffGuard g; + cpp2::DropFTIndexReq req; + req.fulltext_index_name_ref() = name; + req.space_id_ref() = spaceId; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_dropFTIndex(request); }, + [](cpp2::ExecResp&& resp) -> bool { + return resp.get_code() == nebula::cpp2::ErrorCode::SUCCEEDED; + }, + std::move(promise), + true); + return future; +} + +folly::Future>> +MetaClient::listFTIndexes() { + memory::MemoryCheckOffGuard g; + cpp2::ListFTIndexesReq req; + folly::Promise>> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listFTIndexes(request); }, + [](cpp2::ListFTIndexesResp&& resp) -> decltype(auto) { + return std::move(resp).get_indexes(); + }, + std::move(promise)); + return future; +} + +StatusOr> MetaClient::getFTIndexesFromCache() { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + return metadata.fulltextIndexMap_; +} + +StatusOr> MetaClient::getFTIndexBySpaceFromCache( + GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + std::unordered_map indexes; + for (const auto& it : metadata.fulltextIndexMap_) { + if (it.second.get_space_id() == spaceId) { + indexes[it.first] = it.second; + } + } + return indexes; +} + +StatusOr> MetaClient::getFTIndexFromCache( + GraphSpaceID spaceId, int32_t schemaId, const std::string& field) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + for (auto& it : metadata.fulltextIndexMap_) { + auto id = it.second.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type + ? it.second.get_depend_schema().get_edge_type() + : it.second.get_depend_schema().get_tag_id(); + // There will only be one field. However, in order to minimize changes, the IDL was not modified + auto f = it.second.fields()->front(); + if (it.second.get_space_id() == spaceId && id == schemaId && f == field) { + return std::make_pair(it.first, it.second); + } + } + return Status::IndexNotFound(); +} + +StatusOr> MetaClient::getFTIndexFromCache( + GraphSpaceID spaceId, int32_t schemaId) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + std::unordered_map ret; + for (auto& it : metadata.fulltextIndexMap_) { + auto id = it.second.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type + ? it.second.get_depend_schema().get_edge_type() + : it.second.get_depend_schema().get_tag_id(); + if (it.second.get_space_id() == spaceId && id == schemaId) { + ret[it.first] = it.second; + } + } + return ret; +} + +StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID spaceId, + const std::string& name) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + const auto& metadata = *metadata_.load(); + if (metadata.fulltextIndexMap_.find(name) != fulltextIndexMap_.end() && + metadata.fulltextIndexMap_.at(name).get_space_id() != spaceId) { + return Status::IndexNotFound(); + } + return metadata.fulltextIndexMap_.at(name); +} + +folly::Future> MetaClient::createSession( + const std::string& userName, const HostAddr& graphAddr, const std::string& clientIp) { + memory::MemoryCheckOffGuard g; + cpp2::CreateSessionReq req; + req.user_ref() = userName; + req.graph_addr_ref() = graphAddr; + req.client_ip_ref() = clientIp; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_createSession(request); }, + [](cpp2::CreateSessionResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::updateSessions( + const std::vector& sessions) { + memory::MemoryCheckOffGuard g; + cpp2::UpdateSessionsReq req; + req.sessions_ref() = sessions; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_updateSessions(request); }, + [](cpp2::UpdateSessionsResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::listSessions() { + memory::MemoryCheckOffGuard g; + cpp2::ListSessionsReq req; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_listSessions(request); }, + [](cpp2::ListSessionsResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::getSession(SessionID sessionId) { + memory::MemoryCheckOffGuard g; + cpp2::GetSessionReq req; + req.session_id_ref() = sessionId; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getSession(request); }, + [](cpp2::GetSessionResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise)); + return future; +} + +folly::Future> MetaClient::removeSessions( + const std::vector& sessionIds) { + memory::MemoryCheckOffGuard g; + cpp2::RemoveSessionReq req; + req.session_ids_ref() = sessionIds; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_removeSession(request); }, + [](cpp2::RemoveSessionResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::killQuery( + std::unordered_map> killQueries) { + memory::MemoryCheckOffGuard g; + cpp2::KillQueryReq req; + req.kill_queries_ref() = std::move(killQueries); + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_killQuery(request); }, + [](cpp2::ExecResp&& resp) -> decltype(auto) { return std::move(resp); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::getWorkerId(std::string ipAddr) { + memory::MemoryCheckOffGuard g; + cpp2::GetWorkerIdReq req; + req.host_ref() = std::move(ipAddr); + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getWorkerId(request); }, + [](cpp2::GetWorkerIdResp&& resp) -> int64_t { return std::move(resp).get_workerid(); }, + std::move(promise), + true); + return future; +} + +folly::Future> MetaClient::getSegmentId(int64_t length) { + memory::MemoryCheckOffGuard g; + auto req = cpp2::GetSegmentIdReq(); + req.length_ref() = length; + + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_getSegmentId(request); }, + [](cpp2::GetSegmentIdResp&& resp) -> int64_t { return std::move(resp).get_segment_id(); }, + std::move(promise), + true); + return future; +} + +bool MetaClient::loadSessions() { + memory::MemoryCheckOffGuard g; + auto session_list = listSessions().get(); + if (!session_list.ok()) { + LOG(ERROR) << "List sessions failed, status:" << session_list.status(); + return false; + } + sessionMap_.clear(); + killedPlans_.clear(); + for (auto& session : session_list.value().get_sessions()) { + sessionMap_[session.get_session_id()] = session; + for (auto& query : session.get_queries()) { + if (query.second.get_status() == cpp2::QueryStatus::KILLING) { + killedPlans_.insert({session.get_session_id(), query.first}); + } + } + } + return true; +} + +StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& session_id) { + memory::MemoryCheckOffGuard g; + if (!ready_) { + return Status::Error("Not ready!"); + } + folly::rcu_reader guard; + auto& sessionMap = metadata_.load()->sessionMap_; + auto it = sessionMap.find(session_id); + if (it != sessionMap.end()) { + return it->second; + } + return Status::SessionNotFound(); +} + +bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) { + memory::MemoryCheckOffGuard g; + static thread_local int check_counter = 0; + // Inaccurate in a multi-threaded environment, but it is not important + check_counter = (check_counter + 1) & ((1 << FLAGS_check_plan_killed_frequency) - 1); + if (check_counter != 0) { + return false; + } + folly::rcu_reader guard; + return metadata_.load()->killedPlans_.count({sessionId, planId}); +} + +Status MetaClient::verifyVersion() { + memory::MemoryCheckOffGuard g; + auto req = cpp2::VerifyClientVersionReq(); + req.build_version_ref() = getOriginVersion(); + req.host_ref() = options_.localHost_; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_verifyClientVersion(request); }, + [](cpp2::VerifyClientVersionResp&& resp) { return std::move(resp); }, + std::move(promise)); + + auto respStatus = std::move(future).get(); + if (!respStatus.ok()) { + return respStatus.status(); + } + auto resp = std::move(respStatus).value(); + if (resp.get_code() != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Client verified failed: %s", resp.get_error_msg()->c_str()); + } + return Status::OK(); +} + +Status MetaClient::saveVersionToMeta() { + memory::MemoryCheckOffGuard g; + auto req = cpp2::SaveGraphVersionReq(); + req.build_version_ref() = getOriginVersion(); + req.host_ref() = options_.localHost_; + folly::Promise> promise; + auto future = promise.getFuture(); + getResponse( + std::move(req), + [](auto client, auto request) { return client->future_saveGraphVersion(request); }, + [](cpp2::SaveGraphVersionResp&& resp) { return std::move(resp); }, + std::move(promise)); + + auto respStatus = std::move(future).get(); + if (!respStatus.ok()) { + return respStatus.status(); + } + auto resp = std::move(respStatus).value(); + if (resp.get_code() != nebula::cpp2::ErrorCode::SUCCEEDED) { + return Status::Error("Failed to save graph version into meta, error code: %s", + apache::thrift::util::enumNameSafe(resp.get_code()).c_str()); + } + return Status::OK(); +} + +} // namespace meta +} // namespace nebula diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h new file mode 100644 index 0000000..471804d --- /dev/null +++ b/src/clients/meta/MetaClient.h @@ -0,0 +1,869 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_META_METACLIENT_H_ +#define CLIENTS_META_METACLIENT_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "common/base/Base.h" +#include "common/base/ObjectPool.h" +#include "common/base/StatusOr.h" +#include "common/meta/Common.h" +#include "common/meta/GflagsManager.h" +#include "common/meta/NebulaSchemaProvider.h" +#include "common/thread/GenericWorker.h" +#include "common/thrift/ThriftClientManager.h" +#include "interface/gen-cpp2/MetaServiceAsyncClient.h" +#include "interface/gen-cpp2/common_types.h" +#include "interface/gen-cpp2/meta_types.h" +#include "kvstore/DiskManager.h" + +DECLARE_int32(meta_client_retry_times); +DECLARE_int32(heartbeat_interval_secs); + +namespace nebula { +namespace storage { +class MetaClientTestUpdater; +} // namespace storage +} // namespace nebula + +namespace nebula { +namespace meta { + +using PartsAlloc = std::unordered_map>; +using SpaceIdName = std::pair; +using HostStatus = std::pair; + +// struct for in cache +// the different version of tag schemas, from oldest to latest +using TagSchemas = + std::unordered_map>>; + +// Mapping of tagId and a *single* tag schema +using TagSchema = std::unordered_map>; + +// the different version of edge schema, from oldest to latest +using EdgeSchemas = + std::unordered_map>>; + +// Mapping of edgeType and a *single* edge schema +using EdgeSchema = std::unordered_map>; + +// Space and index Name => IndexID +// Get IndexID via space ID and index name +using NameIndexMap = std::unordered_map, IndexID>; + +// Index ID => Index Item +// Get Index Structure by indexID +using Indexes = std::unordered_map>; + +// Listeners is a map of ListenerHost => , used to add/remove listener on local host +using Listeners = + std::unordered_map>>; + +// Get services +using ServiceClientsList = + std::unordered_map>; + +struct SpaceInfoCache { + cpp2::SpaceDesc spaceDesc_; + PartsAlloc partsAlloc_; + std::unordered_map> partsOnHost_; + std::vector tagItemVec_; + TagSchemas tagSchemas_; + std::vector edgeItemVec_; + EdgeSchemas edgeSchemas_; + std::vector tagIndexItemVec_; + Indexes tagIndexes_; + std::vector edgeIndexItemVec_; + Indexes edgeIndexes_; + Listeners listeners_; + std::unordered_map termOfPartition_; + + SpaceInfoCache() = default; + SpaceInfoCache(const SpaceInfoCache& info) + : spaceDesc_(info.spaceDesc_), + partsAlloc_(info.partsAlloc_), + partsOnHost_(info.partsOnHost_), + tagItemVec_(info.tagItemVec_), + tagSchemas_(info.tagSchemas_), + edgeItemVec_(info.edgeItemVec_), + edgeSchemas_(info.edgeSchemas_), + tagIndexItemVec_(info.tagIndexItemVec_), + tagIndexes_(info.tagIndexes_), + edgeIndexItemVec_(info.edgeIndexItemVec_), + edgeIndexes_(info.edgeIndexes_), + listeners_(info.listeners_), + termOfPartition_(info.termOfPartition_) {} + + ~SpaceInfoCache() = default; +}; + +using LocalCache = std::unordered_map>; + +using SpaceNameIdMap = std::unordered_map; +// get tagID via spaceId and tagName +using SpaceTagNameIdMap = std::unordered_map, TagID>; +// get edgeType via spaceId and edgeName +using SpaceEdgeNameTypeMap = std::unordered_map, EdgeType>; +// get tagName via spaceId and tagId +using SpaceTagIdNameMap = std::unordered_map, std::string>; +// get latest tag version via spaceId and TagID +using SpaceNewestTagVerMap = std::unordered_map, SchemaVer>; +// get latest edge version via spaceId and edgeType +using SpaceNewestEdgeVerMap = std::unordered_map, SchemaVer>; +// get edgeName via spaceId and edgeType +using SpaceEdgeTypeNameMap = std::unordered_map, std::string>; +// get all edgeType edgeName via spaceId +using SpaceAllEdgeMap = std::unordered_map>; + +struct LeaderInfo { + // get leader host via spaceId and partId + std::unordered_map, HostAddr> leaderMap_; + // index of picked host in all peers + std::unordered_map, size_t> pickedIndex_; +}; + +using IndexStatus = std::tuple; + +// get user roles by account +using UserRolesMap = std::unordered_map>; +// get user password by account +using UserPasswordMap = std::unordered_map; +// Mapping of user name and remaining wrong password attempts +using UserPasswordAttemptsRemain = folly::ConcurrentHashMap; +// Mapping of user name and the timestamp when the account is locked +using UserLoginLockTime = folly::ConcurrentHashMap; + +// config cache, get config via module and name +using MetaConfigMap = + std::unordered_map, cpp2::ConfigItem>; + +using FTIndexMap = std::unordered_map; + +using SessionMap = std::unordered_map; + +class MetaChangedListener { + public: + virtual ~MetaChangedListener() = default; + + virtual void onSpaceAdded(GraphSpaceID spaceId) = 0; + virtual void onSpaceRemoved(GraphSpaceID spaceId) = 0; + virtual void onSpaceOptionUpdated( + GraphSpaceID spaceId, const std::unordered_map& options) = 0; + virtual void onPartAdded(const PartHosts& partHosts) = 0; + virtual void onPartRemoved(GraphSpaceID spaceId, PartitionID partId) = 0; + virtual void onPartUpdated(const PartHosts& partHosts) = 0; + virtual void fetchLeaderInfo( + std::unordered_map>& leaders) = 0; + virtual void fetchDiskParts(kvstore::SpaceDiskPartsMap& diskParts) = 0; + virtual void onListenerSpaceAdded(GraphSpaceID spaceId, cpp2::ListenerType type) = 0; + virtual void onListenerSpaceRemoved(GraphSpaceID spaceId, cpp2::ListenerType type) = 0; + virtual void onListenerPartAdded(GraphSpaceID spaceId, + PartitionID partId, + cpp2::ListenerType type, + const std::vector& peers) = 0; + virtual void onListenerPartRemoved(GraphSpaceID spaceId, + PartitionID partId, + cpp2::ListenerType type) = 0; + virtual void onCheckRemoteListeners(GraphSpaceID spaceId, + PartitionID partId, + const std::vector& remoteListeners) = 0; +}; + +struct MetaClientOptions { + MetaClientOptions() = default; + MetaClientOptions(const MetaClientOptions& opt) + : localHost_(opt.localHost_), + clusterId_(opt.clusterId_.load()), + serviceName_(opt.serviceName_), + skipConfig_(opt.skipConfig_), + role_(opt.role_), + gitInfoSHA_(opt.gitInfoSHA_), + dataPaths_(opt.dataPaths_), + rootPath_(opt.rootPath_) {} + + // Current host address + HostAddr localHost_{"", 0}; + // Current cluster Id, it is required by storaged only. + std::atomic clusterId_{0}; + // Current service name, used in StatsManager + std::string serviceName_ = ""; + // Whether to skip the config manager + bool skipConfig_ = false; + // Host role(graph/meta/storage) using this client, and UNKNOWN role will not send heartbeat, used + // for tools such as upgrader + cpp2::HostRole role_ = cpp2::HostRole::UNKNOWN; + // gitInfoSHA of Host using this client + std::string gitInfoSHA_{""}; + // Data path list, used in storaged + std::vector dataPaths_; + // Install path, used in metad/graphd/storaged + std::string rootPath_; +}; + +class BaseMetaClient { + public: + virtual folly::Future> getSegmentId(int64_t length) = 0; + + virtual ~BaseMetaClient() = default; +}; + +class MetaClient : public BaseMetaClient { + FRIEND_TEST(ConfigManTest, MetaConfigManTest); + FRIEND_TEST(ConfigManTest, MockConfigTest); + FRIEND_TEST(ConfigManTest, RocksdbOptionsTest); + FRIEND_TEST(MetaClientTest, SimpleTest); + FRIEND_TEST(MetaClientTest, RetryWithExceptionTest); + FRIEND_TEST(MetaClientTest, RetryOnceTest); + FRIEND_TEST(MetaClientTest, RetryUntilLimitTest); + FRIEND_TEST(MetaClientTest, RocksdbOptionsTest); + FRIEND_TEST(MetaClientTest, VerifyClientTest); + FRIEND_TEST(ChainAddEdgesTest, AddEdgesLocalTest); + friend class KillQueryMetaWrapper; + friend class storage::MetaClientTestUpdater; + + public: + MetaClient(std::shared_ptr ioThreadPool, + std::vector addrs, + const MetaClientOptions& options = MetaClientOptions()); + + ~MetaClient() override; + +#ifdef BUILD_STANDALONE + StatusOr checkLocalMachineRegistered(); +#endif + + bool isMetadReady(); + + bool waitForMetadReady(int count = -1, int retryIntervalSecs = FLAGS_heartbeat_interval_secs); + + void notifyStop(); + + void stop(); + + void registerListener(MetaChangedListener* listener) { + folly::SharedMutex::WriteHolder holder(listenerLock_); + CHECK(listener_ == nullptr); + listener_ = listener; + } + + void unRegisterListener() { + folly::SharedMutex::WriteHolder holder(listenerLock_); + listener_ = nullptr; + } + + folly::Future> submitJob(GraphSpaceID spaceId, + cpp2::JobOp op, + cpp2::JobType type, + std::vector paras); + + // Operations for parts + folly::Future> createSpace(meta::cpp2::SpaceDesc spaceDesc, + bool ifNotExists = false); + + folly::Future> createSpaceAs(const std::string& oldSpaceName, + const std::string& newSpaceName, + bool ifNotExists); + + folly::Future>> listSpaces(); + + folly::Future> getSpace(std::string name); + + folly::Future> dropSpace(std::string name, bool ifExists = false); + + // clear space data, but keep the space schema. + folly::Future> clearSpace(std::string name, bool ifExists = false); + + folly::Future>> listHosts( + cpp2::ListHostType type = cpp2::ListHostType::ALLOC); + + folly::Future> alterSpace(const std::string& spaceName, + meta::cpp2::AlterSpaceOp op, + const std::vector& paras); + + folly::Future>> listParts(GraphSpaceID spaceId, + std::vector partIds); + + using PartTerms = std::unordered_map; + folly::Future> getPartsAlloc(GraphSpaceID spaceId, + MetaClient::PartTerms* partTerms = nullptr); + + // Operations for schema + folly::Future> createTagSchema(GraphSpaceID spaceId, + std::string name, + cpp2::Schema schema, + bool ifNotExists = false); + + folly::Future> alterTagSchema(GraphSpaceID spaceId, + std::string name, + std::vector items, + cpp2::SchemaProp schemaProp); + + folly::Future>> listTagSchemas(GraphSpaceID spaceId); + + folly::Future> dropTagSchema(GraphSpaceID spaceId, + std::string name, + bool ifExists = false); + + // Return the latest schema when ver = -1 + folly::Future> getTagSchema(GraphSpaceID spaceId, + std::string name, + SchemaVer version = -1); + + folly::Future> createEdgeSchema(GraphSpaceID spaceId, + std::string name, + cpp2::Schema schema, + bool ifNotExists = false); + + folly::Future> alterEdgeSchema(GraphSpaceID spaceId, + std::string name, + std::vector items, + cpp2::SchemaProp schemaProp); + + folly::Future>> listEdgeSchemas(GraphSpaceID spaceId); + + // Return the latest schema when ver = -1 + folly::Future> getEdgeSchema(GraphSpaceID spaceId, + std::string name, + SchemaVer version = -1); + + folly::Future> dropEdgeSchema(GraphSpaceID spaceId, + std::string name, + bool ifExists = false); + + // Operations for index + folly::Future> createTagIndex( + GraphSpaceID spaceID, + std::string indexName, + std::string tagName, + std::vector fields, + bool ifNotExists = false, + const meta::cpp2::IndexParams* indexParams = nullptr, + const std::string* comment = nullptr); + + // Remove the define of tag index + folly::Future> dropTagIndex(GraphSpaceID spaceId, + std::string name, + bool ifExists = false); + + folly::Future> getTagIndex(GraphSpaceID spaceId, std::string name); + + folly::Future>> listTagIndexes(GraphSpaceID spaceId); + + folly::Future> rebuildTagIndex(GraphSpaceID spaceID, std::string name); + + folly::Future>> listTagIndexStatus(GraphSpaceID spaceId); + + folly::Future> createEdgeIndex(GraphSpaceID spaceID, + std::string indexName, + std::string edgeName, + std::vector fields, + bool ifNotExists = false, + const cpp2::IndexParams* indexParams = nullptr, + const std::string* comment = nullptr); + + // Remove the definition of edge index + folly::Future> dropEdgeIndex(GraphSpaceID spaceId, + std::string name, + bool ifExists = false); + + folly::Future> getEdgeIndex(GraphSpaceID spaceId, std::string name); + + folly::Future>> listEdgeIndexes(GraphSpaceID spaceId); + + folly::Future> rebuildEdgeIndex(GraphSpaceID spaceId, std::string name); + + folly::Future>> listEdgeIndexStatus(GraphSpaceID spaceId); + + // Operations for users. + folly::Future> createUser(std::string account, + std::string password, + bool ifNotExists); + + folly::Future> dropUser(std::string account, bool ifExists); + + folly::Future> alterUser(std::string account, std::string password); + + folly::Future> grantToUser(cpp2::RoleItem roleItem); + + folly::Future> revokeFromUser(cpp2::RoleItem roleItem); + + folly::Future>> listUsers(); + + folly::Future>> listRoles(GraphSpaceID space); + + folly::Future> changePassword(std::string account, + std::string newPwd, + std::string oldPwd); + + folly::Future>> getUserRoles(std::string account); + + // Operations for config + folly::Future> regConfig(const std::vector& items); + + folly::Future>> getConfig(const cpp2::ConfigModule& module, + const std::string& name); + + folly::Future> setConfig(const cpp2::ConfigModule& module, + const std::string& name, + const Value& value); + + folly::Future>> listConfigs( + const cpp2::ConfigModule& module); + + folly::Future> createSnapshot(); + + folly::Future> dropSnapshot(const std::string& name); + + folly::Future>> listSnapshots(); + + // Operations for listener. + + folly::Future> addListener(GraphSpaceID spaceId, + cpp2::ListenerType type, + std::vector hosts); + + folly::Future> removeListener(GraphSpaceID spaceId, cpp2::ListenerType type); + + folly::Future>> listListener(GraphSpaceID spaceId); + + StatusOr>> + getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAddr& host); + + // Given host, get the all peers info. This function is used for listener to start up related + // listener part + StatusOr getListenersByHostFromCache(const HostAddr& host); + + // Get listener address of given (spaceId + partId + type) + StatusOr getListenerHostsBySpacePartType(GraphSpaceID spaceId, + PartitionID partId, + cpp2::ListenerType type); + + // Get all listener type + address of given (spaceId + partId) + StatusOr> getListenerHostTypeBySpacePartType(GraphSpaceID spaceId, + PartitionID partId); + + // Operations for services + folly::Future> signInService(const cpp2::ExternalServiceType& type, + const std::vector& clients); + + folly::Future> signOutService(const cpp2::ExternalServiceType& type); + + folly::Future> listServiceClients( + const cpp2::ExternalServiceType& type); + + StatusOr> getServiceClientsFromCache( + const cpp2::ExternalServiceType& type); + + // Operations for fulltext index. + + folly::Future> createFTIndex(const std::string& name, const cpp2::FTIndex& index); + + folly::Future> dropFTIndex(GraphSpaceID spaceId, const std::string& name); + + folly::Future>> listFTIndexes(); + + StatusOr> getFTIndexesFromCache(); + + StatusOr> getFTIndexBySpaceFromCache( + GraphSpaceID spaceId); + + StatusOr> getFTIndexFromCache(GraphSpaceID spaceId, + int32_t schemaId, + const std::string& field); + + StatusOr> getFTIndexFromCache(GraphSpaceID spaceId, + int32_t schemaId); + + StatusOr getFTIndexByNameFromCache(GraphSpaceID spaceId, const std::string& name); + + // session + folly::Future> createSession(const std::string& userName, + const HostAddr& graphAddr, + const std::string& clientIp); + + folly::Future> updateSessions( + const std::vector& sessions); + + folly::Future> listSessions(); + + folly::Future> getSession(SessionID sessionId); + + folly::Future> removeSessions( + const std::vector& sessionIds); + + folly::Future> killQuery( + std::unordered_map> killQueries); + + // Operations for cache. + StatusOr getSpaceIdByNameFromCache(const std::string& name); + + StatusOr getSpaceNameByIdFromCache(GraphSpaceID spaceId); + + StatusOr getSpaceVidLen(const GraphSpaceID& space); + + StatusOr getSpaceVidType(const GraphSpaceID& space); + + StatusOr getSpaceDesc(const GraphSpaceID& space); + + StatusOr getIsolationLevel(GraphSpaceID spaceId); + + StatusOr getTagIDByNameFromCache(const GraphSpaceID& space, const std::string& name); + + StatusOr getTagNameByIdFromCache(const GraphSpaceID& space, const TagID& tagId); + + StatusOr getLatestTagVersionFromCache(const GraphSpaceID& space, const TagID& tagId); + + StatusOr getLatestEdgeVersionFromCache(const GraphSpaceID& space, + const EdgeType& edgeType); + + StatusOr getEdgeTypeByNameFromCache(const GraphSpaceID& space, const std::string& name); + + StatusOr getEdgeNameByTypeFromCache(const GraphSpaceID& space, + const EdgeType edgeType); + + // get all latest version edge + StatusOr> getAllEdgeFromCache(const GraphSpaceID& space); + + PartsMap getPartsMapFromCache(const HostAddr& host); + + StatusOr getPartHostsFromCache(GraphSpaceID spaceId, PartitionID partId); + + Status checkPartExistInCache(const HostAddr& host, GraphSpaceID spaceId, PartitionID partId); + + Status checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId); + + StatusOr partsNum(GraphSpaceID spaceId); + + PartitionID partId(int32_t numParts, VertexID id) const; + + StatusOr> getTagSchemaFromCache(GraphSpaceID spaceId, + TagID tagID, + SchemaVer ver = -1); + + StatusOr> getEdgeSchemaFromCache(GraphSpaceID spaceId, + EdgeType edgeType, + SchemaVer ver = -1); + + StatusOr getAllVerTagSchema(GraphSpaceID spaceId); + + StatusOr getAllLatestVerTagSchema(const GraphSpaceID& spaceId); + + StatusOr getAllVerEdgeSchema(GraphSpaceID spaceId); + + StatusOr getAllLatestVerEdgeSchemaFromCache(const GraphSpaceID& spaceId); + + StatusOr> getTagIndexByNameFromCache(const GraphSpaceID space, + const std::string& name); + + StatusOr> getEdgeIndexByNameFromCache(const GraphSpaceID space, + const std::string& name); + + StatusOr> getTagIndexFromCache(GraphSpaceID spaceId, + IndexID indexID); + + StatusOr getRelatedTagIDByIndexNameFromCache(const GraphSpaceID space, + const std::string& indexName); + + StatusOr> getEdgeIndexFromCache(GraphSpaceID spaceId, + IndexID indexID); + + StatusOr getRelatedEdgeTypeByIndexNameFromCache(const GraphSpaceID space, + const std::string& indexName); + + StatusOr>> getTagIndexesFromCache( + GraphSpaceID spaceId); + + StatusOr>> getEdgeIndexesFromCache( + GraphSpaceID spaceId); + + Status checkTagIndexed(GraphSpaceID space, IndexID indexID); + + Status checkEdgeIndexed(GraphSpaceID space, IndexID indexID); + + const std::vector& getAddresses(); + + std::vector getRolesByUserFromCache(const std::string& user); + + Status authCheckFromCache(const std::string& account, const std::string& password); + + StatusOr getTermFromCache(GraphSpaceID spaceId, PartitionID); + + bool checkShadowAccountFromCache(const std::string& account); + + StatusOr> getStorageHosts(); + + StatusOr getSessionFromCache(const nebula::SessionID& session_id); + + bool checkIsPlanKilled(SessionID session_id, ExecutionPlanID plan_id); + + StatusOr getStorageLeaderFromCache(GraphSpaceID spaceId, PartitionID partId); + + void updateStorageLeader(GraphSpaceID spaceId, PartitionID partId, const HostAddr& leader); + + void invalidStorageLeader(GraphSpaceID spaceId, PartitionID partId); + + StatusOr getLeaderInfo(); + + folly::Future> addHosts(std::vector hosts); + + folly::Future> dropHosts(std::vector hosts); + + folly::Future> mergeZone(std::vector zones, std::string zoneName); + + folly::Future> divideZone( + std::string zoneName, std::unordered_map> zoneItems); + + folly::Future> renameZone(std::string originalZoneName, std::string zoneName); + + folly::Future> dropZone(std::string zoneName); + + folly::Future> addHostsIntoZone(std::vector hosts, + std::string zoneName, + bool isNew); + + folly::Future>> getZone(std::string zoneName); + + folly::Future>> listZones(); + + Status refreshCache(); + + folly::Future> getStats(GraphSpaceID spaceId); + + folly::Future> reportTaskFinish( + GraphSpaceID spaceId, + int32_t jobId, + int32_t taskId, + nebula::cpp2::ErrorCode taskErrCode, + cpp2::StatsItem* statisticItem); + + folly::Future> getWorkerId(std::string ipAddr); + + folly::Future> getSegmentId(int64_t length) override; + + HostAddr getMetaLeader() { + return leader_; + } + + int64_t HeartbeatTime() { + return heartbeatTime_; + } + + std::string getLocalIp() { + return options_.localHost_.toString(); + } + + protected: + // Return true if load succeeded. + bool loadData(); + bool loadCfg(); + void heartBeatThreadFunc(); + + bool registerCfg(); + void updateGflagsValue(const cpp2::ConfigItem& item); + void updateNestedGflags(const std::unordered_map& nameValues); + + bool loadSchemas(GraphSpaceID spaceId, + std::shared_ptr spaceInfoCache, + SpaceTagNameIdMap& tagNameIdMap, + SpaceTagIdNameMap& tagIdNameMap, + SpaceEdgeNameTypeMap& edgeNameTypeMap, + SpaceEdgeTypeNameMap& edgeTypeNamemap, + SpaceNewestTagVerMap& newestTagVerMap, + SpaceNewestEdgeVerMap& newestEdgeVerMap, + SpaceAllEdgeMap& allEdgemap); + + bool loadUsersAndRoles(); + + bool loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache); + + bool loadListeners(GraphSpaceID spaceId, std::shared_ptr cache); + + bool loadGlobalServiceClients(); + + bool loadFulltextIndexes(); + + bool loadSessions(); + + void loadLeader(const std::vector& hostItems, + const SpaceNameIdMap& spaceIndexByName); + + folly::Future> heartbeat(); + + std::unordered_map> reverse(const PartsAlloc& parts); + + void updateActive() { + folly::SharedMutex::WriteHolder holder(hostLock_); + active_ = addrs_[folly::Random::rand64(addrs_.size())]; + } + + void updateLeader(HostAddr leader = {"", 0}) { + folly::SharedMutex::WriteHolder holder(hostLock_); + if (leader != HostAddr("", 0)) { + leader_ = leader; + } else { + leader_ = addrs_[folly::Random::rand64(addrs_.size())]; + } + } + + // part diff + void diff(const LocalCache& oldCache, const LocalCache& newCache); + + void listenerDiff(const LocalCache& oldCache, const LocalCache& newCache); + + // add remote listener as part peers + void loadRemoteListeners(); + + template + Status handleResponse(const RESP& resp); + + template , Request)>::type::value_type, + class Response = typename std::result_of::type> + void getResponse(Request req, + RemoteFunc remoteFunc, + RespGenerator respGen, + folly::Promise> pro, + bool toLeader = true, + int32_t retry = 0, + int32_t retryLimit = FLAGS_meta_client_retry_times); + + std::vector toSpaceIdName(const std::vector& tIdNames); + + PartsMap doGetPartsMap(const HostAddr& host, const LocalCache& localCache); + + ListenersMap doGetListenersMap(const HostAddr& host, const LocalCache& localCache); + + // Checks if the client version is compatible with the server version by checking the + // whitelist in meta. + Status verifyVersion(); + + // Save the version of the graph service into meta so that it could be looked up. + // This method should be only called in the internal client. + Status saveVersionToMeta(); + + private: + std::shared_ptr ioThreadPool_; + std::shared_ptr> clientsMan_; + + // heartbeat is a single thread, maybe leaderIdsLock_ and diskPartsLock_ is useless? + // leaderIdsLock_ is used to protect leaderIds_ + std::unordered_map> leaderIds_; + folly::SharedMutex leaderIdsLock_; + // diskPartsLock_ is used to protect diskParts_; + kvstore::SpaceDiskPartsMap diskParts_; + folly::SharedMutex diskPartsLock_; + + std::atomic localDataLastUpdateTime_{-1}; + std::atomic localCfgLastUpdateTime_{-1}; + std::atomic metadLastUpdateTime_{0}; + + int64_t metaServerVersion_{-1}; + static constexpr int64_t EXPECT_META_VERSION = 4; + + // leadersLock_ is used to protect leadersInfo + folly::SharedMutex leadersLock_; + LeaderInfo leadersInfo_; + + LocalCache localCache_; + std::vector addrs_; + // The lock used to protect active_ and leader_. + folly::SharedMutex hostLock_; + HostAddr active_; + HostAddr leader_; + HostAddr localHost_; + + // Only report dir info once when started + bool dirInfoReported_ = false; + + struct MetaData { + int64_t localLastUpdateTime_{-2}; + LocalCache localCache_; + SpaceNameIdMap spaceIndexByName_; + SpaceTagNameIdMap spaceTagIndexByName_; + SpaceEdgeNameTypeMap spaceEdgeIndexByName_; + SpaceEdgeTypeNameMap spaceEdgeIndexByType_; + SpaceTagIdNameMap spaceTagIndexById_; + SpaceNewestTagVerMap spaceNewestTagVerMap_; + SpaceNewestEdgeVerMap spaceNewestEdgeVerMap_; + SpaceAllEdgeMap spaceAllEdgeMap_; + + UserRolesMap userRolesMap_; + std::vector storageHosts_; + FTIndexMap fulltextIndexMap_; + UserPasswordMap userPasswordMap_; + + SessionMap sessionMap_; + folly::F14FastSet> killedPlans_; + + ServiceClientsList serviceClientList_; + }; + + void addSchemaField(NebulaSchemaProvider* schema, const cpp2::ColumnDef& col); + + TagSchemas buildTagSchemas(std::vector tagItemVec); + EdgeSchemas buildEdgeSchemas(std::vector edgeItemVec); + + std::unique_ptr bgThread_; + SpaceNameIdMap spaceIndexByName_; + SpaceTagNameIdMap spaceTagIndexByName_; + SpaceEdgeNameTypeMap spaceEdgeIndexByName_; + SpaceEdgeTypeNameMap spaceEdgeIndexByType_; + SpaceTagIdNameMap spaceTagIndexById_; + SpaceNewestTagVerMap spaceNewestTagVerMap_; + SpaceNewestEdgeVerMap spaceNewestEdgeVerMap_; + SpaceAllEdgeMap spaceAllEdgeMap_; + + UserRolesMap userRolesMap_; + UserPasswordMap userPasswordMap_; + UserPasswordAttemptsRemain userPasswordAttemptsRemain_; + UserLoginLockTime userLoginLockTime_; + + NameIndexMap tagNameIndexMap_; + NameIndexMap edgeNameIndexMap_; + + // Global service client + ServiceClientsList serviceClientList_; + FTIndexMap fulltextIndexMap_; + + // The listener_ is the NebulaStore + MetaChangedListener* listener_{nullptr}; + // The lock used to protect listener_ + folly::SharedMutex listenerLock_; + std::atomic clusterId_{0}; + bool isRunning_{false}; + bool sendHeartBeat_{false}; + std::atomic_bool ready_{false}; + MetaConfigMap metaConfigMap_; + folly::SharedMutex configCacheLock_; + cpp2::ConfigModule gflagsModule_{cpp2::ConfigModule::UNKNOWN}; + std::atomic_bool configReady_{false}; + std::vector gflagsDeclared_; + bool skipConfig_ = false; + MetaClientOptions options_; + std::vector storageHosts_; + int64_t heartbeatTime_; + SessionMap sessionMap_; + folly::F14FastSet> killedPlans_; + std::atomic metadata_; +}; + +} // namespace meta +} // namespace nebula +#endif // CLIENTS_META_METACLIENT_H_ diff --git a/src/clients/meta/stats/CMakeLists.txt b/src/clients/meta/stats/CMakeLists.txt new file mode 100644 index 0000000..68a1848 --- /dev/null +++ b/src/clients/meta/stats/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + meta_client_stats_obj + OBJECT + MetaClientStats.cpp + ) diff --git a/src/clients/meta/stats/MetaClientStats.cpp b/src/clients/meta/stats/MetaClientStats.cpp new file mode 100644 index 0000000..3f73f4f --- /dev/null +++ b/src/clients/meta/stats/MetaClientStats.cpp @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/meta/stats/MetaClientStats.h" + +namespace nebula { + +stats::CounterId kNumRpcSentToMetad; +stats::CounterId kNumRpcSentToMetadFailed; + +void initMetaClientStats() { + kNumRpcSentToMetad = stats::StatsManager::registerStats("num_rpc_sent_to_metad", "rate, sum"); + kNumRpcSentToMetadFailed = + stats::StatsManager::registerStats("num_rpc_sent_to_metad_failed", "rate, sum"); +} + +} // namespace nebula diff --git a/src/clients/meta/stats/MetaClientStats.h b/src/clients/meta/stats/MetaClientStats.h new file mode 100644 index 0000000..9e55d37 --- /dev/null +++ b/src/clients/meta/stats/MetaClientStats.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTIS_META_STATS_METACLIENTSTATS_H +#define CLIENTIS_META_STATS_METACLIENTSTATS_H +#include "common/stats/StatsManager.h" + +namespace nebula { + +extern stats::CounterId kNumRpcSentToMetad; +extern stats::CounterId kNumRpcSentToMetadFailed; + +void initMetaClientStats(); + +} // namespace nebula +#endif diff --git a/src/clients/meta/test/CMakeLists.txt b/src/clients/meta/test/CMakeLists.txt new file mode 100644 index 0000000..e517b7c --- /dev/null +++ b/src/clients/meta/test/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_test( + NAME file_based_cluster_id_man_test + SOURCES FileBasedClusterIdManTest.cpp + OBJECTS + $ + $ + $ + $ + LIBRARIES gtest +) diff --git a/src/clients/meta/test/FileBasedClusterIdManTest.cpp b/src/clients/meta/test/FileBasedClusterIdManTest.cpp new file mode 100644 index 0000000..f701e19 --- /dev/null +++ b/src/clients/meta/test/FileBasedClusterIdManTest.cpp @@ -0,0 +1,33 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "clients/meta/FileBasedClusterIdMan.h" +#include "common/base/Base.h" +#include "common/fs/TempDir.h" + +namespace nebula { +namespace meta { + +TEST(FileBasedClusterIdManTest, ReadWriteTest) { + fs::TempDir rootPath("/tmp/FileBasedClusterIdManTest.XXXXXX"); + auto clusterId = FileBasedClusterIdMan::create("127.0.0.1:44500"); + CHECK_NE(0, clusterId); + auto file = folly::stringPrintf("%s/cluster.id", rootPath.path()); + CHECK(FileBasedClusterIdMan::persistInFile(clusterId, file)); + auto ret = FileBasedClusterIdMan::getClusterIdFromFile(file); + CHECK_EQ(clusterId, ret); +} + +} // namespace meta +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + return RUN_ALL_TESTS(); +} diff --git a/src/clients/storage/CMakeLists.txt b/src/clients/storage/CMakeLists.txt new file mode 100644 index 0000000..3429372 --- /dev/null +++ b/src/clients/storage/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + storage_client_obj OBJECT + StorageClient.cpp +) + + +nebula_add_library( + storage_client_base_obj OBJECT + StorageClientBase.cpp +) + + +nebula_add_library( + internal_storage_client_obj OBJECT + InternalStorageClient.cpp +) + +nebula_add_subdirectory(stats) + diff --git a/src/clients/storage/GeneralStorageClient.h b/src/clients/storage/GeneralStorageClient.h new file mode 100644 index 0000000..ed5e71b --- /dev/null +++ b/src/clients/storage/GeneralStorageClient.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_GENERALSTORAGECLIENT_H_ +#define CLIENTS_STORAGE_GENERALSTORAGECLIENT_H_ + +#include "clients/meta/MetaClient.h" +#include "clients/storage/StorageClientBase.h" +#include "common/base/Base.h" +#include "common/datatypes/KeyValue.h" +#include "common/thrift/ThriftClientManager.h" +#include "interface/gen-cpp2/GeneralStorageServiceAsyncClient.h" + +namespace nebula { +namespace storage { + +/** + * A wrapper class for GeneralStorageServiceAsyncClient thrift API + * + * The class is NOT reentrant + */ +class GeneralStorageClient + : public StorageClientBase< + cpp2::GeneralStorageServiceAsyncClient, + thrift::ThriftClientManager> { + using Parent = + StorageClientBase>; + + public: + GeneralStorageClient(std::shared_ptr ioThreadPool, + meta::MetaClient* metaClient) + : Parent(ioThreadPool, metaClient) {} + virtual ~GeneralStorageClient() {} + + folly::SemiFuture> get(GraphSpaceID space, + std::vector&& keys, + bool returnPartly = false, + folly::EventBase* evb = nullptr); + + folly::SemiFuture> put(GraphSpaceID space, + std::vector kvs, + folly::EventBase* evb = nullptr); + + folly::SemiFuture> remove(GraphSpaceID space, + std::vector keys, + folly::EventBase* evb = nullptr); +}; + +} // namespace storage +} // namespace nebula +#endif // CLIENTS_STORAGE_GENERALSTORAGECLIENT_H_ diff --git a/src/clients/storage/InternalStorageClient.cpp b/src/clients/storage/InternalStorageClient.cpp new file mode 100644 index 0000000..f7e9457 --- /dev/null +++ b/src/clients/storage/InternalStorageClient.cpp @@ -0,0 +1,183 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/storage/InternalStorageClient.h" + +#include "common/base/Base.h" + +namespace nebula { +namespace storage { + +constexpr int32_t kInternalPortOffset = -2; + +template +::nebula::cpp2::ErrorCode getErrorCode(T& tryResp) { + if (!tryResp.hasValue()) { + LOG(ERROR) << tryResp.exception().what(); + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; + } + + auto& stResp = tryResp.value(); + if (!stResp.ok()) { + switch (stResp.status().code()) { + case Status::Code::kLeaderChanged: + return nebula::cpp2::ErrorCode::E_LEADER_CHANGED; + case Status::Code::kError: + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; + default: + LOG(ERROR) << "not impl error transform: code=" + << static_cast(stResp.status().code()); + } + return nebula::cpp2::ErrorCode::E_UNKNOWN; + } + + auto& failedPart = stResp.value().get_result().get_failed_parts(); + for (auto& p : failedPart) { + return p.code; + } + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + +void InternalStorageClient::chainUpdateEdge(cpp2::UpdateEdgeRequest& reversedRequest, + TermID termOfSrc, + std::optional optVersion, + folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::EventBase* evb) { + auto spaceId = reversedRequest.get_space_id(); + auto partId = reversedRequest.get_part_id(); + auto optLeader = getLeader(spaceId, partId); + if (!optLeader.ok()) { + LOG(WARNING) << folly::sformat("failed to get leader, space {}, part {}. ", spaceId, partId) + << optLeader.status(); + p.setValue(::nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND); + return; + } + HostAddr& leader = optLeader.value(); + leader.port += kInternalPortOffset; + VLOG(1) << "leader host: " << leader; + + cpp2::ChainUpdateEdgeRequest chainReq; + chainReq.update_edge_request_ref() = reversedRequest; + chainReq.term_ref() = termOfSrc; + if (optVersion) { + chainReq.edge_version_ref() = optVersion.value(); + } + auto resp = getResponse( + evb, + leader, + chainReq, + [](cpp2::InternalStorageServiceAsyncClient* client, const cpp2::ChainUpdateEdgeRequest& r) { + return client->future_chainUpdateEdge(r); + }); + + std::move(resp).thenTry([=, p = std::move(p)](auto&& t) mutable { + auto code = getErrorCode(t); + VLOG(1) << "chainUpdateEdge rpc: " << apache::thrift::util::enumNameSafe(code); + if (code == ::nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + chainUpdateEdge(reversedRequest, termOfSrc, optVersion, std::move(p)); + } else { + p.setValue(code); + } + return; + }); +} + +void InternalStorageClient::chainAddEdges(cpp2::AddEdgesRequest& directReq, + TermID termId, + std::optional optVersion, + folly::Promise&& p, + folly::EventBase* evb) { + auto spaceId = directReq.get_space_id(); + auto partId = directReq.get_parts().begin()->first; + auto optLeader = getLeader(directReq.get_space_id(), partId); + if (!optLeader.ok()) { + LOG(WARNING) << folly::sformat("failed to get leader, space {}, part {}", spaceId, partId) + << optLeader.status(); + p.setValue(::nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND); + return; + } + HostAddr& leader = optLeader.value(); + leader.port += kInternalPortOffset; + VLOG(2) << "leader host: " << leader; + + cpp2::ChainAddEdgesRequest chainReq = makeChainAddReq(directReq, termId, optVersion); + auto resp = getResponse( + evb, + leader, + chainReq, + [](cpp2::InternalStorageServiceAsyncClient* client, const cpp2::ChainAddEdgesRequest& r) { + return client->future_chainAddEdges(r); + }); + + std::move(resp).thenTry([=, p = std::move(p)](auto&& t) mutable { + auto code = getErrorCode(t); + if (code == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + chainAddEdges(directReq, termId, optVersion, std::move(p)); + } else { + p.setValue(code); + } + return; + }); +} + +cpp2::ChainAddEdgesRequest InternalStorageClient::makeChainAddReq(const cpp2::AddEdgesRequest& req, + TermID termId, + std::optional ver) { + cpp2::ChainAddEdgesRequest ret; + ret.space_id_ref() = req.get_space_id(); + ret.parts_ref() = req.get_parts(); + ret.prop_names_ref() = req.get_prop_names(); + ret.if_not_exists_ref() = req.get_if_not_exists(); + ret.term_ref() = termId; + if (ver) { + ret.edge_version_ref() = ver.value(); + } + return ret; +} + +void InternalStorageClient::chainDeleteEdges(cpp2::DeleteEdgesRequest& req, + const std::string& txnId, + TermID termId, + folly::Promise&& p, + folly::EventBase* evb) { + auto spaceId = req.get_space_id(); + auto partId = req.get_parts().begin()->first; + auto optLeader = getLeader(req.get_space_id(), partId); + if (!optLeader.ok()) { + LOG(WARNING) << folly::sformat("failed to get leader, space {}, part {}", spaceId, partId) + << optLeader.status(); + p.setValue(::nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND); + return; + } + HostAddr& leader = optLeader.value(); + leader.port += kInternalPortOffset; + VLOG(2) << "leader host: " << leader; + + cpp2::ChainDeleteEdgesRequest chainReq; + chainReq.space_id_ref() = req.get_space_id(); + chainReq.parts_ref() = req.get_parts(); + chainReq.txn_id_ref() = txnId; + chainReq.term_ref() = termId; + auto resp = getResponse( + evb, + leader, + chainReq, + [](cpp2::InternalStorageServiceAsyncClient* client, const cpp2::ChainDeleteEdgesRequest& r) { + return client->future_chainDeleteEdges(r); + }); + + std::move(resp).thenTry([=, p = std::move(p)](auto&& t) mutable { + auto code = getErrorCode(t); + if (code == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + chainDeleteEdges(req, txnId, termId, std::move(p)); + } else { + p.setValue(code); + } + return; + }); +} + +} // namespace storage +} // namespace nebula diff --git a/src/clients/storage/InternalStorageClient.h b/src/clients/storage/InternalStorageClient.h new file mode 100644 index 0000000..c704086 --- /dev/null +++ b/src/clients/storage/InternalStorageClient.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_INTERNALSTORAGEClient_H_ +#define CLIENTS_STORAGE_INTERNALSTORAGEClient_H_ + +#include "clients/storage/StorageClientBase.h" +#include "common/base/Base.h" +#include "common/base/ErrorOr.h" +#include "common/thrift/ThriftClientManager.h" +#include "interface/gen-cpp2/InternalStorageServiceAsyncClient.h" + +namespace nebula { +namespace storage { + +/** + * A wrapper class for InternalStorageServiceAsyncClient thrift API + * + * The class is NOT reentrant + */ +class InternalStorageClient + : public StorageClientBase< + cpp2::InternalStorageServiceAsyncClient, + thrift::ThriftClientManager> { + using Parent = + StorageClientBase>; + + public: + InternalStorageClient(std::shared_ptr ioThreadPool, + meta::MetaClient* metaClient) + : Parent(ioThreadPool, metaClient) {} + virtual ~InternalStorageClient() = default; + + virtual void chainUpdateEdge(cpp2::UpdateEdgeRequest& reversedRequest, + TermID termOfSrc, + std::optional optVersion, + folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::EventBase* evb = nullptr); + + virtual void chainAddEdges(cpp2::AddEdgesRequest& req, + TermID termId, + std::optional optVersion, + folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::EventBase* evb = nullptr); + + virtual void chainDeleteEdges(cpp2::DeleteEdgesRequest& req, + const std::string& txnId, + TermID termId, + folly::Promise<::nebula::cpp2::ErrorCode>&& p, + folly::EventBase* evb = nullptr); + + private: + cpp2::ChainAddEdgesRequest makeChainAddReq(const cpp2::AddEdgesRequest& req, + TermID termId, + std::optional optVersion); +}; + +} // namespace storage +} // namespace nebula + +#endif // CLIENTS_STORAGE_INTERNALSTORAGEClient_H_ diff --git a/src/clients/storage/StorageClient.cpp b/src/clients/storage/StorageClient.cpp new file mode 100644 index 0000000..a86032b --- /dev/null +++ b/src/clients/storage/StorageClient.cpp @@ -0,0 +1,927 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/storage/StorageClient.h" + +#include "common/base/Base.h" + +using nebula::cpp2::PropertyType; +using nebula::storage::cpp2::ExecResponse; +using nebula::storage::cpp2::GetDstBySrcResponse; +using nebula::storage::cpp2::GetNeighborsResponse; +using nebula::storage::cpp2::GetPropResponse; + +namespace nebula { +namespace storage { + +StorageClient::CommonRequestParam::CommonRequestParam(GraphSpaceID space_, + SessionID sess, + ExecutionPlanID plan_, + bool profile_, + bool experimental, + folly::EventBase* evb_) + : space(space_), + session(sess), + plan(plan_), + profile(profile_), + useExperimentalFeature(experimental), + evb(evb_) {} + +cpp2::RequestCommon StorageClient::CommonRequestParam::toReqCommon() const { + cpp2::RequestCommon common; + common.session_id_ref() = session; + common.plan_id_ref() = plan; + common.profile_detail_ref() = profile; + return common; +} + +StorageRpcRespFuture StorageClient::getNeighbors( + const CommonRequestParam& param, + std::vector colNames, + const std::vector& vids, + const std::vector& edgeTypes, + cpp2::EdgeDirection edgeDirection, + const std::vector* statProps, + const std::vector* vertexProps, + const std::vector* edgeProps, + const std::vector* expressions, + bool dedup, + bool random, + const std::vector& orderBy, + int64_t limit, + const Expression* filter, + const Expression* tagFilter) { + auto cbStatus = getIdFromValue(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + auto status = clusterIdsToHosts(param.space, vids, std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + auto common = param.toReqCommon(); + std::unordered_map requests; + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.column_names_ref() = colNames; + req.parts_ref() = std::move(c.second); + req.common_ref() = common; + cpp2::TraverseSpec spec; + spec.edge_types_ref() = edgeTypes; + spec.edge_direction_ref() = edgeDirection; + spec.dedup_ref() = dedup; + spec.random_ref() = random; + if (statProps != nullptr) { + spec.stat_props_ref() = *statProps; + } + if (vertexProps != nullptr) { + spec.vertex_props_ref() = *vertexProps; + } + if (edgeProps != nullptr) { + spec.edge_props_ref() = *edgeProps; + } + if (expressions != nullptr) { + spec.expressions_ref() = *expressions; + } + if (!orderBy.empty()) { + spec.order_by_ref() = orderBy; + } + spec.limit_ref() = limit; + if (filter != nullptr) { + spec.filter_ref() = filter->encode(); + } + if (tagFilter != nullptr) { + spec.tag_filter_ref() = tagFilter->encode(); + } + req.traverse_spec_ref() = std::move(spec); + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::GetNeighborsRequest& r) { + return client->future_getNeighbors(r); + }); +} + +StorageRpcRespFuture StorageClient::getDstBySrc( + const CommonRequestParam& param, + const std::vector& vertices, + const std::vector& edgeTypes) { + auto cbStatus = getIdFromValue(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, vertices, std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + auto common = param.toReqCommon(); + std::unordered_map requests; + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.edge_types_ref() = edgeTypes; + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::GetDstBySrcRequest& r) { + return client->future_getDstBySrc(r); + }); +} + +StorageRpcRespFuture StorageClient::addVertices( + const CommonRequestParam& param, + std::vector vertices, + std::unordered_map> propNames, + bool ifNotExists, + bool ignoreExistedIndex) { + auto cbStatus = getIdFromNewVertex(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, std::move(vertices), std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.if_not_exists_ref() = ifNotExists; + req.ignore_existed_index_ref() = ignoreExistedIndex; + req.parts_ref() = std::move(c.second); + req.prop_names_ref() = propNames; + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::AddVerticesRequest& r) { + return client->future_addVertices(r); + }); +} + +StorageRpcRespFuture StorageClient::addEdges(const CommonRequestParam& param, + std::vector edges, + std::vector propNames, + bool ifNotExists, + bool ignoreExistedIndex) { + auto cbStatus = getIdFromNewEdge(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, std::move(edges), std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.if_not_exists_ref() = ifNotExists; + req.ignore_existed_index_ref() = ignoreExistedIndex; + req.parts_ref() = std::move(c.second); + req.prop_names_ref() = propNames; + req.common_ref() = common; + } + return collectResponse(param.evb, + std::move(requests), + [useToss = param.useExperimentalFeature](ThriftClientType* client, + const cpp2::AddEdgesRequest& r) { + return useToss ? client->future_chainAddEdges(r) + : client->future_addEdges(r); + }); +} + +StorageRpcRespFuture StorageClient::getProps( + const CommonRequestParam& param, + const DataSet& input, + const std::vector* vertexProps, + const std::vector* edgeProps, + const std::vector* expressions, + bool dedup, + const std::vector& orderBy, + int64_t limit, + const Expression* filter) { + auto cbStatus = getIdFromRow(param.space, edgeProps != nullptr); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, input.rows, std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.dedup_ref() = dedup; + if (vertexProps != nullptr) { + req.vertex_props_ref() = *vertexProps; + } + if (edgeProps != nullptr) { + req.edge_props_ref() = *edgeProps; + } + if (expressions != nullptr) { + req.expressions_ref() = *expressions; + } + if (!orderBy.empty()) { + req.order_by_ref() = orderBy; + } + req.limit_ref() = limit; + if (filter != nullptr) { + req.filter_ref() = filter->encode(); + } + req.common_ref() = common; + } + + return collectResponse( + param.evb, std::move(requests), [](ThriftClientType* client, const cpp2::GetPropRequest& r) { + return client->future_getProps(r); + }); +} + +StorageRpcRespFuture StorageClient::deleteEdges( + const CommonRequestParam& param, std::vector edges) { + auto cbStatus = getIdFromEdgeKey(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, std::move(edges), std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [useToss = param.useExperimentalFeature]( + ThriftClientType* client, const cpp2::DeleteEdgesRequest& r) { + return useToss ? client->future_chainDeleteEdges(r) + : client->future_deleteEdges(r); + }); +} + +StorageRpcRespFuture StorageClient::deleteVertices( + const CommonRequestParam& param, std::vector ids) { + auto cbStatus = getIdFromValue(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, std::move(ids), std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::DeleteVerticesRequest& r) { + return client->future_deleteVertices(r); + }); +} + +StorageRpcRespFuture StorageClient::deleteTags( + const CommonRequestParam& param, std::vector delTags) { + auto cbStatus = getIdFromDelTags(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>( + std::runtime_error(cbStatus.status().toString())); + } + + auto status = clusterIdsToHosts(param.space, std::move(delTags), std::move(cbStatus).value()); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::DeleteTagsRequest& r) { + return client->future_deleteTags(r); + }); +} + +folly::Future> StorageClient::updateVertex( + const CommonRequestParam& param, + Value vertexId, + TagID tagId, + std::vector updatedProps, + bool insertable, + std::vector returnProps, + std::string condition) { + auto cbStatus = getIdFromValue(param.space); + if (!cbStatus.ok()) { + return folly::makeFuture>(cbStatus.status()); + } + + DCHECK(!!metaClient_); + auto status = metaClient_->partsNum(param.space); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", param.space); + } + auto numParts = status.value(); + status = metaClient_->partId(numParts, std::move(cbStatus).value()(vertexId)); + if (!status.ok()) { + return folly::makeFuture>(status.status()); + } + + auto part = status.value(); + auto host = this->getLeader(param.space, part); + if (!host.ok()) { + return folly::makeFuture>(host.status()); + } + cpp2::UpdateVertexRequest req; + req.space_id_ref() = param.space; + req.vertex_id_ref() = vertexId; + req.tag_id_ref() = tagId; + req.part_id_ref() = part; + req.updated_props_ref() = std::move(updatedProps); + req.return_props_ref() = std::move(returnProps); + req.insertable_ref() = insertable; + req.common_ref() = param.toReqCommon(); + if (condition.size() > 0) { + req.condition_ref() = std::move(condition); + } + + return getResponse(param.evb, + host.value(), + req, + [](ThriftClientType* client, const cpp2::UpdateVertexRequest& r) { + return client->future_updateVertex(r); + }); +} + +folly::Future> StorageClient::updateEdge( + const CommonRequestParam& param, + storage::cpp2::EdgeKey edgeKey, + std::vector updatedProps, + bool insertable, + std::vector returnProps, + std::string condition) { + auto space = param.space; + auto cbStatus = getIdFromEdgeKey(space); + if (!cbStatus.ok()) { + return folly::makeFuture>(cbStatus.status()); + } + + DCHECK(!!metaClient_); + auto status = metaClient_->partsNum(space); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", space); + } + auto numParts = status.value(); + status = metaClient_->partId(numParts, std::move(cbStatus).value()(edgeKey)); + if (!status.ok()) { + return folly::makeFuture>(status.status()); + } + + auto part = status.value(); + auto host = this->getLeader(space, part); + if (!host.ok()) { + return folly::makeFuture>(host.status()); + } + cpp2::UpdateEdgeRequest req; + req.space_id_ref() = space; + req.edge_key_ref() = edgeKey; + req.part_id_ref() = part; + req.updated_props_ref() = std::move(updatedProps); + req.return_props_ref() = std::move(returnProps); + req.insertable_ref() = insertable; + req.common_ref() = param.toReqCommon(); + if (condition.size() > 0) { + req.condition_ref() = std::move(condition); + } + + return getResponse(param.evb, + host.value(), + req, + [useExperimentalFeature = param.useExperimentalFeature]( + ThriftClientType* client, const cpp2::UpdateEdgeRequest& r) { + return useExperimentalFeature ? client->future_chainUpdateEdge(r) + : client->future_updateEdge(r); + }); +} + +folly::Future> StorageClient::getUUID(GraphSpaceID space, + const std::string& name, + folly::EventBase* evb) { + DCHECK(!!metaClient_); + auto status = metaClient_->partsNum(space); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", space); + } + auto numParts = status.value(); + status = metaClient_->partId(numParts, name); + if (!status.ok()) { + return folly::makeFuture>(status.status()); + } + + auto part = status.value(); + auto host = this->getLeader(space, part); + if (!host.ok()) { + return folly::makeFuture>(host.status()); + } + cpp2::GetUUIDReq req; + req.space_id_ref() = space; + req.part_id_ref() = part; + req.name_ref() = name; + + return getResponse( + evb, host.value(), req, [](ThriftClientType* client, const cpp2::GetUUIDReq& r) { + return client->future_getUUID(r); + }); +} + +StorageRpcRespFuture StorageClient::lookupIndex( + const CommonRequestParam& param, + const std::vector& contexts, + bool isEdge, + int32_t tagOrEdge, + const std::vector& returnCols, + std::vector orderBy, + int64_t limit) { + // TODO(sky) : instead of isEdge and tagOrEdge to nebula::cpp2::SchemaID for graph layer. + auto space = param.space; + auto status = getHostParts(space); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + nebula::cpp2::SchemaID schemaId; + if (isEdge) { + schemaId.edge_type_ref() = tagOrEdge; + } else { + schemaId.tag_id_ref() = tagOrEdge; + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = space; + req.parts_ref() = std::move(c.second); + req.return_columns_ref() = returnCols; + + cpp2::IndexSpec spec; + spec.contexts_ref() = contexts; + spec.schema_id_ref() = schemaId; + req.indices_ref() = spec; + req.common_ref() = common; + req.limit_ref() = limit; + req.order_by_ref() = orderBy; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::LookupIndexRequest& r) { + return client->future_lookupIndex(r); + }); +} + +StorageRpcRespFuture StorageClient::lookupAndTraverse( + const CommonRequestParam& param, cpp2::IndexSpec indexSpec, cpp2::TraverseSpec traverseSpec) { + auto space = param.space; + auto status = getHostParts(space); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + auto common = param.toReqCommon(); + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = space; + req.parts_ref() = std::move(c.second); + req.indices_ref() = indexSpec; + req.traverse_spec_ref() = traverseSpec; + req.common_ref() = common; + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::LookupAndTraverseRequest& r) { + return client->future_lookupAndTraverse(r); + }); +} + +StorageRpcRespFuture StorageClient::scanEdge( + const CommonRequestParam& param, + const std::vector& edgeProp, + int64_t limit, + const Expression* filter) { + std::unordered_map requests; + auto status = getHostPartsWithCursor(param.space); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + auto& clusters = status.value(); + for (const auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.return_columns_ref() = edgeProp; + req.limit_ref() = limit; + if (filter != nullptr) { + req.filter_ref() = filter->encode(); + } + req.common_ref() = param.toReqCommon(); + } + + return collectResponse( + param.evb, std::move(requests), [](ThriftClientType* client, const cpp2::ScanEdgeRequest& r) { + return client->future_scanEdge(r); + }); +} + +StorageRpcRespFuture StorageClient::scanVertex( + const CommonRequestParam& param, + const std::vector& vertexProp, + int64_t limit, + const Expression* filter) { + std::unordered_map requests; + auto status = getHostPartsWithCursor(param.space); + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + auto& clusters = status.value(); + for (const auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = param.space; + req.parts_ref() = std::move(c.second); + req.return_columns_ref() = vertexProp; + req.limit_ref() = limit; + if (filter != nullptr) { + req.filter_ref() = filter->encode(); + } + req.common_ref() = param.toReqCommon(); + } + + return collectResponse(param.evb, + std::move(requests), + [](ThriftClientType* client, const cpp2::ScanVertexRequest& r) { + return client->future_scanVertex(r); + }); +} + +folly::SemiFuture> StorageClient::get( + GraphSpaceID space, std::vector&& keys, bool returnPartly, folly::EventBase* evb) { + auto status = clusterIdsToHosts( + space, std::move(keys), [](const std::string& v) -> const std::string& { return v; }); + + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = space; + req.parts_ref() = std::move(c.second); + req.return_partly_ref() = returnPartly; + } + + return collectResponse( + evb, std::move(requests), [](ThriftClientType* client, const cpp2::KVGetRequest& r) { + return client->future_get(r); + }); +} + +folly::SemiFuture> StorageClient::put( + GraphSpaceID space, std::vector kvs, folly::EventBase* evb) { + auto status = clusterIdsToHosts( + space, std::move(kvs), [](const KeyValue& v) -> const std::string& { return v.key; }); + + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = space; + req.parts_ref() = std::move(c.second); + } + + return collectResponse( + evb, std::move(requests), [](ThriftClientType* client, const cpp2::KVPutRequest& r) { + return client->future_put(r); + }); +} + +folly::SemiFuture> StorageClient::remove( + GraphSpaceID space, std::vector keys, folly::EventBase* evb) { + auto status = clusterIdsToHosts( + space, std::move(keys), [](const std::string& v) -> const std::string& { return v; }); + + if (!status.ok()) { + return folly::makeFuture>( + std::runtime_error(status.status().toString())); + } + + auto& clusters = status.value(); + std::unordered_map requests; + for (auto& c : clusters) { + auto& host = c.first; + auto& req = requests[host]; + req.space_id_ref() = space; + req.parts_ref() = std::move(c.second); + } + + return collectResponse( + evb, std::move(requests), [](ThriftClientType* client, const cpp2::KVRemoveRequest& r) { + return client->future_remove(r); + }); +} + +StatusOr> StorageClient::getIdFromRow( + GraphSpaceID space, bool isEdgeProps) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + if (isEdgeProps) { + cb = [](const Row& r) -> const VertexID& { + // The first column has to be the src, the third column has to be the + // dst + DCHECK_EQ(Value::Type::INT, r.values[0].type()); + DCHECK_EQ(Value::Type::INT, r.values[3].type()); + auto& mutableR = const_cast(r); + mutableR.values[0] = + Value(std::string(reinterpret_cast(&r.values[0].getInt()), 8)); + mutableR.values[3] = + Value(std::string(reinterpret_cast(&r.values[3].getInt()), 8)); + return mutableR.values[0].getStr(); + }; + } else { + cb = [](const Row& r) -> const VertexID& { + // The first column has to be the vid + DCHECK_EQ(Value::Type::INT, r.values[0].type()); + auto& mutableR = const_cast(r); + mutableR.values[0] = + Value(std::string(reinterpret_cast(&r.values[0].getInt()), 8)); + return mutableR.values[0].getStr(); + }; + } + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const Row& r) -> const VertexID& { + // The first column has to be the vid + DCHECK_EQ(Value::Type::STRING, r.values[0].type()); + return r.values[0].getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + + return cb; +} + +StatusOr> StorageClient::getIdFromNewVertex( + GraphSpaceID space) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + cb = [](const cpp2::NewVertex& v) -> const VertexID& { + DCHECK_EQ(Value::Type::INT, v.get_id().type()); + auto& mutableV = const_cast(v); + mutableV.id_ref() = + Value(std::string(reinterpret_cast(&v.get_id().getInt()), 8)); + return mutableV.get_id().getStr(); + }; + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const cpp2::NewVertex& v) -> const VertexID& { + DCHECK_EQ(Value::Type::STRING, v.get_id().type()); + return v.get_id().getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + return cb; +} + +StatusOr> StorageClient::getIdFromNewEdge( + GraphSpaceID space) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + cb = [](const cpp2::NewEdge& e) -> const VertexID& { + DCHECK_EQ(Value::Type::INT, e.get_key().get_src().type()); + DCHECK_EQ(Value::Type::INT, e.get_key().get_dst().type()); + auto& mutableE = const_cast(e); + (*mutableE.key_ref()) + .src_ref() + .emplace(Value( + std::string(reinterpret_cast(&e.get_key().get_src().getInt()), 8))); + (*mutableE.key_ref()) + .dst_ref() + .emplace(Value( + std::string(reinterpret_cast(&e.get_key().get_dst().getInt()), 8))); + return mutableE.get_key().get_src().getStr(); + }; + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const cpp2::NewEdge& e) -> const VertexID& { + DCHECK_EQ(Value::Type::STRING, e.get_key().get_src().type()); + DCHECK_EQ(Value::Type::STRING, e.get_key().get_dst().type()); + return e.get_key().get_src().getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + return cb; +} + +StatusOr> StorageClient::getIdFromEdgeKey( + GraphSpaceID space) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + cb = [](const cpp2::EdgeKey& eKey) -> const VertexID& { + DCHECK_EQ(Value::Type::INT, eKey.get_src().type()); + DCHECK_EQ(Value::Type::INT, eKey.get_dst().type()); + auto& mutableEK = const_cast(eKey); + mutableEK.src_ref() = + Value(std::string(reinterpret_cast(&eKey.get_src().getInt()), 8)); + mutableEK.dst_ref() = + Value(std::string(reinterpret_cast(&eKey.get_dst().getInt()), 8)); + return mutableEK.get_src().getStr(); + }; + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const cpp2::EdgeKey& eKey) -> const VertexID& { + DCHECK_EQ(Value::Type::STRING, eKey.get_src().type()); + DCHECK_EQ(Value::Type::STRING, eKey.get_dst().type()); + return eKey.get_src().getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + return cb; +} + +StatusOr> StorageClient::getIdFromValue( + GraphSpaceID space) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + cb = [](const Value& v) -> const VertexID& { + DCHECK_EQ(Value::Type::INT, v.type()); + auto& mutableV = const_cast(v); + mutableV = Value(std::string(reinterpret_cast(&v.getInt()), 8)); + return mutableV.getStr(); + }; + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const Value& v) -> const VertexID& { + DCHECK_EQ(Value::Type::STRING, v.type()); + return v.getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + return cb; +} + +StatusOr> StorageClient::getIdFromDelTags( + GraphSpaceID space) const { + auto vidTypeStatus = metaClient_->getSpaceVidType(space); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + auto vidType = std::move(vidTypeStatus).value(); + + std::function cb; + if (vidType == PropertyType::INT64) { + cb = [](const cpp2::DelTags& delTags) -> const VertexID& { + const auto& vId = delTags.get_id(); + DCHECK_EQ(Value::Type::INT, vId.type()); + auto& mutableV = const_cast(vId); + mutableV = Value(std::string(reinterpret_cast(&vId.getInt()), 8)); + return mutableV.getStr(); + }; + } else if (vidType == PropertyType::FIXED_STRING) { + cb = [](const cpp2::DelTags& delTags) -> const VertexID& { + const auto& vId = delTags.get_id(); + DCHECK_EQ(Value::Type::STRING, vId.type()); + return vId.getStr(); + }; + } else { + return Status::Error("Only support integer/string type vid."); + } + return cb; +} + +} // namespace storage +} // namespace nebula diff --git a/src/clients/storage/StorageClient.h b/src/clients/storage/StorageClient.h new file mode 100644 index 0000000..f60dc77 --- /dev/null +++ b/src/clients/storage/StorageClient.h @@ -0,0 +1,200 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_STORAGECLIENT_H +#define CLIENTS_STORAGE_STORAGECLIENT_H + +#include "clients/storage/StorageClientBase.h" +#include "common/base/Base.h" +#include "common/thrift/ThriftClientManager.h" +#include "common/thrift/ThriftLocalClientManager.h" +#include "interface/gen-cpp2/GraphStorageServiceAsyncClient.h" +#include "storage/GraphStorageLocalServer.h" + +namespace nebula { +namespace storage { + +template +using StorageRpcRespFuture = folly::SemiFuture>; + +/** + * A wrapper class for GraphStorageServiceAsyncClient thrift API + * + * The class is NOT reentrant + */ +#ifndef BUILD_STANDALONE +using ThriftClientType = cpp2::GraphStorageServiceAsyncClient; +template +using ThriftClientManType = thrift::ThriftClientManager; +#else +using ThriftClientType = GraphStorageLocalServer; +template +using ThriftClientManType = thrift::LocalClientManager; + +#endif +class StorageClient + : public StorageClientBase> { + FRIEND_TEST(StorageClientTest, LeaderChangeTest); + + public: + struct CommonRequestParam { + GraphSpaceID space; + SessionID session; + ExecutionPlanID plan; + bool profile{false}; + bool useExperimentalFeature{false}; + folly::EventBase* evb{nullptr}; + + CommonRequestParam(GraphSpaceID space_, + SessionID sess, + ExecutionPlanID plan_, + bool profile_ = false, + bool experimental = false, + folly::EventBase* evb_ = nullptr); + + cpp2::RequestCommon toReqCommon() const; + }; + + StorageClient(std::shared_ptr ioThreadPool, + meta::MetaClient* metaClient) + : StorageClientBase>(ioThreadPool, + metaClient) {} + virtual ~StorageClient() {} + + StorageRpcRespFuture getNeighbors( + const CommonRequestParam& param, + std::vector colNames, + // The first column has to be the VertexID + const std::vector& vids, + const std::vector& edgeTypes, + cpp2::EdgeDirection edgeDirection, + const std::vector* statProps, + const std::vector* vertexProps, + const std::vector* edgeProps, + const std::vector* expressions, + bool dedup = false, + bool random = false, + const std::vector& orderBy = std::vector(), + int64_t limit = std::numeric_limits::max(), + const Expression* filter = nullptr, + const Expression* tagFilter = nullptr); + + StorageRpcRespFuture getDstBySrc( + const CommonRequestParam& param, + const std::vector& vertices, + const std::vector& edgeTypes); + + StorageRpcRespFuture getProps( + const CommonRequestParam& param, + const DataSet& input, + const std::vector* vertexProps, + const std::vector* edgeProps, + const std::vector* expressions, + bool dedup = false, + const std::vector& orderBy = std::vector(), + int64_t limit = std::numeric_limits::max(), + const Expression* filter = nullptr); + + StorageRpcRespFuture addVertices( + const CommonRequestParam& param, + std::vector vertices, + std::unordered_map> propNames, + bool ifNotExists, + bool ignoreExistedIndex); + + StorageRpcRespFuture addEdges(const CommonRequestParam& param, + std::vector edges, + std::vector propNames, + bool ifNotExists, + bool ignoreExistedIndex); + + StorageRpcRespFuture deleteEdges(const CommonRequestParam& param, + std::vector edges); + + StorageRpcRespFuture deleteVertices(const CommonRequestParam& param, + std::vector ids); + + StorageRpcRespFuture deleteTags(const CommonRequestParam& param, + std::vector delTags); + + folly::Future> updateVertex( + const CommonRequestParam& param, + Value vertexId, + TagID tagId, + std::vector updatedProps, + bool insertable, + std::vector returnProps, + std::string condition); + + folly::Future> updateEdge( + const CommonRequestParam& param, + storage::cpp2::EdgeKey edgeKey, + std::vector updatedProps, + bool insertable, + std::vector returnProps, + std::string condition); + + folly::Future> getUUID(GraphSpaceID space, + const std::string& name, + folly::EventBase* evb = nullptr); + + StorageRpcRespFuture lookupIndex( + const CommonRequestParam& param, + const std::vector& contexts, + bool isEdge, + int32_t tagOrEdge, + const std::vector& returnCols, + std::vector orderBy, + int64_t limit); + + StorageRpcRespFuture lookupAndTraverse( + const CommonRequestParam& param, cpp2::IndexSpec indexSpec, cpp2::TraverseSpec traverseSpec); + + StorageRpcRespFuture scanEdge(const CommonRequestParam& param, + const std::vector& vertexProp, + int64_t limit, + const Expression* filter); + + StorageRpcRespFuture scanVertex( + const CommonRequestParam& param, + const std::vector& vertexProp, + int64_t limit, + const Expression* filter); + + folly::SemiFuture> get(GraphSpaceID space, + std::vector&& keys, + bool returnPartly = false, + folly::EventBase* evb = nullptr); + + folly::SemiFuture> put(GraphSpaceID space, + std::vector kvs, + folly::EventBase* evb = nullptr); + + folly::SemiFuture> remove(GraphSpaceID space, + std::vector keys, + folly::EventBase* evb = nullptr); + + private: + StatusOr> getIdFromRow(GraphSpaceID space, + bool isEdgeProps) const; + + StatusOr> getIdFromNewVertex( + GraphSpaceID space) const; + + StatusOr> getIdFromNewEdge( + GraphSpaceID space) const; + + StatusOr> getIdFromEdgeKey( + GraphSpaceID space) const; + + StatusOr> getIdFromValue(GraphSpaceID space) const; + + StatusOr> getIdFromDelTags( + GraphSpaceID space) const; +}; + +} // namespace storage +} // namespace nebula +#endif diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h new file mode 100644 index 0000000..0cb5b32 --- /dev/null +++ b/src/clients/storage/StorageClientBase-inl.h @@ -0,0 +1,323 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_STORAGECLIENTBASE_INL_H +#define CLIENTS_STORAGE_STORAGECLIENTBASE_INL_H + +#include +#include +#include + +#include +#include + +#include "clients/storage/stats/StorageClientStats.h" +#include "common/base/Logging.h" +#include "common/base/StatusOr.h" +#include "common/datatypes/HostAddr.h" +#include "common/memory/MemoryTracker.h" +#include "common/ssl/SSLConfig.h" +#include "common/stats/StatsManager.h" +#include "common/thrift/ThriftTypes.h" +#include "common/time/WallClock.h" +#include "interface/gen-cpp2/common_types.h" + +namespace nebula { +namespace storage { + +template +StorageClientBase::StorageClientBase( + std::shared_ptr threadPool, meta::MetaClient* metaClient) + : metaClient_(metaClient), ioThreadPool_(threadPool) { + clientsMan_ = std::make_unique(FLAGS_enable_ssl); +} + +template +StorageClientBase::~StorageClientBase() { + VLOG(3) << "Destructing StorageClientBase"; + if (nullptr != metaClient_) { + metaClient_ = nullptr; + } +} + +template +StatusOr StorageClientBase::getLeader( + GraphSpaceID spaceId, PartitionID partId) const { + return metaClient_->getStorageLeaderFromCache(spaceId, partId); +} + +template +void StorageClientBase::updateLeader(GraphSpaceID spaceId, + PartitionID partId, + const HostAddr& leader) { + metaClient_->updateStorageLeader(spaceId, partId, leader); +} + +template +void StorageClientBase::invalidLeader(GraphSpaceID spaceId, + PartitionID partId) { + metaClient_->invalidStorageLeader(spaceId, partId); +} + +template +void StorageClientBase::invalidLeader( + GraphSpaceID spaceId, std::vector& partsId) { + for (const auto& partId : partsId) { + invalidLeader(spaceId, partId); + } +} + +template +template +folly::SemiFuture> +StorageClientBase::collectResponse( + folly::EventBase* evb, + std::unordered_map requests, + RemoteFunc&& remoteFunc) { + memory::MemoryCheckOffGuard offGuard; + std::vector>> respFutures; + respFutures.reserve(requests.size()); + + auto hosts = std::make_shared>(requests.size()); + auto totalLatencies = std::make_shared>(requests.size()); + + for (const auto& req : requests) { + auto start = time::WallClock::fastNowInMicroSec(); + + size_t i = respFutures.size(); + (*hosts)[i] = req.first; + // Future process code will be executed on the IO thread + // Since all requests are sent using the same eventbase, all + // then-callback will be executed on the same IO thread + auto fut = getResponse(evb, req.first, req.second, std::move(remoteFunc)) + .ensure([totalLatencies, i, start]() { + (*totalLatencies)[i] = time::WallClock::fastNowInMicroSec() - start; + }); + + respFutures.emplace_back(std::move(fut)); + } + + return folly::collectAll(respFutures) + .deferValue([this, requests = std::move(requests), totalLatencies, hosts]( + std::vector>>&& resps) { + // throw in MemoryCheckGuard verified + memory::MemoryCheckGuard guard; + StorageRpcResponse rpcResp(resps.size()); + for (size_t i = 0; i < resps.size(); i++) { + const auto& host = hosts->at(i); + folly::Try>& tryResp = resps[i]; + if (tryResp.hasException()) { + std::string errMsg = tryResp.exception().what().toStdString(); + rpcResp.markFailure(); + LOG(ERROR) << "There some RPC errors: " << errMsg; + const auto& req = requests.at(host); + const auto& parts = getReqPartsId(req); + rpcResp.appendFailedParts(parts, nebula::cpp2::ErrorCode::E_RPC_FAILURE); + } else { + StatusOr status = std::move(tryResp).value(); + if (status.ok()) { + auto resp = std::move(status).value(); + const auto& result = resp.get_result(); + + if (!result.get_failed_parts().empty()) { + rpcResp.markFailure(); + for (auto& part : result.get_failed_parts()) { + rpcResp.emplaceFailedPart(part.get_part_id(), part.get_code()); + } + } + + // Adjust the latency + auto latency = result.get_latency_in_us(); + rpcResp.setLatency(host, latency, totalLatencies->at(i)); + // Keep the response + rpcResp.addResponse(std::move(resp)); + } else { + rpcResp.markFailure(); + Status s = std::move(status).status(); + nebula::cpp2::ErrorCode errorCode = + s.code() == Status::Code::kGraphMemoryExceeded + ? nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED + : nebula::cpp2::ErrorCode::E_RPC_FAILURE; + LOG(ERROR) << "There some RPC errors: " << s.message(); + const auto& req = requests.at(host); + const auto& parts = getReqPartsId(req); + rpcResp.appendFailedParts(parts, errorCode); + } + } + } + + return rpcResp; + }); +} + +template +template +folly::Future> StorageClientBase::getResponse( + folly::EventBase* evb, const HostAddr& host, const Request& request, RemoteFunc&& remoteFunc) { + static_assert( + folly::isFuture>::value); + + stats::StatsManager::addValue(kNumRpcSentToStoraged); + if (evb == nullptr) { + evb = DCHECK_NOTNULL(ioThreadPool_)->getEventBase(); + } + + auto spaceId = request.get_space_id(); + return folly::via(evb) + .thenValue([remoteFunc = std::move(remoteFunc), request, evb, host, this](auto&&) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + // NOTE: Create new channel on each thread to avoid TIMEOUT RPC error + auto client = clientsMan_->client(host, evb, false, FLAGS_storage_client_timeout_ms); + // Encoding invoke Cpp2Ops::write the request to protocol is in current thread, + // do not need to turn on in Cpp2Ops::write + return remoteFunc(client.get(), request); + }) + .thenValue([spaceId, this](Response&& resp) mutable -> StatusOr { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + auto& result = resp.get_result(); + for (auto& part : result.get_failed_parts()) { + auto partId = part.get_part_id(); + auto code = part.get_code(); + + VLOG(3) << "Failure! Failed part " << partId << ", failed part " + << static_cast(code); + + switch (code) { + case nebula::cpp2::ErrorCode::E_LEADER_CHANGED: { + auto* leader = part.get_leader(); + if (isValidHostPtr(leader)) { + updateLeader(spaceId, partId, *leader); + } else { + invalidLeader(spaceId, partId); + } + break; + } + case nebula::cpp2::ErrorCode::E_PART_NOT_FOUND: + case nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND: { + invalidLeader(spaceId, partId); + break; + } + default: + break; + } + } + return std::move(resp); + }) + .thenError( + folly::tag_t{}, + [](const std::bad_alloc&) { + return folly::makeFuture>(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + }) + .thenError([request, host, spaceId, this]( + folly::exception_wrapper&& exWrapper) mutable -> StatusOr { + stats::StatsManager::addValue(kNumRpcSentToStoragedFailed); + + using TransportException = apache::thrift::transport::TTransportException; + auto ex = exWrapper.get_exception(); + if (ex) { + if (ex->getType() == TransportException::TIMED_OUT) { + LOG(ERROR) << "Request to " << host << " time out: " << ex->what(); + return Status::Error("RPC failure in StorageClient with timeout: %s", ex->what()); + } else { + LOG(ERROR) << "Request to " << host << " failed: " << ex->what(); + return Status::Error("RPC failure in StorageClient: %s", ex->what()); + } + } else { + auto partsId = getReqPartsId(request); + invalidLeader(spaceId, partsId); + LOG(ERROR) << "Request to " << host << " failed."; + return Status::Error("RPC failure in StorageClient."); + } + }); +} + +template +template +StatusOr>>> +StorageClientBase::clusterIdsToHosts(GraphSpaceID spaceId, + const Container& ids, + GetIdFunc f) const { + std::unordered_map>> + clusters; + + auto status = metaClient_->partsNum(spaceId); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", spaceId); + } + auto numParts = status.value(); + std::unordered_map leaders; + for (int32_t partId = 1; partId <= numParts; ++partId) { + auto leader = getLeader(spaceId, partId); + if (!leader.ok()) { + return leader.status(); + } + leaders[partId] = std::move(leader).value(); + } + for (auto& id : ids) { + CHECK(!!metaClient_); + status = metaClient_->partId(numParts, f(id)); + if (!status.ok()) { + return status.status(); + } + + auto part = status.value(); + const auto& leader = leaders[part]; + clusters[leader][part].emplace_back(std::move(id)); + } + return clusters; +} + +template +StatusOr>> +StorageClientBase::getHostParts(GraphSpaceID spaceId) const { + std::unordered_map> hostParts; + auto status = metaClient_->partsNum(spaceId); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", spaceId); + } + + auto parts = status.value(); + for (auto partId = 1; partId <= parts; partId++) { + auto leader = getLeader(spaceId, partId); + if (!leader.ok()) { + return leader.status(); + } + hostParts[leader.value()].emplace_back(partId); + } + return hostParts; +} + +template +StatusOr>> +StorageClientBase::getHostPartsWithCursor( + GraphSpaceID spaceId) const { + std::unordered_map> hostParts; + auto status = metaClient_->partsNum(spaceId); + if (!status.ok()) { + return Status::Error("Space not found, spaceid: %d", spaceId); + } + + // TODO support cursor + cpp2::ScanCursor c; + auto parts = status.value(); + for (auto partId = 1; partId <= parts; partId++) { + auto leader = getLeader(spaceId, partId); + if (!leader.ok()) { + return leader.status(); + } + hostParts[leader.value()].emplace(partId, c); + } + return hostParts; +} + +} // namespace storage +} // namespace nebula +#endif diff --git a/src/clients/storage/StorageClientBase.cpp b/src/clients/storage/StorageClientBase.cpp new file mode 100644 index 0000000..b915e15 --- /dev/null +++ b/src/clients/storage/StorageClientBase.cpp @@ -0,0 +1,15 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/storage/StorageClientBase.h" + +DEFINE_int32(storage_client_timeout_ms, 60 * 1000, "storage client timeout"); +DEFINE_uint32(storage_client_retry_interval_ms, + 1000, + "storage client sleep interval milliseconds between retry"); + +namespace nebula { +namespace storage {} // namespace storage +} // namespace nebula diff --git a/src/clients/storage/StorageClientBase.h b/src/clients/storage/StorageClientBase.h new file mode 100644 index 0000000..4be5241 --- /dev/null +++ b/src/clients/storage/StorageClientBase.h @@ -0,0 +1,214 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_STORAGECLIENTBASE_H_ +#define CLIENTS_STORAGE_STORAGECLIENTBASE_H_ + +#include +#include + +#include "clients/meta/MetaClient.h" +#include "common/base/Base.h" +#include "common/base/StatusOr.h" +#include "common/datatypes/HostAddr.h" +#include "common/meta/Common.h" +#include "common/thrift/ThriftClientManager.h" +#include "interface/gen-cpp2/storage_types.h" + +DECLARE_int32(storage_client_timeout_ms); +DECLARE_uint32(storage_client_retry_interval_ms); + +namespace nebula { +namespace storage { + +template +class StorageRpcResponse final { + public: + enum class Result { + ALL_SUCCEEDED = 0, + PARTIAL_SUCCEEDED = 1, + }; + + explicit StorageRpcResponse(size_t reqsSent) : totalReqsSent_(reqsSent) { + responses_.reserve(reqsSent); + } + + bool succeeded() const { + return result_ == Result::ALL_SUCCEEDED; + } + + int32_t maxLatency() const { + return maxLatency_; + } + + void setLatency(const HostAddr& host, int32_t latency, int32_t e2eLatency) { + if (latency > maxLatency_) { + maxLatency_ = latency; + } + hostLatency_.emplace_back(std::make_tuple(host, latency, e2eLatency)); + } + + void markFailure() { + result_ = Result::PARTIAL_SUCCEEDED; + ++failedReqs_; + } + + // A value between [0, 100], representing a percentage + int32_t completeness() const { + DCHECK_NE(totalReqsSent_, 0); + return totalReqsSent_ == 0 ? 0 : (totalReqsSent_ - failedReqs_) * 100 / totalReqsSent_; + } + + void emplaceFailedPart(PartitionID partId, nebula::cpp2::ErrorCode errorCode) { + failedParts_.emplace(partId, errorCode); + } + + void appendFailedParts(const std::vector& partsId, + nebula::cpp2::ErrorCode errorCode) { + failedParts_.reserve(failedParts_.size() + partsId.size()); + for (const auto& partId : partsId) { + failedParts_.emplace(partId, errorCode); + } + } + + void addResponse(Response&& resp) { + responses_.emplace_back(std::move(resp)); + } + + const std::unordered_map& failedParts() const { + return failedParts_; + } + + std::vector& responses() { + return responses_; + } + + const std::vector& responses() const { + return responses_; + } + + const std::vector>& hostLatency() const { + return hostLatency_; + } + + private: + const size_t totalReqsSent_; + size_t failedReqs_{0}; + + Result result_{Result::ALL_SUCCEEDED}; + std::unordered_map failedParts_; + int32_t maxLatency_{0}; + std::vector responses_; + std::vector> hostLatency_; +}; + +/** + * A base class for all storage clients + */ +template +class StorageClientBase { + public: + StatusOr getLeader(GraphSpaceID spaceId, PartitionID partId) const; + + protected: + StorageClientBase(std::shared_ptr ioThreadPool, + meta::MetaClient* metaClient); + virtual ~StorageClientBase(); + + void updateLeader(GraphSpaceID spaceId, PartitionID partId, const HostAddr& leader); + void invalidLeader(GraphSpaceID spaceId, PartitionID partId); + void invalidLeader(GraphSpaceID spaceId, std::vector& partsId); + + template ::type::value_type> + folly::SemiFuture> collectResponse( + folly::EventBase* evb, + std::unordered_map requests, + RemoteFunc&& remoteFunc); + + template ::type::value_type> + folly::Future> getResponse(folly::EventBase* evb, + const HostAddr& host, + const Request& request, + RemoteFunc&& remoteFunc); + + // Cluster given ids into the host they belong to + // The method returns a map + // host_addr (A host, but in most case, the leader will be chosen) + // => (partition -> [ids that belong to the shard]) + template + StatusOr>>> + clusterIdsToHosts(GraphSpaceID spaceId, const Container& ids, GetIdFunc f) const; + + StatusOr>> + getHostPartsWithCursor(GraphSpaceID spaceId) const; + + virtual StatusOr getPartHosts(GraphSpaceID spaceId, PartitionID partId) const { + CHECK(metaClient_ != nullptr); + return metaClient_->getPartHostsFromCache(spaceId, partId); + } + + virtual StatusOr>> getHostParts( + GraphSpaceID spaceId) const; + + // from map + template + std::vector getReqPartsIdFromContainer( + const std::unordered_map& t) const { + std::vector partsId; + partsId.reserve(t.size()); + for (const auto& part : t) { + partsId.emplace_back(part.first); + } + return partsId; + } + + // from list + std::vector getReqPartsIdFromContainer(const std::vector& t) const { + return t; + } + + template + std::vector getReqPartsId(const Request& req) const { + return getReqPartsIdFromContainer(req.get_parts()); + } + + std::vector getReqPartsId(const cpp2::UpdateVertexRequest& req) const { + return {req.get_part_id()}; + } + + std::vector getReqPartsId(const cpp2::UpdateEdgeRequest& req) const { + return {req.get_part_id()}; + } + + std::vector getReqPartsId(const cpp2::GetUUIDReq& req) const { + return {req.get_part_id()}; + } + + bool isValidHostPtr(const HostAddr* addr) { + return addr != nullptr && !addr->host.empty() && addr->port != 0; + } + + protected: + meta::MetaClient* metaClient_{nullptr}; + + private: + std::shared_ptr ioThreadPool_; + std::unique_ptr clientsMan_; +}; + +} // namespace storage +} // namespace nebula + +#include "clients/storage/StorageClientBase-inl.h" + +#endif // CLIENTS_STORAGE_STORAGECLIENTBASE_H_ diff --git a/src/clients/storage/stats/CMakeLists.txt b/src/clients/storage/stats/CMakeLists.txt new file mode 100644 index 0000000..a1acb23 --- /dev/null +++ b/src/clients/storage/stats/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + storage_client_stats_obj + OBJECT + StorageClientStats.cpp + ) diff --git a/src/clients/storage/stats/StorageClientStats.cpp b/src/clients/storage/stats/StorageClientStats.cpp new file mode 100644 index 0000000..097f4cf --- /dev/null +++ b/src/clients/storage/stats/StorageClientStats.cpp @@ -0,0 +1,20 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "clients/meta/stats/MetaClientStats.h" + +namespace nebula { + +stats::CounterId kNumRpcSentToStoraged; +stats::CounterId kNumRpcSentToStoragedFailed; + +void initStorageClientStats() { + kNumRpcSentToStoraged = + stats::StatsManager::registerStats("num_rpc_sent_to_storaged", "rate, sum"); + kNumRpcSentToStoragedFailed = + stats::StatsManager::registerStats("num_rpc_sent_to_storaged_failed", "rate, sum"); +} + +} // namespace nebula diff --git a/src/clients/storage/stats/StorageClientStats.h b/src/clients/storage/stats/StorageClientStats.h new file mode 100644 index 0000000..c3a5a6f --- /dev/null +++ b/src/clients/storage/stats/StorageClientStats.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CLIENTS_STORAGE_STATS_STORAGECLIENTSTATS_H +#define CLIENTS_STORAGE_STATS_STORAGECLIENTSTATS_H + +#include "common/stats/StatsManager.h" + +namespace nebula { + +extern stats::CounterId kNumRpcSentToStoraged; +extern stats::CounterId kNumRpcSentToStoragedFailed; + +void initStorageClientStats(); + +} // namespace nebula +#endif diff --git a/src/codec/CMakeLists.txt b/src/codec/CMakeLists.txt new file mode 100644 index 0000000..73a2182 --- /dev/null +++ b/src/codec/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + codec_obj OBJECT + RowReaderV2.cpp + RowWriterV2.cpp + RowReaderWrapper.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/codec/Common.h b/src/codec/Common.h new file mode 100644 index 0000000..0e56686 --- /dev/null +++ b/src/codec/Common.h @@ -0,0 +1,111 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_COMMON_H_ +#define CODEC_COMMON_H_ + +#include "common/base/Base.h" + +namespace nebula { + +template +typename std::enable_if::type>::type>::value, + bool>::type +intToBool(IntType iVal) { + return iVal != 0; +} + +inline bool strToBool(folly::StringPiece str) { + return str == "Y" || str == "y" || str == "T" || str == "t" || str == "yes" || str == "Yes" || + str == "YES" || str == "true" || str == "True" || str == "TRUE"; +} + +inline std::string toHexStr(folly::StringPiece str) { + // clang-format off + static const char* hex[] = { + "00", "01", "02", "03", "04", "05", "06", "07", + "08", "09", "0A", "0B", "0C", "0D", "0E", "0F", + "10", "11", "12", "13", "14", "15", "16", "17", + "18", "19", "1A", "1B", "1C", "1D", "1E", "1F", + "20", "21", "22", "23", "24", "25", "26", "27", + "28", "29", "2A", "2B", "2C", "2D", "2E", "2F", + "30", "31", "32", "33", "34", "35", "36", "37", + "38", "39", "3A", "3B", "3C", "3D", "3E", "3F", + "40", "41", "42", "43", "44", "45", "46", "47", + "48", "49", "4A", "4B", "4C", "4D", "4E", "4F", + "50", "51", "52", "53", "54", "55", "56", "57", + "58", "59", "5A", "5B", "5C", "5D", "5E", "5F", + "60", "61", "62", "63", "64", "65", "66", "67", + "68", "69", "6A", "6B", "6C", "6D", "6E", "6F", + "70", "71", "72", "73", "74", "75", "76", "77", + "78", "79", "7A", "7B", "7C", "7D", "7E", "7F", + "80", "81", "82", "83", "84", "85", "86", "87", + "88", "89", "8A", "8B", "8C", "8D", "8E", "8F", + "90", "91", "92", "93", "94", "95", "96", "97", + "98", "99", "9A", "9B", "9C", "9D", "9E", "9F", + "A0", "A1", "A2", "A3", "A4", "A5", "A6", "A7", + "A8", "A9", "AA", "AB", "AC", "AD", "AE", "AF", + "B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", + "B8", "B9", "BA", "BB", "BC", "BD", "BE", "BF", + "C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", + "C8", "C9", "CA", "CB", "CC", "CD", "CE", "CF", + "D0", "D1", "D2", "D3", "D4", "D5", "D6", "D7", + "D8", "D9", "DA", "DB", "DC", "DD", "DE", "DF", + "E0", "E1", "E2", "E3", "E4", "E5", "E6", "E7", + "E8", "E9", "EA", "EB", "EC", "ED", "EE", "EF", + "F0", "F1", "F2", "F3", "F4", "F5", "F6", "F7", + "F8", "F9", "FA", "FB", "FC", "FD", "FE", "FF", + }; + // clang-format on + + if (str.empty()) { + return std::string(); + } + + std::string buf; + buf.reserve(str.size() * 3 - 1); + + buf.append(hex[static_cast(str[0])]); + for (size_t i = 1; i < str.size(); i++) { + buf.append(" "); + buf.append(hex[static_cast(str[i])]); + } + + return buf; +} + +// If v is longer than maxLen, return a safepoint to be cut which contains legal utf-8 characters, +// and make sure the returned size is less than or equal to maxLen +inline size_t utf8CutSize(folly::StringPiece v, size_t maxLen) { + DCHECK_GT(v.size(), maxLen); + size_t len = 0; + size_t curLen = 0; // current length of utf-8 character + while (len < maxLen) { + auto tmp = static_cast(v[len]); + if (tmp >= 0xFC) { + curLen = 6; + } else if (tmp >= 0xF8) { + curLen = 5; + } else if (tmp >= 0xF0) { + curLen = 4; + } else if (tmp >= 0xE0) { + curLen = 3; + } else if (tmp >= 0xC0) { + curLen = 2; + } else { + curLen = 1; + } + if (len + curLen <= maxLen) { + len += curLen; + } else { + break; + } + } + return len; +} + +} // namespace nebula +#endif // CODEC_COMMON_H_ diff --git a/src/codec/NebulaCodecImpl.cpp b/src/codec/NebulaCodecImpl.cpp new file mode 100644 index 0000000..293140f --- /dev/null +++ b/src/codec/NebulaCodecImpl.cpp @@ -0,0 +1,123 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "codec/NebulaCodecImpl.h" + +#include "codec/RowReaderWrapper.h" +#include "codec/RowWriterV2.h" +#include "common/base/Base.h" + +/** + * Report error message + */ +#define PRINT_ERROR_MESSAGE(code, value) \ + if (ResultType::SUCCEEDED == code) { \ + result[field] = value; \ + } else { \ + LOG(ERROR) << "ResultType : " << apache::thrift::util::enumNameSafe(code) << " Value " \ + << value << std::endl; \ + } + +namespace nebula { + +std::string NebulaCodecImpl::encode(std::vector values, + std::shared_ptr schema) { + RowWriter writer(schema); + for (auto& value : values) { + if (value.type() == typeid(int32_t)) { + writer << std::any_cast(value); + } else if (value.type() == typeid(int64_t)) { + writer << std::any_cast(value); + } else if (value.type() == typeid(std::string)) { + writer << std::any_cast(value); + } else if (value.type() == typeid(double)) { + writer << std::any_cast(value); + } else if (value.type() == typeid(float)) { + writer << std::any_cast(value); + } else if (value.type() == typeid(bool)) { + writer << std::any_cast(value); + } else { + LOG(ERROR) << "Value Type :" << value.type().name() << std::endl; + } + } + std::string result = writer.encode(); + return result; +} + +StatusOr> NebulaCodecImpl::decode( + std::string encoded, std::shared_ptr schema) { + if (encoded.empty()) { + return Status::Error("encoded string is empty"); + } + if (!schema) { + return Status::Error("schema is not set"); + } + + folly::StringPiece piece; + ResultType code; + auto reader = RowReaderWrapper::getRowReader(encoded, schema); + std::unordered_map result; + for (size_t index = 0; index < schema->getNumFields(); index++) { + auto field = schema->getFieldName(index); + if (UNLIKELY(nullptr == field)) { + return Status::Error("invalid field index"); + } + switch (schema->getFieldType(index).get_type()) { + case cpp2::SupportedType::BOOL: + bool b; + code = reader->getBool(field, b); + PRINT_ERROR_MESSAGE(code, b); + break; + case cpp2::SupportedType::INT: + int32_t i; + code = reader->getInt(field, i); + PRINT_ERROR_MESSAGE(code, i); + break; + case cpp2::SupportedType::STRING: + code = reader->getString(field, piece); + PRINT_ERROR_MESSAGE(code, piece.toString()); + break; + case cpp2::SupportedType::VID: + int64_t v; + code = reader->getVid(field, v); + PRINT_ERROR_MESSAGE(code, v); + break; + case cpp2::SupportedType::FLOAT: + float f; + code = reader->getFloat(field, f); + PRINT_ERROR_MESSAGE(code, f); + break; + case cpp2::SupportedType::DOUBLE: + double d; + code = reader->getDouble(field, d); + PRINT_ERROR_MESSAGE(code, d) + break; + case cpp2::SupportedType::TIMESTAMP: + // TODO(darion) Support TIMESTAMP + break; + case cpp2::SupportedType::YEAR: + // TODO(darion) Support YEAR + break; + case cpp2::SupportedType::YEARMONTH: + // TODO(darion) Support YEARMONTH + break; + case cpp2::SupportedType::DATE: + // TODO(darion) Support DATE + break; + case cpp2::SupportedType::DATETIME: + // TODO(darion) Support DATETIME + break; + case cpp2::SupportedType::PATH: + // TODO(darion) Support PATH + break; + default: + // UNKNOWN + break; + } + } + return result; +} + +} // namespace nebula diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp new file mode 100644 index 0000000..9c9ef34 --- /dev/null +++ b/src/codec/RowReaderV2.cpp @@ -0,0 +1,220 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "codec/RowReaderV2.h" + +namespace nebula { + +using nebula::cpp2::PropertyType; + +bool RowReaderV2::resetImpl(meta::NebulaSchemaProvider const* schema, folly::StringPiece row) { + schema_ = schema; + data_ = row; + + DCHECK(!!schema_); + + size_t numVerBytes = data_[0] & 0x07; + headerLen_ = numVerBytes + 1; + +#ifndef NDEBUG + // Get the schema version + SchemaVer schemaVer = 0; + if (numVerBytes > 0) { + memcpy(reinterpret_cast(&schemaVer), &data_[1], numVerBytes); + } + DCHECK_EQ(schemaVer, schema_->getVersion()); +#endif + + // Null flags + size_t numNullables = schema_->getNumNullableFields(); + if (numNullables > 0) { + numNullBytes_ = ((numNullables - 1) >> 3) + 1; + } else { + numNullBytes_ = 0; + } + + return true; +} + +bool RowReaderV2::isNull(size_t pos) const { + static const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; + + size_t offset = headerLen_ + (pos >> 3); + int8_t flag = data_[offset] & bits[pos & 0x0000000000000007L]; + return flag != 0; +} + +Value RowReaderV2::getValueByName(const std::string& prop) const { + int64_t index = schema_->getFieldIndex(prop); + return getValueByIndex(index); +} + +Value RowReaderV2::getValueByIndex(const int64_t index) const { + if (index < 0 || static_cast(index) >= schema_->getNumFields()) { + return Value(NullType::UNKNOWN_PROP); + } + + auto field = schema_->field(index); + size_t offset = headerLen_ + numNullBytes_ + field->offset(); + + if (field->nullable() && isNull(field->nullFlagPos())) { + return NullType::__NULL__; + } + + switch (field->type()) { + case PropertyType::BOOL: { + if (data_[offset]) { + return true; + } else { + return false; + } + } + case PropertyType::INT8: { + return static_cast(data_[offset]); + } + case PropertyType::INT16: { + int16_t val; + memcpy(reinterpret_cast(&val), &data_[offset], sizeof(int16_t)); + return val; + } + case PropertyType::INT32: { + int32_t val; + memcpy(reinterpret_cast(&val), &data_[offset], sizeof(int32_t)); + return val; + } + case PropertyType::INT64: { + int64_t val; + memcpy(reinterpret_cast(&val), &data_[offset], sizeof(int64_t)); + return val; + } + case PropertyType::VID: { + // This is to be compatible with V1, so we treat it as + // 8-byte long string + return std::string(&data_[offset], sizeof(int64_t)); + } + case PropertyType::FLOAT: { + float val; + memcpy(reinterpret_cast(&val), &data_[offset], sizeof(float)); + return val; + } + case PropertyType::DOUBLE: { + double val; + memcpy(reinterpret_cast(&val), &data_[offset], sizeof(double)); + return val; + } + case PropertyType::STRING: { + int32_t strOffset; + int32_t strLen; + memcpy(reinterpret_cast(&strOffset), &data_[offset], sizeof(int32_t)); + memcpy(reinterpret_cast(&strLen), &data_[offset + sizeof(int32_t)], sizeof(int32_t)); + if (static_cast(strOffset) == data_.size() && strLen == 0) { + return std::string(); + } + CHECK_LT(strOffset, data_.size()); + return std::string(&data_[strOffset], strLen); + } + case PropertyType::FIXED_STRING: { + return std::string(&data_[offset], field->size()); + } + case PropertyType::TIMESTAMP: { + Timestamp ts; + memcpy(reinterpret_cast(&ts), &data_[offset], sizeof(Timestamp)); + return ts; + } + case PropertyType::DATE: { + Date dt; + memcpy(reinterpret_cast(&dt.year), &data_[offset], sizeof(int16_t)); + memcpy(reinterpret_cast(&dt.month), &data_[offset + sizeof(int16_t)], sizeof(int8_t)); + memcpy(reinterpret_cast(&dt.day), + &data_[offset + sizeof(int16_t) + sizeof(int8_t)], + sizeof(int8_t)); + return dt; + } + case PropertyType::TIME: { + Time t; + memcpy(reinterpret_cast(&t.hour), &data_[offset], sizeof(int8_t)); + memcpy(reinterpret_cast(&t.minute), &data_[offset + sizeof(int8_t)], sizeof(int8_t)); + memcpy(reinterpret_cast(&t.sec), &data_[offset + 2 * sizeof(int8_t)], sizeof(int8_t)); + memcpy(reinterpret_cast(&t.microsec), + &data_[offset + 3 * sizeof(int8_t)], + sizeof(int32_t)); + return t; + } + case PropertyType::DATETIME: { + DateTime dt; + int16_t year; + int8_t month; + int8_t day; + int8_t hour; + int8_t minute; + int8_t sec; + int32_t microsec; + memcpy(reinterpret_cast(&year), &data_[offset], sizeof(int16_t)); + memcpy(reinterpret_cast(&month), &data_[offset + sizeof(int16_t)], sizeof(int8_t)); + memcpy(reinterpret_cast(&day), + &data_[offset + sizeof(int16_t) + sizeof(int8_t)], + sizeof(int8_t)); + memcpy(reinterpret_cast(&hour), + &data_[offset + sizeof(int16_t) + 2 * sizeof(int8_t)], + sizeof(int8_t)); + memcpy(reinterpret_cast(&minute), + &data_[offset + sizeof(int16_t) + 3 * sizeof(int8_t)], + sizeof(int8_t)); + memcpy(reinterpret_cast(&sec), + &data_[offset + sizeof(int16_t) + 4 * sizeof(int8_t)], + sizeof(int8_t)); + memcpy(reinterpret_cast(µsec), + &data_[offset + sizeof(int16_t) + 5 * sizeof(int8_t)], + sizeof(int32_t)); + dt.year = year; + dt.month = month; + dt.day = day; + dt.hour = hour; + dt.minute = minute; + dt.sec = sec; + dt.microsec = microsec; + return dt; + } + case PropertyType::DURATION: { + Duration d; + memcpy(reinterpret_cast(&d.seconds), &data_[offset], sizeof(int64_t)); + memcpy(reinterpret_cast(&d.microseconds), + &data_[offset + sizeof(int64_t)], + sizeof(int32_t)); + memcpy(reinterpret_cast(&d.months), + &data_[offset + sizeof(int64_t) + sizeof(int32_t)], + sizeof(int32_t)); + return d; + } + case PropertyType::GEOGRAPHY: { + int32_t strOffset; + int32_t strLen; + memcpy(reinterpret_cast(&strOffset), &data_[offset], sizeof(int32_t)); + memcpy(reinterpret_cast(&strLen), &data_[offset + sizeof(int32_t)], sizeof(int32_t)); + if (static_cast(strOffset) == data_.size() && strLen == 0) { + return Value::kEmpty; // Is it ok to return Value::kEmpty? + } + CHECK_LT(strOffset, data_.size()); + auto wkb = std::string(&data_[strOffset], strLen); + // Parse a geography from the wkb, normalize it and then verify its validity. + auto geogRet = Geography::fromWKB(wkb, true, true); + if (!geogRet.ok()) { + LOG(WARNING) << "Geography::fromWKB failed: " << geogRet.status(); + return Value::kNullBadData; // Is it ok to return Value::kNullBadData? + } + return std::move(geogRet).value(); + } + case PropertyType::UNKNOWN: + break; + } + LOG(FATAL) << "Should not reach here, illegal property type: " << static_cast(field->type()); + return Value::kNullBadType; +} + +int64_t RowReaderV2::getTimestamp() const noexcept { + return *reinterpret_cast(data_.begin() + (data_.size() - sizeof(int64_t))); +} + +} // namespace nebula diff --git a/src/codec/RowReaderV2.h b/src/codec/RowReaderV2.h new file mode 100644 index 0000000..d3ce9ec --- /dev/null +++ b/src/codec/RowReaderV2.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_ROWREADERV2_H_ +#define CODEC_ROWREADERV2_H_ + +#include + +#include "common/base/Base.h" +#include "common/meta/NebulaSchemaProvider.h" + +namespace nebula { + +class RowReaderWrapper; + +/** + * This class decodes the data from version 2.0 + */ +class RowReaderV2 { + friend class RowReaderWrapper; + + FRIEND_TEST(RowReaderV2, encodedData); + + public: + ~RowReaderV2() = default; + + Value getValueByName(const std::string& prop) const; + Value getValueByIndex(const int64_t index) const; + int64_t getTimestamp() const noexcept; + + size_t headerLen() const noexcept { + return headerLen_; + } + + const meta::NebulaSchemaProvider* getSchema() const { + return schema_; + } + + SchemaVer schemaVer() const noexcept { + return schema_->getVersion(); + } + + size_t numFields() const noexcept { + return schema_->getNumFields(); + } + + const std::string getData() const { + return data_.toString(); + } + + private: + bool resetImpl(meta::NebulaSchemaProvider const* schema, folly::StringPiece row); + + private: + meta::NebulaSchemaProvider const* schema_; + folly::StringPiece data_; + size_t headerLen_; + size_t numNullBytes_; + + RowReaderV2() = default; + + // Check whether the flag at the given position is set or not + bool isNull(size_t pos) const; +}; + +} // namespace nebula +#endif // CODEC_ROWREADERV2_H_ diff --git a/src/codec/RowReaderWrapper.cpp b/src/codec/RowReaderWrapper.cpp new file mode 100644 index 0000000..431145d --- /dev/null +++ b/src/codec/RowReaderWrapper.cpp @@ -0,0 +1,176 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "codec/RowReaderWrapper.h" + +namespace nebula { + +// static +RowReaderWrapper RowReaderWrapper::getTagPropReader(meta::SchemaManager* schemaMan, + GraphSpaceID space, + TagID tag, + folly::StringPiece row) { + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (schemaVer >= 0) { + auto schema = schemaMan->getTagSchema(space, tag, schemaVer); + if (schema == nullptr) { + return RowReaderWrapper(); + } + return RowReaderWrapper(schema.get(), row, readerVer); + } else { + LOG(WARNING) << "Invalid schema version in the row data!"; + return RowReaderWrapper(); + } +} + +// static +RowReaderWrapper RowReaderWrapper::getEdgePropReader(meta::SchemaManager* schemaMan, + GraphSpaceID space, + EdgeType edge, + folly::StringPiece row) { + if (schemaMan == nullptr) { + LOG(ERROR) << "schemaMan should not be nullptr!"; + return RowReaderWrapper(); + } + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (schemaVer >= 0) { + auto schema = schemaMan->getEdgeSchema(space, edge, schemaVer); + if (schema == nullptr) { + return RowReaderWrapper(); + } + return RowReaderWrapper(schema.get(), row, readerVer); + } else { + LOG(WARNING) << "Invalid schema version in the row data!"; + return RowReaderWrapper(); + } +} + +// static +RowReaderWrapper RowReaderWrapper::getRowReader(const meta::NebulaSchemaProvider* schema, + folly::StringPiece row) { + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (schemaVer != schema->getVersion()) { + return RowReaderWrapper(); + } + return RowReaderWrapper(schema, row, readerVer); +} + +// static +RowReaderWrapper RowReaderWrapper::getRowReader( + const std::vector>& schemas, + folly::StringPiece row) { + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (static_cast(schemaVer) >= schemas.size() || + schemaVer != schemas[schemaVer]->getVersion()) { + return RowReaderWrapper(); + } + return RowReaderWrapper(schemas[schemaVer].get(), row, readerVer); +} + +RowReaderWrapper::RowReaderWrapper(const meta::NebulaSchemaProvider* schema, + const folly::StringPiece& row, + int32_t& readerVer) { + CHECK_EQ(readerVer, 2); + CHECK_NOTNULL(schema); + readerV2_.resetImpl(schema, row); + currReader_ = &readerV2_; +} + +bool RowReaderWrapper::reset(meta::NebulaSchemaProvider const* schema, + folly::StringPiece row, + int32_t readerVer) { + CHECK_EQ(readerVer, 2); + CHECK_NOTNULL(schema); + readerV2_.resetImpl(schema, row); + currReader_ = &readerV2_; + return true; +} + +bool RowReaderWrapper::reset(meta::NebulaSchemaProvider const* schema, folly::StringPiece row) { + currReader_ = nullptr; + if (schema == nullptr) { + return false; + } + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (schemaVer != schema->getVersion()) { + return false; + } + return reset(schema, row, readerVer); +} + +bool RowReaderWrapper::reset( + const std::vector>& schemas, + folly::StringPiece row) { + currReader_ = nullptr; + SchemaVer schemaVer; + int32_t readerVer; + RowReaderWrapper::getVersions(row, schemaVer, readerVer); + if (static_cast(schemaVer) >= schemas.size()) { + return false; + } + // the schema is stored from oldest to newest, so just use version as idx + if (schemaVer != schemas[schemaVer]->getVersion()) { + return false; + } + return reset(schemas[schemaVer].get(), row, readerVer); +} + +// static +void RowReaderWrapper::getVersions(const folly::StringPiece& row, + SchemaVer& schemaVer, + int32_t& readerVer) { + size_t index = 0; + if (row.empty()) { + LOG(WARNING) << "Row data is empty, so there is no version info"; + schemaVer = -1; + readerVer = 2; + return; + } + + readerVer = ((row[index] & 0x18) >> 3) + 1; + + size_t verBytes = 0; + if (readerVer == 1) { + // The first three bits indicate the number of bytes for the + // schema version. If the number is zero, no schema version + // presents + verBytes = row[index++] >> 5; + } else if (readerVer == 2) { + // The last three bits indicate the number of bytes for the + // schema version. If the number is zero, no schema version + // presents + verBytes = row[index++] & 0x07; + } else { + LOG(WARNING) << "Invalid reader version: " << readerVer; + schemaVer = -1; + return; + } + + schemaVer = 0; + if (verBytes > 0) { + if (verBytes + 1 > row.size()) { + // Data is too short + LOG(WARNING) << "Row data is too short: " << toHexStr(row); + schemaVer = -1; + return; + } + // Schema Version is stored in Little Endian + memcpy(reinterpret_cast(&schemaVer), &row[index], verBytes); + } + + return; +} + +} // namespace nebula diff --git a/src/codec/RowReaderWrapper.h b/src/codec/RowReaderWrapper.h new file mode 100644 index 0000000..d914cbf --- /dev/null +++ b/src/codec/RowReaderWrapper.h @@ -0,0 +1,258 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_ROWREADERWRAPPER_H_ +#define CODEC_ROWREADERWRAPPER_H_ + +#include + +#include "codec/RowReaderV2.h" +#include "common/base/Base.h" +#include "common/datatypes/Value.h" +#include "common/meta/NebulaSchemaProvider.h" +#include "common/meta/SchemaManager.h" + +namespace nebula { + +/** + * @brief A wrapper class to hide details of RowReaderV1 and RowReaderV2 + */ +class RowReaderWrapper { + FRIEND_TEST(RowReaderV2, encodedData); + + public: + RowReaderWrapper() = default; + + RowReaderWrapper(const RowReaderWrapper&) = delete; + + RowReaderWrapper& operator=(const RowReaderWrapper&) = delete; + + /** + * @brief Move constructor of row reader wrapper + * + * @param rhs + */ + RowReaderWrapper(RowReaderWrapper&& rhs) { + this->readerV2_ = std::move(rhs.readerV2_); + this->currReader_ = &(this->readerV2_); + } + + /** + * @brief Move assign operator + * + * @param rhs + * @return RowReaderWrapper& + */ + RowReaderWrapper& operator=(RowReaderWrapper&& rhs) { + this->readerV2_ = std::move(rhs.readerV2_); + this->currReader_ = &(this->readerV2_); + return *this; + } + + /** + * @brief Generate a row reader wrapper of tag data + * + * @param schemaMan Schema manager, used to find the related schema of data + * @param space SpaceId + * @param tag TagId + * @param row Value in kv engine + * @return RowReaderWrapper The row reader wrapper + */ + static RowReaderWrapper getTagPropReader(meta::SchemaManager* schemaMan, + GraphSpaceID space, + TagID tag, + folly::StringPiece row); + + /** + * @brief Generate a row reader wrapper of edge data + * + * @param schemaMan Schema manager, used to find the related schema of data + * @param space SpaceId + * @param tag TagId + * @param row Value in kv engine + * @return RowReaderWrapper The row reader wrapper + */ + static RowReaderWrapper getEdgePropReader(meta::SchemaManager* schemaMan, + GraphSpaceID space, + EdgeType edge, + folly::StringPiece row); + + /** + * @brief Generate a row reader wrapper of data + * + * @param schema + * @param row + * @return RowReaderWrapper + */ + static RowReaderWrapper getRowReader(meta::NebulaSchemaProvider const* schema, + folly::StringPiece row); + + /** + * @brief Generate a row reader wrapper of data, the schemas are stored in vector. + * notice: the schemas are from oldest to newest, + * usually from getAllVerTagSchema or getAllVerEdgeSchema in SchemaMan + * + * @param schemas + * @param row + * @return RowReaderWrapper + */ + static RowReaderWrapper getRowReader( + const std::vector>& schemas, + folly::StringPiece row); + + /** + * @brief Construct a new row reader wrapper + * + * @param schema + * @param row + * @param readerVer Row reader version + */ + RowReaderWrapper(const meta::NebulaSchemaProvider* schema, + const folly::StringPiece& row, + int32_t& readerVer); + + /** + * @brief Reset current row reader wrapper to of given schema, data and reader version + * + * @param schema + * @param row + * @param readVer + * @return Whether reset succeed + */ + bool reset(meta::NebulaSchemaProvider const* schema, folly::StringPiece row, int32_t readVer); + + /** + * @brief Reset current row reader wrapper to of given schema and data + * + * @param schema + * @param row + * @return Whether reset succeed + */ + bool reset(meta::NebulaSchemaProvider const* schema, folly::StringPiece row); + + /** + * @brief Reset current row reader wrapper of given schemas and data, the schemas are stored in + * vector. + * + * @param schemas + * @param row + * @return Whether reset succeed + */ + bool reset(const std::vector>& schemas, + folly::StringPiece row); + + Value getValueByName(const std::string& prop) const { + DCHECK(!!currReader_); + return currReader_->getValueByName(prop); + } + + Value getValueByIndex(const int64_t index) const { + DCHECK(!!currReader_); + return currReader_->getValueByIndex(index); + } + + int64_t getTimestamp() const noexcept { + DCHECK(!!currReader_); + return currReader_->getTimestamp(); + } + + // Return the number of bytes used for the header info + size_t headerLen() const noexcept { + DCHECK(!!currReader_); + return currReader_->headerLen(); + } + + SchemaVer schemaVer() const noexcept { + DCHECK(!!currReader_); + return currReader_->schemaVer(); + } + + size_t numFields() const noexcept { + DCHECK(!!currReader_); + return currReader_->numFields(); + } + + const meta::NebulaSchemaProvider* getSchema() const { + DCHECK(!!currReader_); + return currReader_->getSchema(); + } + + const std::string getData() const { + DCHECK(!!currReader_); + return currReader_->getData(); + } + + /** + * @brief Get schema version and reader version by data + * + * @param row Row data + * @param schemaVer Schema version + * @param readerVer Row reader version + */ + static void getVersions(const folly::StringPiece& row, SchemaVer& schemaVer, int32_t& readerVer); + + /** + * @brief Return whether wrapper points to a valid data + */ + operator bool() const noexcept { + return operator!=(nullptr); + } + + /** + * @brief Return whether wrapper points to a valid data + */ + bool operator==(std::nullptr_t) const noexcept { + return !operator!=(nullptr); + } + + /** + * @brief Return whether wrapper points to a valid data + */ + bool operator!=(std::nullptr_t) const noexcept { + return currReader_ != nullptr; + } + + /** + * @brief Return this row reader wrapper + */ + RowReaderWrapper* operator->() const noexcept { + return get(); + } + + /** + * @brief Return this row reader wrapper + */ + RowReaderWrapper* get() const noexcept { + return const_cast(this); + } + + /** + * @brief Return this row reader wrapper + */ + RowReaderWrapper* get() noexcept { + return this; + } + + /** + * @brief Return this row reader wrapper + */ + RowReaderWrapper& operator*() const noexcept { + return *get(); + } + + /** + * @brief Reset to an empty row reader + */ + void reset() noexcept { + currReader_ = nullptr; + } + + private: + RowReaderV2 readerV2_; + RowReaderV2* currReader_ = nullptr; +}; + +} // namespace nebula +#endif // CODEC_ROWREADERWRAPPER_H_ diff --git a/src/codec/RowWriterV2.cpp b/src/codec/RowWriterV2.cpp new file mode 100644 index 0000000..1e42455 --- /dev/null +++ b/src/codec/RowWriterV2.cpp @@ -0,0 +1,958 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "codec/RowWriterV2.h" + +#include + +#include "codec/Common.h" +#include "common/time/TimeUtils.h" +#include "common/time/WallClock.h" +#include "common/utils/DefaultValueContext.h" + +namespace nebula { + +using nebula::cpp2::PropertyType; + +RowWriterV2::RowWriterV2(const meta::NebulaSchemaProvider* schema) + : schema_(schema), numNullBytes_(0), approxStrLen_(0), finished_(false), outOfSpaceStr_(false) { + CHECK(!!schema_); + + // Reserve space for the header, the data, and the string values + buf_.reserve(schema_->size() + schema_->getNumFields() / 8 + 8 + 1024); + + char header = 0; + + // Header and schema version + // + // The maximum number of bytes for the header and the schema version is 8 + // + // The first byte is the header (os signature), it has the fourth-bit (from + // the right side) set to one (0x08), and the right three bits indicate + // the number of bytes used for the schema version. + // + // If all three bits are zero, the schema version is zero. If the number + // of schema version bytes is one, the maximum schema version that can be + // represented is 255 (0xFF). If the number of schema version is two, the + // maximum schema version could be 65535 (0xFFFF), and so on. + // + // The maximum schema version we support is 0x00FFFFFFFFFFFFFF (7 bytes) + int64_t ver = schema_->getVersion(); + if (ver > 0) { + if (ver <= 0x00FF) { + header = 0x09; // 0x08 | 0x01, one byte for the schema version + headerLen_ = 2; + } else if (ver < 0x00FFFF) { + header = 0x0A; // 0x08 | 0x02, two bytes for the schema version + headerLen_ = 3; + } else if (ver < 0x00FFFFFF) { + header = 0x0B; // 0x08 | 0x03, three bytes for the schema version + headerLen_ = 4; + } else if (ver < 0x00FFFFFFFF) { + header = 0x0C; // 0x08 | 0x04, four bytes for the schema version + headerLen_ = 5; + } else if (ver < 0x00FFFFFFFFFF) { + header = 0x0D; // 0x08 | 0x05, five bytes for the schema version + headerLen_ = 6; + } else if (ver < 0x00FFFFFFFFFFFF) { + header = 0x0E; // 0x08 | 0x06, six bytes for the schema version + headerLen_ = 7; + } else if (ver < 0x00FFFFFFFFFFFFFF) { + header = 0x0F; // 0x08 | 0x07, seven bytes for the schema version + headerLen_ = 8; + } else { + LOG(FATAL) << "Schema version too big"; + header = 0x0F; // 0x08 | 0x07, seven bytes for the schema version + headerLen_ = 8; + } + buf_.append(&header, 1); + buf_.append(reinterpret_cast(&ver), buf_[0] & 0x07); + } else { + header = 0x08; + headerLen_ = 1; + buf_.append(&header, 1); + } + + // Null flags + size_t numNullables = schema_->getNumNullableFields(); + if (numNullables > 0) { + numNullBytes_ = ((numNullables - 1) >> 3) + 1; + } + + // Reserve the space for the data, including the Null bits + // All variant length string will be appended to the end + buf_.resize(headerLen_ + numNullBytes_ + schema_->size(), '\0'); + + isSet_.resize(schema_->getNumFields(), false); +} + +RowWriterV2::RowWriterV2(const meta::NebulaSchemaProvider* schema, std::string&& encoded) + : schema_(schema), finished_(false), outOfSpaceStr_(false) { + auto len = encoded.size(); + buf_ = std::move(encoded).substr(0, len - sizeof(int64_t)); + processV2EncodedStr(); +} + +RowWriterV2::RowWriterV2(const meta::NebulaSchemaProvider* schema, const std::string& encoded) + : schema_(schema), + buf_(encoded.substr(0, encoded.size() - sizeof(int64_t))), + finished_(false), + outOfSpaceStr_(false) { + processV2EncodedStr(); +} + +RowWriterV2::RowWriterV2(RowReaderWrapper& reader) : RowWriterV2(reader.getSchema()) { + for (size_t i = 0; i < reader.numFields(); i++) { + Value v = reader.getValueByIndex(i); + switch (v.type()) { + case Value::Type::NULLVALUE: + setNull(i); + break; + case Value::Type::BOOL: + set(i, v.getBool()); + break; + case Value::Type::INT: + set(i, v.getInt()); + break; + case Value::Type::FLOAT: + set(i, v.getFloat()); + break; + case Value::Type::STRING: + approxStrLen_ += v.getStr().size(); + set(i, v.moveStr()); + break; + case Value::Type::DATE: + set(i, v.moveDate()); + break; + case Value::Type::TIME: + set(i, v.moveTime()); + break; + case Value::Type::DATETIME: + set(i, v.moveDateTime()); + break; + case Value::Type::GEOGRAPHY: + set(i, v.moveGeography()); + break; + case Value::Type::DURATION: + set(i, v.moveDuration()); + break; + default: + LOG(FATAL) << "Invalid data: " << v << ", type: " << v.typeName(); + isSet_[i] = false; + continue; + } + isSet_[i] = true; + } +} + +void RowWriterV2::processV2EncodedStr() { + CHECK_EQ(0x08, buf_[0] & 0x18); + int32_t verBytes = buf_[0] & 0x07; + SchemaVer ver = 0; + if (verBytes > 0) { + memcpy(reinterpret_cast(&ver), &buf_[1], verBytes); + } + CHECK_EQ(ver, schema_->getVersion()) + << "The data is encoded by schema version " << ver + << ", while the provided schema version is " << schema_->getVersion(); + + headerLen_ = verBytes + 1; + + // Null flags + size_t numNullables = schema_->getNumNullableFields(); + if (numNullables > 0) { + numNullBytes_ = ((numNullables - 1) >> 3) + 1; + } else { + numNullBytes_ = 0; + } + + approxStrLen_ = buf_.size() - headerLen_ - numNullBytes_ - schema_->size() - sizeof(int64_t); + isSet_.resize(schema_->getNumFields(), true); +} + +void RowWriterV2::setNullBit(ssize_t pos) { + static const uint8_t orBits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; + + size_t offset = headerLen_ + (pos >> 3); + buf_[offset] = buf_[offset] | orBits[pos & 0x0000000000000007L]; +} + +void RowWriterV2::clearNullBit(ssize_t pos) { + static const uint8_t andBits[] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD, 0xFE}; + + size_t offset = headerLen_ + (pos >> 3); + buf_[offset] = buf_[offset] & andBits[pos & 0x0000000000000007L]; +} + +bool RowWriterV2::checkNullBit(ssize_t pos) const { + static const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; + + size_t offset = headerLen_ + (pos >> 3); + int8_t flag = buf_[offset] & bits[pos & 0x0000000000000007L]; + return flag != 0; +} + +WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) { + CHECK(!finished_) << "You have called finish()"; + if (index < 0 || static_cast(index) >= schema_->getNumFields()) { + return WriteResult::UNKNOWN_FIELD; + } + + switch (val.type()) { + case Value::Type::NULLVALUE: { + if (val.isBadNull()) { + // Property value never be bad null + return WriteResult::TYPE_MISMATCH; + } + return setNull(index); + } + case Value::Type::BOOL: + return write(index, val.getBool()); + case Value::Type::INT: + return write(index, val.getInt()); + case Value::Type::FLOAT: + return write(index, val.getFloat()); + case Value::Type::STRING: + return write(index, val.getStr()); + case Value::Type::DATE: + return write(index, val.getDate()); + case Value::Type::TIME: + return write(index, val.getTime()); + case Value::Type::DATETIME: + return write(index, val.getDateTime()); + case Value::Type::GEOGRAPHY: + return write(index, val.getGeography()); + case Value::Type::DURATION: + return write(index, val.getDuration()); + default: + return WriteResult::TYPE_MISMATCH; + } +} + +WriteResult RowWriterV2::setValue(const std::string& name, const Value& val) { + CHECK(!finished_) << "You have called finish()"; + int64_t index = schema_->getFieldIndex(name); + return setValue(index, val); +} + +WriteResult RowWriterV2::setNull(ssize_t index) { + CHECK(!finished_) << "You have called finish()"; + if (index < 0 || static_cast(index) >= schema_->getNumFields()) { + return WriteResult::UNKNOWN_FIELD; + } + + // Make sure the field is nullable + auto field = schema_->field(index); + if (!field->nullable()) { + return WriteResult::NOT_NULLABLE; + } + + setNullBit(field->nullFlagPos()); + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::setNull(const std::string& name) { + CHECK(!finished_) << "You have called finish()"; + int64_t index = schema_->getFieldIndex(name); + return setNull(index); +} + +WriteResult RowWriterV2::write(ssize_t index, bool v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::BOOL: + case PropertyType::INT8: + buf_[offset] = v ? 0x01 : 0; + break; + case PropertyType::INT64: + buf_[offset + 7] = 0; + buf_[offset + 6] = 0; + buf_[offset + 5] = 0; + buf_[offset + 4] = 0; // fallthrough + case PropertyType::INT32: + buf_[offset + 3] = 0; + buf_[offset + 2] = 0; // fallthrough + case PropertyType::INT16: + buf_[offset + 1] = 0; + buf_[offset + 0] = v ? 0x01 : 0; + break; + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, float v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::INT8: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int8_t iv = std::round(v); + buf_[offset] = iv; + break; + } + case PropertyType::INT16: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int16_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + if (v > static_cast(std::numeric_limits::max()) || + v < static_cast(std::numeric_limits::min())) { + return WriteResult::OUT_OF_RANGE; + } + int32_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int32_t)); + break; + } + case PropertyType::INT64: { + if (v > static_cast(std::numeric_limits::max()) || + v < static_cast(std::numeric_limits::min())) { + return WriteResult::OUT_OF_RANGE; + } + int64_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + memcpy(&buf_[offset], reinterpret_cast(&v), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + double dv = v; + memcpy(&buf_[offset], reinterpret_cast(&dv), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, double v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::INT8: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int8_t iv = std::round(v); + buf_[offset] = iv; + break; + } + case PropertyType::INT16: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int16_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int32_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int32_t)); + break; + } + case PropertyType::INT64: { + if (v > static_cast(std::numeric_limits::max()) || + v < static_cast(std::numeric_limits::min())) { + return WriteResult::OUT_OF_RANGE; + } + int64_t iv = std::round(v); + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::lowest()) { + return WriteResult::OUT_OF_RANGE; + } + float fv = v; + memcpy(&buf_[offset], reinterpret_cast(&fv), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + memcpy(&buf_[offset], reinterpret_cast(&v), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, uint8_t v) { + return write(index, static_cast(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, int8_t v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::BOOL: { + buf_[offset] = v == 0 ? 0x00 : 0x01; + break; + } + case PropertyType::INT8: { + buf_[offset] = v; + break; + } + case PropertyType::INT16: { + int16_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + int32_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int32_t)); + break; + } + case PropertyType::INT64: { + int64_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + float fv = v; + memcpy(&buf_[offset], reinterpret_cast(&fv), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + double dv = v; + memcpy(&buf_[offset], reinterpret_cast(&dv), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, uint16_t v) { + return write(index, static_cast(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, int16_t v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::BOOL: { + buf_[offset] = v == 0 ? 0x00 : 0x01; + break; + } + case PropertyType::INT8: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int8_t iv = v; + buf_[offset] = iv; + break; + } + case PropertyType::INT16: { + memcpy(&buf_[offset], reinterpret_cast(&v), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + int32_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int32_t)); + break; + } + case PropertyType::INT64: { + int64_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + float fv = v; + memcpy(&buf_[offset], reinterpret_cast(&fv), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + double dv = v; + memcpy(&buf_[offset], reinterpret_cast(&dv), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, uint32_t v) { + return write(index, static_cast(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, int32_t v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::BOOL: { + buf_[offset] = v == 0 ? 0x00 : 0x01; + break; + } + case PropertyType::INT8: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int8_t iv = v; + buf_[offset] = iv; + break; + } + case PropertyType::INT16: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int16_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + memcpy(&buf_[offset], reinterpret_cast(&v), sizeof(int32_t)); + break; + } + case PropertyType::TIMESTAMP: { + // 32-bit timestamp can only support upto 2038-01-19 + auto ret = time::TimeUtils::toTimestamp(v); + if (!ret.ok()) { + return WriteResult::OUT_OF_RANGE; + } + auto ts = ret.value().getInt(); + memcpy(&buf_[offset], reinterpret_cast(&ts), sizeof(int64_t)); + break; + } + case PropertyType::INT64: { + int64_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + float fv = v; + memcpy(&buf_[offset], reinterpret_cast(&fv), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + double dv = v; + memcpy(&buf_[offset], reinterpret_cast(&dv), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, uint64_t v) { + return write(index, static_cast(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, int64_t v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::BOOL: { + buf_[offset] = v == 0 ? 0x00 : 0x01; + break; + } + case PropertyType::INT8: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int8_t iv = v; + buf_[offset] = iv; + break; + } + case PropertyType::INT16: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int16_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int16_t)); + break; + } + case PropertyType::INT32: { + if (v > std::numeric_limits::max() || v < std::numeric_limits::min()) { + return WriteResult::OUT_OF_RANGE; + } + int32_t iv = v; + memcpy(&buf_[offset], reinterpret_cast(&iv), sizeof(int32_t)); + break; + } + case PropertyType::TIMESTAMP: { + // 64-bit timestamp has way broader time range + auto ret = time::TimeUtils::toTimestamp(v); + if (!ret.ok()) { + return WriteResult::OUT_OF_RANGE; + } + auto ts = ret.value().getInt(); + memcpy(&buf_[offset], reinterpret_cast(&ts), sizeof(int64_t)); + break; + } + case PropertyType::INT64: { + memcpy(&buf_[offset], reinterpret_cast(&v), sizeof(int64_t)); + break; + } + case PropertyType::FLOAT: { + float fv = v; + memcpy(&buf_[offset], reinterpret_cast(&fv), sizeof(float)); + break; + } + case PropertyType::DOUBLE: { + double dv = v; + memcpy(&buf_[offset], reinterpret_cast(&dv), sizeof(double)); + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const std::string& v) { + return write(index, folly::StringPiece(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, const char* v) { + return write(index, folly::StringPiece(v)); +} + +WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v, bool isWKB) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::GEOGRAPHY: { + // If v is a not a WKB string, we need report error. + if (!isWKB) { + return WriteResult::TYPE_MISMATCH; + } + [[fallthrough]]; + } + case PropertyType::STRING: { + if (isSet_[index]) { + // The string value has already been set, we need to turn it + // into out-of-space strings then + outOfSpaceStr_ = true; + } + + int32_t strOffset; + int32_t strLen; + if (outOfSpaceStr_) { + strList_.emplace_back(v.data(), v.size()); + strOffset = 0; + // Length field is the index to the out-of-space string list + strLen = strList_.size() - 1; + } else { + // Append to the end + strOffset = buf_.size(); + strLen = v.size(); + buf_.append(v.data(), strLen); + } + memcpy(&buf_[offset], reinterpret_cast(&strOffset), sizeof(int32_t)); + memcpy(&buf_[offset + sizeof(int32_t)], reinterpret_cast(&strLen), sizeof(int32_t)); + approxStrLen_ += v.size(); + break; + } + case PropertyType::FIXED_STRING: { + // In-place string. If the pass-in string is longer than the pre-defined + // fixed length, the string will be truncated to the fixed length + size_t len = v.size() > field->size() ? utf8CutSize(v, field->size()) : v.size(); + strncpy(&buf_[offset], v.data(), len); + if (len < field->size()) { + memset(&buf_[offset + len], 0, field->size() - len); + } + break; + } + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const Date& v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::DATE: + memcpy(&buf_[offset], reinterpret_cast(&v.year), sizeof(int16_t)); + buf_[offset + sizeof(int16_t)] = v.month; + buf_[offset + sizeof(int16_t) + sizeof(int8_t)] = v.day; + break; + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const Time& v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::TIME: + buf_[offset] = v.hour; + buf_[offset + sizeof(int8_t)] = v.minute; + buf_[offset + 2 * sizeof(int8_t)] = v.sec; + memcpy(&buf_[offset + 3 * sizeof(int8_t)], + reinterpret_cast(&v.microsec), + sizeof(int32_t)); + break; + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + int16_t year = v.year; + int8_t month = v.month; + int8_t day = v.day; + int8_t hour = v.hour; + int8_t minute = v.minute; + int8_t sec = v.sec; + int32_t microsec = v.microsec; + switch (field->type()) { + case PropertyType::DATETIME: + memcpy(&buf_[offset], reinterpret_cast(&year), sizeof(int16_t)); + buf_[offset + sizeof(int16_t)] = month; + buf_[offset + sizeof(int16_t) + sizeof(int8_t)] = day; + buf_[offset + sizeof(int16_t) + 2 * sizeof(int8_t)] = hour; + buf_[offset + sizeof(int16_t) + 3 * sizeof(int8_t)] = minute; + buf_[offset + sizeof(int16_t) + 4 * sizeof(int8_t)] = sec; + memcpy(&buf_[offset + sizeof(int16_t) + 5 * sizeof(int8_t)], + reinterpret_cast(µsec), + sizeof(int32_t)); + break; + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const Duration& v) { + auto field = schema_->field(index); + auto offset = headerLen_ + numNullBytes_ + field->offset(); + switch (field->type()) { + case PropertyType::DURATION: + memcpy(&buf_[offset], reinterpret_cast(&v.seconds), sizeof(int64_t)); + memcpy(&buf_[offset + sizeof(int64_t)], + reinterpret_cast(&v.microseconds), + sizeof(int32_t)); + memcpy(&buf_[offset + sizeof(int64_t) + sizeof(int32_t)], + reinterpret_cast(&v.months), + sizeof(int32_t)); + break; + default: + return WriteResult::TYPE_MISMATCH; + } + if (field->nullable()) { + clearNullBit(field->nullFlagPos()); + } + isSet_[index] = true; + return WriteResult::SUCCEEDED; +} + +WriteResult RowWriterV2::write(ssize_t index, const Geography& v) { + auto field = schema_->field(index); + auto geoShape = field->geoShape(); + if (geoShape != meta::cpp2::GeoShape::ANY && + folly::to(geoShape) != folly::to(v.shape())) { + return WriteResult::TYPE_MISMATCH; + } + // Geography is stored as WKB format. + // WKB is a binary string. + std::string wkb = v.asWKB(); + return write(index, folly::StringPiece(wkb), true); +} + +WriteResult RowWriterV2::checkUnsetFields() { + DefaultValueContext expCtx; + for (size_t i = 0; i < schema_->getNumFields(); i++) { + if (!isSet_[i]) { + auto field = schema_->field(i); + if (!field->nullable() && !field->hasDefault()) { + // The field neither can be NULL, nor has a default value + return WriteResult::FIELD_UNSET; + } + + WriteResult r = WriteResult::SUCCEEDED; + if (field->hasDefault()) { + ObjectPool pool; + auto& exprStr = field->defaultValue(); + auto expr = Expression::decode(&pool, folly::StringPiece(exprStr.data(), exprStr.size())); + auto defVal = Expression::eval(expr, expCtx); + switch (defVal.type()) { + case Value::Type::NULLVALUE: + setNullBit(field->nullFlagPos()); + break; + case Value::Type::BOOL: + r = write(i, defVal.getBool()); + break; + case Value::Type::INT: + r = write(i, defVal.getInt()); + break; + case Value::Type::FLOAT: + r = write(i, defVal.getFloat()); + break; + case Value::Type::STRING: + r = write(i, defVal.getStr()); + break; + case Value::Type::DATE: + r = write(i, defVal.getDate()); + break; + case Value::Type::TIME: + r = write(i, defVal.getTime()); + break; + case Value::Type::DATETIME: + r = write(i, defVal.getDateTime()); + break; + case Value::Type::GEOGRAPHY: + r = write(i, defVal.getGeography()); + break; + case Value::Type::DURATION: + r = write(i, defVal.getDuration()); + break; + default: + LOG(FATAL) << "Unsupported default value type: " << defVal.typeName() + << ", default value: " << defVal + << ", default value expr: " << field->defaultValue(); + return WriteResult::TYPE_MISMATCH; + } + } else { + // Set NULL + setNullBit(field->nullFlagPos()); + } + + if (r != WriteResult::SUCCEEDED) { + return r; + } + } + } + + return WriteResult::SUCCEEDED; +} + +std::string RowWriterV2::processOutOfSpace() { + std::string temp; + // Reserve enough space to avoid memory re-allocation + temp.reserve(headerLen_ + numNullBytes_ + schema_->size() + approxStrLen_ + sizeof(int64_t)); + // Copy the data except the strings + temp.append(buf_.data(), headerLen_ + numNullBytes_ + schema_->size()); + + // Now let's process all strings + for (size_t i = 0; i < schema_->getNumFields(); i++) { + auto field = schema_->field(i); + if (field->type() != PropertyType::STRING && field->type() != PropertyType::GEOGRAPHY) { + continue; + } + + size_t offset = headerLen_ + numNullBytes_ + field->offset(); + int32_t oldOffset; + int32_t newOffset = temp.size(); + int32_t strLen; + + if (field->nullable() && checkNullBit(field->nullFlagPos())) { + // Null string + newOffset = strLen = 0; + } else { + // load the old offset and string length + memcpy(reinterpret_cast(&oldOffset), &buf_[offset], sizeof(int32_t)); + memcpy(reinterpret_cast(&strLen), &buf_[offset + sizeof(int32_t)], sizeof(int32_t)); + + if (oldOffset > 0) { + temp.append(&buf_[oldOffset], strLen); + } else { + // Out of space string + CHECK_LT(strLen, strList_.size()); + temp.append(strList_[strLen]); + strLen = strList_[strLen].size(); + } + } + + // Set the new offset and length + memcpy(&temp[offset], reinterpret_cast(&newOffset), sizeof(int32_t)); + memcpy(&temp[offset + sizeof(int32_t)], reinterpret_cast(&strLen), sizeof(int32_t)); + } + return temp; +} + +WriteResult RowWriterV2::finish() { + CHECK(!finished_) << "You have called finish()"; + + // First to check whether all fields are set. If not, to check whether + // it can be NULL or there is a default value for the field + WriteResult res = checkUnsetFields(); + if (res != WriteResult::SUCCEEDED) { + return res; + } + + // Next to process out-of-space strings + if (outOfSpaceStr_) { + buf_ = processOutOfSpace(); + } + + // The timestamp will be saved to the tail of buf_ + auto ts = time::WallClock::fastNowInMicroSec(); + buf_.append(reinterpret_cast(&ts), sizeof(int64_t)); + + finished_ = true; + return WriteResult::SUCCEEDED; +} + +} // namespace nebula diff --git a/src/codec/RowWriterV2.h b/src/codec/RowWriterV2.h new file mode 100644 index 0000000..6c544d0 --- /dev/null +++ b/src/codec/RowWriterV2.h @@ -0,0 +1,269 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_ROWWRITERV2_H_ +#define CODEC_ROWWRITERV2_H_ + +#include "codec/RowReaderWrapper.h" +#include "common/base/Base.h" +#include "common/meta/NebulaSchemaProvider.h" + +namespace nebula { + +enum class WriteResult { + SUCCEEDED = 0, + UNKNOWN_FIELD = -1, + TYPE_MISMATCH = -2, + OUT_OF_RANGE = -3, + NOT_NULLABLE = -4, + FIELD_UNSET = -5, + INCORRECT_VALUE = -6, +}; + +/******************************************************************************** + + Encoder version 2 + + This is the second version of encoder. It is not compatible with the first + version. The purpose of this new version is to improve the read performance + + The first byte of the encoded string indicates which version of the encoder + used to encode the properties. Here is the explanation of the header byte + + Version 1: + v v v 0 0 b b b + In version 1, the middle two bits are always zeros. The left three bits + indicates the number of bytes used for the schema version, while the right + three bits indicates the number of bytes used for the block offsets + + Version 2: + 0 0 0 0 1 v v v + In version 2, the left three bits are reserved. The middle two bits + indicate the encoder version, and the right three bits indicate the number + of bytes used for the schema version + + The biggest change from version 1 to version 2 is every property now has a + fixed length. As long as you know the property type, you know how many bytes + the property occupies. In other words, given the sequence number of the + property, you get the start point of the property in O(1) time. + + The types supported by the encoder version 2 are: + BOOL (1 byte) + INT8 (1 byte) + INT16 (2 bytes) + INT32 (4 bytes) + INT64 (8 bytes) + FLOAT (4 bytes) + DOUBLE (8 bytes) + STRING (8 bytes) * + FIXED_STRING (Length defined in the schema) + TIMESTAMP (8 bytes) + DATE (4 bytes) + DATETIME (15 bytes) + GEOGRAPHY (8 bytes) * + + All except STRING typed properties are stored in-place. The STRING property + stored the offset of the string content in the first 4 bytes and the length + of the string in the last 4 bytes. The string content is appended to the end + of the encoded string + + The encoder version 2 also supports the NULL value for all types. It uses + one bit flag to indicate whether a property is NULL if the property is + nullable. So one byte can represent NULL values for 8 nullable properties. + The total number of bytes needed for the NULL flags is + + ((number_of_nullable_props - 1) >> 3) + 1 + + Here is the overall byte sequence for the version 2 encoding + +
+ | | | | + 1 byte 0 - 7 bytes 0+ bytes N bytes + +********************************************************************************/ +class RowWriterV2 { + public: + explicit RowWriterV2(const meta::NebulaSchemaProvider* schema); + // This constructor only takes a V2 encoded string + RowWriterV2(const meta::NebulaSchemaProvider* schema, std::string&& encoded); + // This constructor only takes a V2 encoded string + RowWriterV2(const meta::NebulaSchemaProvider* schema, const std::string& encoded); + // This constructor can handle constructed from RowReaderWrapper, which is V2 reader + explicit RowWriterV2(RowReaderWrapper& reader); + + ~RowWriterV2() = default; + + /** + * @brief Return the exact length of the encoded binary array + * + * @return int64_t + */ + int64_t size() const noexcept { + return buf_.size(); + } + + /** + * @brief Return the related schema + * + * @return const meta::NebulaSchemaProvider* + */ + const meta::NebulaSchemaProvider* schema() const { + return schema_; + } + + /** + * @brief Get the encoded string + * + * @return const std::string& + */ + const std::string& getEncodedStr() const { + CHECK(finished_) << "You need to call finish() first"; + return buf_; + } + + /** + * @brief Get the encoded string with move + * + * @return std::string + */ + std::string moveEncodedStr() { + CHECK(finished_) << "You need to call finish() first"; + return std::move(buf_); + } + + /** + * @brief Finish setting fields, begin to encode + * + * @return WriteResult Whether encode succeed + */ + WriteResult finish(); + + // Data write + /** + * @brief Set propertyfield value by index + * + * @tparam T + * @param index Field index + * @param v Value to write + * @return WriteResult + */ + template + WriteResult set(size_t index, T&& v) { + CHECK(!finished_) << "You have called finish()"; + if (index >= schema_->getNumFields()) { + return WriteResult::UNKNOWN_FIELD; + } + return write(index, std::forward(v)); + } + + // Data write + /** + * @brief Set property value by property name + * + * @tparam T + * @param name Property name + * @param v Value to write + * @return WriteResult + */ + template + WriteResult set(const std::string& name, T&& v) { + CHECK(!finished_) << "You have called finish()"; + int64_t index = schema_->getFieldIndex(name); + if (index >= 0) { + return write(static_cast(index), std::forward(v)); + } else { + return WriteResult::UNKNOWN_FIELD; + } + } + + /** + * @brief Set the value by index + * + * @param index + * @param val + * @return WriteResult + */ + WriteResult setValue(ssize_t index, const Value& val); + + /** + * @brief Set the value by index + * + * @param index + * @param val + * @return WriteResult + */ + WriteResult setValue(const std::string& name, const Value& val); + + /** + * @brief Set null by index + * + * @param index + * @return WriteResult + */ + WriteResult setNull(ssize_t index); + + /** + * @brief Set null by property name + * + * @param name + * @return WriteResult + */ + WriteResult setNull(const std::string& name); + + private: + const meta::NebulaSchemaProvider* schema_; + std::string buf_; + std::vector isSet_; + // The number of bytes occupied by header and the schema version + size_t headerLen_; + size_t numNullBytes_; + size_t approxStrLen_; + bool finished_; + + // When outOfSpaceStr_ is true, variant length string fields + // could hold an index, referring to the strings in the strList_ + // By default, outOfSpaceStr_ is false. It turns true only when + // the existing variant length string is modified + bool outOfSpaceStr_; + std::vector strList_; + + WriteResult checkUnsetFields(); + std::string processOutOfSpace(); + + void processV2EncodedStr(); + + void setNullBit(ssize_t pos); + void clearNullBit(ssize_t pos); + // Return true if the flag at the given position is NULL; + // otherwise, return false + bool checkNullBit(ssize_t pos) const; + + WriteResult write(ssize_t index, bool v); + WriteResult write(ssize_t index, float v); + WriteResult write(ssize_t index, double v); + + WriteResult write(ssize_t index, int8_t v); + WriteResult write(ssize_t index, int16_t v); + WriteResult write(ssize_t index, int32_t v); + WriteResult write(ssize_t index, int64_t v); + WriteResult write(ssize_t index, uint8_t v); + WriteResult write(ssize_t index, uint16_t v); + WriteResult write(ssize_t index, uint32_t v); + WriteResult write(ssize_t index, uint64_t v); + + WriteResult write(ssize_t index, const std::string& v); + WriteResult write(ssize_t index, folly::StringPiece v, bool isWKB = false); + WriteResult write(ssize_t index, const char* v); + + WriteResult write(ssize_t index, const Date& v); + WriteResult write(ssize_t index, const Time& v); + WriteResult write(ssize_t index, const DateTime& v); + WriteResult write(ssize_t index, const Duration& v); + + WriteResult write(ssize_t index, const Geography& v); +}; + +} // namespace nebula +#endif // CODEC_ROWWRITERV2_H_ diff --git a/src/codec/include/NebulaCodec.h b/src/codec/include/NebulaCodec.h new file mode 100644 index 0000000..5316be3 --- /dev/null +++ b/src/codec/include/NebulaCodec.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_INCLUDE_NEBULACODEC_H +#define CODEC_INCLUDE_NEBULACODEC_H + +#include "common/base/StatusOr.h" +#include "common/meta/NebulaSchemaProvider.h" + +namespace nebula { + +class NebulaCodec { + public: + using Value = std::any; + + virtual ~NebulaCodec() = default; + + virtual std::string encode( + std::vector values, + std::shared_ptr schema = nullptr) = 0; + + virtual StatusOr> decode( + std::string encoded, std::shared_ptr schema) = 0; +}; + +} // namespace nebula +#endif diff --git a/src/codec/test/CMakeLists.txt b/src/codec/test/CMakeLists.txt new file mode 100644 index 0000000..4234962 --- /dev/null +++ b/src/codec/test/CMakeLists.txt @@ -0,0 +1,78 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +set(CODEC_TEST_LIBS + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) + + +nebula_add_test( + NAME row_reader_v2_test + SOURCES RowReaderV2Test.cpp + OBJECTS ${CODEC_TEST_LIBS} + LIBRARIES + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + wangle + gtest +) + + +nebula_add_test( + NAME row_writer_v2_test + SOURCES RowWriterV2Test.cpp + OBJECTS ${CODEC_TEST_LIBS} + LIBRARIES + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + wangle + gtest +) + + +nebula_add_executable( + NAME row_writer_bm + SOURCES + RowWriterBenchmark.cpp + RowWriterV1.cpp + OBJECTS ${CODEC_TEST_LIBS} + LIBRARIES + ${THRIFT_LIBRARIES} + ${PROXYGEN_LIBRARIES} + follybenchmark + wangle + boost_regex +) diff --git a/src/codec/test/RowReaderV2Test.cpp b/src/codec/test/RowReaderV2Test.cpp new file mode 100644 index 0000000..13aeea9 --- /dev/null +++ b/src/codec/test/RowReaderV2Test.cpp @@ -0,0 +1,330 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "codec/RowReaderWrapper.h" +#include "common/base/Base.h" +#include "common/datatypes/Value.h" + +namespace nebula { + +using nebula::cpp2::PropertyType; + +TEST(RowReaderV2, headerInfo) { + // Simplest row, nothing in it + char data1[] = {0x08}; + meta::NebulaSchemaProvider schema1; + auto reader = RowReaderWrapper::getRowReader(&schema1, folly::StringPiece(data1, sizeof(data1))); + ASSERT_TRUE(!!reader); + EXPECT_EQ(0, reader->schemaVer()); + EXPECT_EQ(sizeof(data1), reader->headerLen()); + + // With schema version + char data2[] = {0x0A, 0x01, static_cast(0xFF)}; + meta::NebulaSchemaProvider schema2(0x00FF01); + ASSERT_TRUE(reader->reset(&schema2, folly::StringPiece(data2, sizeof(data2)))); + EXPECT_EQ(0x0000FF01, reader->schemaVer()); + EXPECT_EQ(sizeof(data2), reader->headerLen()); + + // Insert 33 fields into schema, so we will get 2 offsets + meta::NebulaSchemaProvider schema3(0x00FFFF01); + for (int i = 0; i < 33; i++) { + schema3.addField(folly::stringPrintf("Column%02d", i), PropertyType::INT64); + } + + // With schema version and offsets + char data3[] = {0x0B, 0x01, static_cast(0xFF), static_cast(0xFF)}; + ASSERT_TRUE(reader->reset(&schema3, folly::StringPiece(data3, sizeof(data3)))); + EXPECT_EQ(0x00FFFF01, reader->schemaVer()); + EXPECT_EQ(sizeof(data3), reader->headerLen()); + + // No schema version, with offsets + meta::NebulaSchemaProvider schema4; + for (int i = 0; i < 33; i++) { + schema4.addField(folly::stringPrintf("Column%02d", i), PropertyType::INT64); + } + + char data4[] = {0x08}; + ASSERT_TRUE(reader->reset(&schema4, folly::StringPiece(data4, sizeof(data4)))); + EXPECT_EQ(0, reader->schemaVer()); + EXPECT_EQ(sizeof(data4), reader->headerLen()); + + // Empty row, return illegal schema version + meta::NebulaSchemaProvider schema5; + auto reader2 = RowReaderWrapper::getRowReader(&schema5, folly::StringPiece("")); + ASSERT_FALSE(!!reader2); + ASSERT_FALSE(reader2->reset(&schema5, folly::StringPiece(""))); +} + +TEST(RowReaderV2, encodedData) { + meta::NebulaSchemaProvider schema; + // Col 0: bool_col1 -- BOOL + schema.addField("bool_col1", PropertyType::BOOL); + // Col 1: str_col1 -- FIXED_STRING + schema.addField("fixed_str_col", PropertyType::FIXED_STRING, 12); + // Col 2: int_col1 -- INT32 + schema.addField("int32_col", PropertyType::INT32); + // Col 3: int_col2 -- INT64 + schema.addField("int64_col", PropertyType::INT64); + // Col 4: vid_col -- VID + schema.addField("vid_col", PropertyType::VID); + // Col 5: str_col2 -- STRING + schema.addField("str_col", PropertyType::STRING); + // Col 6: bool_col2 -- BOOL + schema.addField("bool_col2", PropertyType::BOOL); + // Col 7: float_col -- FLOAT + schema.addField("float_col", PropertyType::FLOAT); + // Col 8: double_col -- DOUBLE + schema.addField("double_col", PropertyType::DOUBLE); + // Col 9: timestamp_col -- TIMESTAMP + schema.addField("timestamp_col", PropertyType::TIMESTAMP); + // Col 10: date_col -- DATE + schema.addField("date_col", PropertyType::DATE); + // Col 11: datetime_col -- DATETIME + schema.addField("datetime_col", PropertyType::DATETIME); + // Col 12: time_col -- TIME + schema.addField("time_col", PropertyType::TIME); + + std::string encoded; + // Single byte header (Schema version is 0, no offset) + encoded.append(1, 0x08); + // There is no nullable fields, so no need to reserve the space for + // the NULL flag + + const char* str1 = "Hello World!"; + const char* str2 = "Welcome to the future!"; + + // Col 0 + encoded.append(1, 0x01); + + // Col 1 + encoded.append(str1, strlen(str1)); + + // Col 2 + encoded.append(1, 100); + encoded.append(3, 0); + + // Col 3 + encoded.append(8, static_cast(0xFF)); + + // Col 4 + encoded.append(1, 0x11) + .append(1, 0x22) + .append(1, 0x33) + .append(1, 0x44) + .append(1, 0x55) + .append(1, 0x66) + .append(1, 0x77) + .append(1, static_cast(0x88)); + + // Col 5 + int32_t offset = 1 + schema.size(); // Header and data (no NULL flag) + int32_t len = strlen(str2); + encoded.append(reinterpret_cast(&offset), sizeof(int32_t)); + encoded.append(reinterpret_cast(&len), sizeof(int32_t)); + // String content will be append at the end + + // Col 6 + encoded.append(1, 0x00); + + // Col 7 + float pi = 3.1415926; + encoded.append(reinterpret_cast(&pi), sizeof(float)); + + // Col 8 + double e = 2.71828182845904523536028747135266249775724709369995; + encoded.append(reinterpret_cast(&e), sizeof(double)); + + // Col 9 + Timestamp ts = 1551331827; + encoded.append(reinterpret_cast(&ts), sizeof(Timestamp)); + + // Col 10 + int16_t year = 2020; + int8_t month = 2; + int8_t day = 20; + encoded.append(reinterpret_cast(&year), sizeof(int16_t)); + encoded.append(reinterpret_cast(&month), sizeof(int8_t)); + encoded.append(reinterpret_cast(&day), sizeof(int8_t)); + + // Col 11 + int8_t hour = 10; + int8_t minute = 30; + int8_t sec = 45; + int32_t microsec = 54321; + encoded.append(reinterpret_cast(&year), sizeof(int16_t)); + encoded.append(reinterpret_cast(&month), sizeof(int8_t)); + encoded.append(reinterpret_cast(&day), sizeof(int8_t)); + encoded.append(reinterpret_cast(&hour), sizeof(int8_t)); + encoded.append(reinterpret_cast(&minute), sizeof(int8_t)); + encoded.append(reinterpret_cast(&sec), sizeof(int8_t)); + encoded.append(reinterpret_cast(µsec), sizeof(int32_t)); + + // Col 12 + encoded.append(reinterpret_cast(&hour), sizeof(int8_t)); + encoded.append(reinterpret_cast(&minute), sizeof(int8_t)); + encoded.append(reinterpret_cast(&sec), sizeof(int8_t)); + encoded.append(reinterpret_cast(µsec), sizeof(int32_t)); + + // Append the Col5's string content + encoded.append(str2, strlen(str2)); + + /************************** + * Now let's read it + *************************/ + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + + // Header info + RowReaderWrapper* r = dynamic_cast(reader.get()); + ASSERT_EQ(&(r->readerV2_), r->currReader_); + EXPECT_EQ(0, reader->schemaVer()); + EXPECT_EQ(1, reader->headerLen()); + // There is no nullable fields, so no space reserved for the NULL flag + EXPECT_EQ(0, r->readerV2_.numNullBytes_); + + Value val; + + // Col 0 + val = reader->getValueByIndex(0); + EXPECT_EQ(Value::Type::BOOL, val.type()); + EXPECT_TRUE(val.getBool()); + val = reader->getValueByName("bool_col1"); + EXPECT_EQ(Value::Type::BOOL, val.type()); + EXPECT_TRUE(val.getBool()); + + // Col 1 + val = reader->getValueByIndex(1); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(str1, val.getStr()); + val = reader->getValueByName("fixed_str_col"); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(str1, val.getStr()); + + // Col 2 + val = reader->getValueByIndex(2); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(100, val.getInt()); + val = reader->getValueByName("int32_col"); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(100, val.getInt()); + + // Col 3 + val = reader->getValueByIndex(3); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(0xFFFFFFFFFFFFFFFFL, val.getInt()); + val = reader->getValueByName("int64_col"); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(0xFFFFFFFFFFFFFFFFL, val.getInt()); + + // Col 4 + val = reader->getValueByIndex(4); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(sizeof(int64_t), val.getStr().size()); + int64_t vid = 0; + memcpy(reinterpret_cast(&vid), val.getStr().data(), sizeof(int64_t)); + EXPECT_EQ(0x8877665544332211L, vid); + val = reader->getValueByName("vid_col"); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(sizeof(int64_t), val.getStr().size()); + vid = 0; + memcpy(reinterpret_cast(&vid), val.getStr().data(), sizeof(int64_t)); + EXPECT_EQ(0x8877665544332211L, vid); + + // Col 5 + val = reader->getValueByIndex(5); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(str2, val.getStr()); + val = reader->getValueByName("str_col"); + EXPECT_EQ(Value::Type::STRING, val.type()); + EXPECT_EQ(str2, val.getStr()); + + // Col 6 + val = reader->getValueByIndex(6); + EXPECT_EQ(Value::Type::BOOL, val.type()); + EXPECT_FALSE(val.getBool()); + val = reader->getValueByName("bool_col2"); + EXPECT_EQ(Value::Type::BOOL, val.type()); + EXPECT_FALSE(val.getBool()); + + // Col 7 + val = reader->getValueByIndex(7); + EXPECT_EQ(Value::Type::FLOAT, val.type()); + EXPECT_DOUBLE_EQ(pi, val.getFloat()); + val = reader->getValueByName("float_col"); + EXPECT_EQ(Value::Type::FLOAT, val.type()); + EXPECT_DOUBLE_EQ(pi, val.getFloat()); + + // Col 8 + val = reader->getValueByIndex(8); + EXPECT_EQ(Value::Type::FLOAT, val.type()); + EXPECT_DOUBLE_EQ(e, val.getFloat()); + val = reader->getValueByName("double_col"); + EXPECT_EQ(Value::Type::FLOAT, val.type()); + EXPECT_DOUBLE_EQ(e, val.getFloat()); + + // Col 9 + val = reader->getValueByIndex(9); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(1551331827, val.getInt()); + val = reader->getValueByName("timestamp_col"); + EXPECT_EQ(Value::Type::INT, val.type()); + EXPECT_EQ(1551331827, val.getInt()); + + // Col 10 + Date date; + date.year = 2020; + date.month = 2; + date.day = 20; + val = reader->getValueByIndex(10); + EXPECT_EQ(Value::Type::DATE, val.type()); + EXPECT_EQ(date, val.getDate()); + val = reader->getValueByName("date_col"); + EXPECT_EQ(Value::Type::DATE, val.type()); + EXPECT_EQ(date, val.getDate()); + + // Col 11 + DateTime dt; + dt.year = 2020; + dt.month = 2; + dt.day = 20; + dt.hour = 10; + dt.minute = 30; + dt.sec = 45; + dt.microsec = 54321; + val = reader->getValueByIndex(11); + EXPECT_EQ(Value::Type::DATETIME, val.type()); + EXPECT_EQ(dt, val.getDateTime()); + val = reader->getValueByName("datetime_col"); + EXPECT_EQ(Value::Type::DATETIME, val.type()); + EXPECT_EQ(dt, val.getDateTime()); + + // Col 12 + Time t; + t.hour = 10; + t.minute = 30; + t.sec = 45; + t.microsec = 54321; + val = reader->getValueByIndex(12); + EXPECT_EQ(Value::Type::TIME, val.type()); + EXPECT_EQ(t, val.getTime()); + val = reader->getValueByName("time_col"); + EXPECT_EQ(Value::Type::TIME, val.type()); + EXPECT_EQ(t, val.getTime()); + + // Col 13 -- non-existing column + val = reader->getValueByIndex(13); + EXPECT_EQ(Value::Type::NULLVALUE, val.type()); +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/codec/test/RowWriterBenchmark.cpp b/src/codec/test/RowWriterBenchmark.cpp new file mode 100644 index 0000000..0832df2 --- /dev/null +++ b/src/codec/test/RowWriterBenchmark.cpp @@ -0,0 +1,110 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "codec/RowWriterV2.h" +#include "codec/test/RowWriterV1.h" +#include "common/base/Base.h" + +using nebula::RowWriterV1; +using nebula::RowWriterV2; +using nebula::cpp2::PropertyType; +using nebula::meta::NebulaSchemaProvider; + +NebulaSchemaProvider schemaShort; +NebulaSchemaProvider schemaLong; + +const double e = 2.71828182845904523536028747135266249775724709369995; +const float pi = 3.14159265358979; +const std::string str = "Hello world!"; // NOLINT + +void prepareSchema(NebulaSchemaProvider* schema, size_t numRepeats) { + int32_t index = 1; + for (size_t i = 0; i < numRepeats; i++) { + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::BOOL); + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::INT64); + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::TIMESTAMP); + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::FLOAT); + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::DOUBLE); + schema->addField(folly::stringPrintf("col%02d", index++), PropertyType::STRING); + } +} + +void writeDataV1(NebulaSchemaProvider* schema, int32_t iters) { + for (int32_t i = 0; i < iters; i++) { + RowWriterV1 writer(schema); + for (size_t j = 0; j < schema->getNumFields() / 6; j++) { + writer << true << j << 1551331827 << pi << e << str; + } + std::string encoded = writer.encode(); + folly::doNotOptimizeAway(encoded); + } +} + +void writeDataV2(NebulaSchemaProvider* schema, int32_t iters) { + for (int32_t i = 0; i < iters; i++) { + RowWriterV2 writer(schema); + size_t idx = 0; + for (size_t j = 0; j < schema->getNumFields() / 6; j++) { + writer.set(idx++, true); + writer.set(idx++, j); + writer.set(idx++, 1551331827); + writer.set(idx++, pi); + writer.set(idx++, e); + writer.set(idx++, str); + } + writer.finish(); + std::string encoded = writer.moveEncodedStr(); + folly::doNotOptimizeAway(encoded); + } +} + +/************************* + * Beginning of benchmarks + ************************/ +BENCHMARK(WriteShortRowV1, iters) { + writeDataV1(&schemaShort, iters); +} + +BENCHMARK_RELATIVE(WriteShortRowV2, iters) { + writeDataV2(&schemaShort, iters); +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(WriteLongRowV1, iters) { + writeDataV1(&schemaLong, iters); +} + +BENCHMARK_RELATIVE(WriteLongRowV2, iters) { + writeDataV2(&schemaLong, iters); +} +/************************* + * End of benchmarks + ************************/ + +int main(int argc, char** argv) { + folly::init(&argc, &argv, true); + + prepareSchema(&schemaShort, 2); + prepareSchema(&schemaLong, 24); + + folly::runBenchmarks(); + return 0; +} + +/* +Benchmarked in WSL 1.0 running on i9-9880H +============================================================================ +RowWriterBenchmark.cpp relative time/iter iters/s +============================================================================ +WriteShortRowV1 1.57us 636.27K +WriteShortRowV2 131.25% 1.20us 835.11K +---------------------------------------------------------------------------- +WriteLongRowV1 7.07us 141.35K +WriteLongRowV2 91.29% 7.75us 129.04K +============================================================================ +*/ diff --git a/src/codec/test/RowWriterV1-inl.h b/src/codec/test/RowWriterV1-inl.h new file mode 100644 index 0000000..fd876c0 --- /dev/null +++ b/src/codec/test/RowWriterV1-inl.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_TEST_ROWWRITERV1_INL_H +#define CODEC_TEST_ROWWRITERV1_INL_H + +namespace nebula { + +template +typename std::enable_if::value, RowWriterV1&>::type RowWriterV1::operator<<( + T v) noexcept { + switch (schema_->getFieldType(colNum_)) { + case nebula::cpp2::PropertyType::INT64: + case nebula::cpp2::PropertyType::TIMESTAMP: { + writeInt(v); + break; + } + case nebula::cpp2::PropertyType::VID: { + cord_ << (uint64_t)v; + break; + } + default: { + LOG(WARNING) << "Incompatible value type \"int\""; + writeInt(0); + break; + } + } + + colNum_++; + if (colNum_ != 0 && (colNum_ >> 4 << 4) == colNum_) { + /* We need to record offset for every 16 fields */ + blockOffsets_.emplace_back(cord_.size()); + } + + return *this; +} + +template +typename std::enable_if::value>::type RowWriterV1::writeInt(T v) { + uint8_t buf[10]; + size_t len = folly::encodeVarint(v, buf); + DCHECK_GT(len, 0UL); + cord_.write(reinterpret_cast(&buf[0]), len); +} + +} // namespace nebula +#endif diff --git a/src/codec/test/RowWriterV1.cpp b/src/codec/test/RowWriterV1.cpp new file mode 100644 index 0000000..3247281 --- /dev/null +++ b/src/codec/test/RowWriterV1.cpp @@ -0,0 +1,233 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "codec/test/RowWriterV1.h" + +namespace nebula { + +using meta::NebulaSchemaProvider; +using meta::cpp2::Schema; +using nebula::cpp2::PropertyType; + +RowWriterV1::RowWriterV1(const NebulaSchemaProvider* schema) : schema_(schema) { + CHECK(!!schema_); +} + +int64_t RowWriterV1::size() const noexcept { + auto offsetBytes = calcOccupiedBytes(cord_.size()); + SchemaVer verBytes = 0; + if (schema_->getVersion() > 0) { + verBytes = calcOccupiedBytes(schema_->getVersion()); + } + return cord_.size() // data length + + offsetBytes * blockOffsets_.size() // block offsets length + + verBytes // version number length + + 1; // Header +} + +std::string RowWriterV1::encode() noexcept { + std::string encoded; + // Reserve enough space so resize will not happen + encoded.reserve(sizeof(int64_t) * blockOffsets_.size() + cord_.size() + 11); + encodeTo(encoded); + + return encoded; +} + +void RowWriterV1::encodeTo(std::string& encoded) noexcept { + operator<<(Skip(schema_->getNumFields() - colNum_)); + + // Header information + auto offsetBytes = calcOccupiedBytes(cord_.size()); + char header = offsetBytes - 1; + + SchemaVer ver = schema_->getVersion(); + if (ver > 0) { + auto verBytes = calcOccupiedBytes(ver); + header |= verBytes << 5; + encoded.append(&header, 1); + // Schema version is stored in Little Endian + encoded.append(reinterpret_cast(&ver), verBytes); + } else { + encoded.append(&header, 1); + } + + // Offsets are stored in Little Endian + for (auto offset : blockOffsets_) { + encoded.append(reinterpret_cast(&offset), offsetBytes); + } + + cord_.appendTo(encoded); +} + +int64_t RowWriterV1::calcOccupiedBytes(uint64_t v) const noexcept { + int64_t bytes = 0; + do { + bytes++; + v >>= 8; + } while (v); + + return bytes; +} + +/**************************** + * + * Data Stream + * + ***************************/ +RowWriterV1& RowWriterV1::operator<<(bool v) noexcept { + switch (schema_->getFieldType(colNum_)) { + case PropertyType::BOOL: + cord_ << v; + break; + default: + LOG(WARNING) << "Incompatible value type \"bool\""; + // Output a default value + cord_ << false; + break; + } + + colNum_++; + if (colNum_ != 0 && (colNum_ >> 4 << 4) == colNum_) { + /* We need to record offset for every 16 fields */ + blockOffsets_.emplace_back(cord_.size()); + } + + return *this; +} + +RowWriterV1& RowWriterV1::operator<<(float v) noexcept { + switch (schema_->getFieldType(colNum_)) { + case PropertyType::FLOAT: + cord_ << v; + break; + case PropertyType::DOUBLE: + cord_ << static_cast(v); + break; + default: + LOG(WARNING) << "Incompatible value type \"float\""; + cord_ << static_cast(0.0); + break; + } + + colNum_++; + if (colNum_ != 0 && (colNum_ >> 4 << 4) == colNum_) { + /* We need to record offset for every 16 fields */ + blockOffsets_.emplace_back(cord_.size()); + } + + return *this; +} + +RowWriterV1& RowWriterV1::operator<<(double v) noexcept { + switch (schema_->getFieldType(colNum_)) { + case PropertyType::FLOAT: + cord_ << static_cast(v); + break; + case PropertyType::DOUBLE: + cord_ << v; + break; + default: + LOG(WARNING) << "Incompatible value type \"double\""; + cord_ << static_cast(0.0); + break; + } + + colNum_++; + if (colNum_ != 0 && (colNum_ >> 4 << 4) == colNum_) { + /* We need to record offset for every 16 fields */ + blockOffsets_.emplace_back(cord_.size()); + } + + return *this; +} + +RowWriterV1& RowWriterV1::operator<<(const std::string& v) noexcept { + return operator<<(folly::StringPiece(v)); +} + +RowWriterV1& RowWriterV1::operator<<(const char* v) noexcept { + return operator<<(folly::StringPiece(v)); +} + +RowWriterV1& RowWriterV1::operator<<(folly::StringPiece v) noexcept { + switch (schema_->getFieldType(colNum_)) { + case PropertyType::STRING: { + writeInt(v.size()); + cord_.write(v.data(), v.size()); + break; + } + default: { + LOG(WARNING) << "Incompatible value type \"string\""; + writeInt(0); + break; + } + } + + colNum_++; + if (colNum_ != 0 && (colNum_ >> 4 << 4) == colNum_) { + /* We need to record offset for every 16 fields */ + blockOffsets_.emplace_back(cord_.size()); + } + + return *this; +} + +/**************************** + * + * Control Stream + * + ***************************/ +RowWriterV1& RowWriterV1::operator<<(Skip&& skip) noexcept { + if (skip.toSkip_ <= 0) { + VLOG(4) << "Nothing to skip"; + return *this; + } + + int32_t skipTo = std::min(colNum_ + skip.toSkip_, static_cast(schema_->getNumFields())); + for (int i = colNum_; i < skipTo; i++) { + switch (schema_->getFieldType(i)) { + case PropertyType::BOOL: { + cord_ << false; + break; + } + case PropertyType::INT64: + case PropertyType::TIMESTAMP: { + writeInt(0); + break; + } + case PropertyType::FLOAT: { + cord_ << static_cast(0.0); + break; + } + case PropertyType::DOUBLE: { + cord_ << static_cast(0.0); + break; + } + case PropertyType::STRING: { + writeInt(0); + break; + } + case PropertyType::VID: { + cord_ << static_cast(0); + break; + } + default: { + LOG(FATAL) << "Support for this value type has not been implemented"; + } + } + + // Update block offsets + if (i != 0 && (i >> 4 << 4) == i) { + // We need to record block offset for every 16 fields + blockOffsets_.emplace_back(cord_.size()); + } + } + colNum_ = skipTo; + + return *this; +} + +} // namespace nebula diff --git a/src/codec/test/RowWriterV1.h b/src/codec/test/RowWriterV1.h new file mode 100644 index 0000000..c911fea --- /dev/null +++ b/src/codec/test/RowWriterV1.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef CODEC_TEST_ROWWRITERV1_H_ +#define CODEC_TEST_ROWWRITERV1_H_ + +#include "common/base/Base.h" +#include "common/base/ICord.h" +#include "common/meta/NebulaSchemaProvider.h" + +namespace nebula { + +/** + * It's a write-only data streamer, used to encode one row of data + * + * It can only be used with a schema provided + */ +class RowWriterV1 { + public: + /******************* + * + * Stream Control + * + ******************/ + // Skip next few columns. Default values will be written for those + // fields + // This cannot be used if a schema is not provided + struct Skip { + friend class RowWriterV1; + explicit Skip(int64_t toSkip) : toSkip_(toSkip) {} + + private: + int64_t toSkip_; + }; + + public: + explicit RowWriterV1(const meta::NebulaSchemaProvider* schema); + + // Encode into a binary array + std::string encode() noexcept; + // Encode and attach to the given string + // For the sake of performance, th caller needs to make sure the sting + // is large enough so that resize will not happen + void encodeTo(std::string& encoded) noexcept; + + // Calculate the exact length of the encoded binary array + int64_t size() const noexcept; + + const meta::NebulaSchemaProvider* schema() const { + return schema_; + } + + // Data stream + RowWriterV1& operator<<(bool v) noexcept; + RowWriterV1& operator<<(float v) noexcept; + RowWriterV1& operator<<(double v) noexcept; + + template + typename std::enable_if::value, RowWriterV1&>::type operator<<(T v) noexcept; + + RowWriterV1& operator<<(const std::string& v) noexcept; + RowWriterV1& operator<<(folly::StringPiece v) noexcept; + RowWriterV1& operator<<(const char* v) noexcept; + + // Control stream + RowWriterV1& operator<<(Skip&& skip) noexcept; + + private: + const meta::NebulaSchemaProvider* schema_; + ICord<> cord_; + + int64_t colNum_ = 0; + + // Block offsets for every 16 fields + std::vector blockOffsets_; + + template + typename std::enable_if::value>::type writeInt(T v); + + // Calculate the number of bytes occupied (ignore the leading 0s) + int64_t calcOccupiedBytes(uint64_t v) const noexcept; +}; + +} // namespace nebula + +#include "codec/test/RowWriterV1-inl.h" + +#endif // CODEC_TEST_ROWWRITERV1_H_ diff --git a/src/codec/test/RowWriterV2Test.cpp b/src/codec/test/RowWriterV2Test.cpp new file mode 100644 index 0000000..03e4eec --- /dev/null +++ b/src/codec/test/RowWriterV2Test.cpp @@ -0,0 +1,818 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "codec/RowReaderWrapper.h" +#include "codec/RowWriterV2.h" +#include "common/base/Base.h" +#include "common/expression/ConstantExpression.h" +#include "common/time/WallClock.h" + +namespace nebula { + +using nebula::cpp2::PropertyType; + +const double e = 2.71828182845904523536028747135266249775724709369995; +const float pi = 3.14159265358979; +const std::string str = "Hello world!"; // NOLINT +const std::string fixed = "Nebula Graph"; // NOLINT +const Timestamp now = 1582183355; +// Convert timestamp now to datetime string +const Date date = {2020, 2, 20}; +const DateTime dt = {2020, 2, 20, 10, 30, 45, 0}; +const Value sVal("Hello world!"); +const Value iVal(64); +const Time t = {10, 30, 45, 0}; +// POINT(179.0 89.9) +const Geography geogPoint = Point(Coordinate(179.0, 89.9)); +// LINESTRING(0 1, 1 2, 3 7) +const Geography geogLineString = + LineString(std::vector{Coordinate(0, 1), Coordinate(1, 2), Coordinate(3, 7)}); +// POLYGON((-108.7 35.0, -100.0 46.5, -90.7 34.9, -108.7 35.0), +// (-100.1 41.4, -102.9 37.6, -96.8 37.5, -100.1 41.4)) +const Geography geogPolygon = Polygon( + std::vector>{std::vector{Coordinate(-108.7, 35.0), + Coordinate(-100.0, 46.5), + Coordinate(-90.7, 34.9), + Coordinate(-108.7, 35.0)}, + std::vector{Coordinate(-100.1, 41.4), + Coordinate(-102.9, 37.6), + Coordinate(-96.8, 37.5), + Coordinate(-100.1, 41.4)}}); +const Duration du = Duration(1, 2, 3); + +TEST(RowWriterV2, NoDefaultValue) { + meta::NebulaSchemaProvider schema(12 /*Schema version*/); + schema.addField("Col01", PropertyType::BOOL); + schema.addField("Col02", PropertyType::INT8); + schema.addField("Col03", PropertyType::INT16); + schema.addField("Col04", PropertyType::INT32); + schema.addField("Col05", PropertyType::INT64); + schema.addField("Col06", PropertyType::FLOAT); + schema.addField("Col07", PropertyType::DOUBLE); + schema.addField("Col08", PropertyType::STRING); + schema.addField("Col09", PropertyType::FIXED_STRING, 12); + schema.addField("Col10", PropertyType::TIMESTAMP); + schema.addField("Col11", PropertyType::DATE); + schema.addField("Col12", PropertyType::TIME); + schema.addField("Col13", PropertyType::DATETIME); + schema.addField("Col14", PropertyType::INT64, 0, true); + schema.addField("Col15", PropertyType::INT32, 0, true); + schema.addField("Col16", PropertyType::GEOGRAPHY, 0, false, "", meta::cpp2::GeoShape::POINT); + schema.addField("Col17", PropertyType::GEOGRAPHY, 0, false, "", meta::cpp2::GeoShape::LINESTRING); + schema.addField("Col18", PropertyType::GEOGRAPHY, 0, false, "", meta::cpp2::GeoShape::POLYGON); + schema.addField("Col19", PropertyType::GEOGRAPHY, 0, true, "", meta::cpp2::GeoShape::ANY); + schema.addField("Col20", PropertyType::DURATION); + + ASSERT_EQ(Value::Type::STRING, sVal.type()); + ASSERT_EQ(Value::Type::INT, iVal.type()); + + RowWriterV2 writer1(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(0, true)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(1, 8)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(2, 16)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(3, 32)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.setValue(4, iVal)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(5, pi)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(6, e)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.setValue(7, sVal)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(8, fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(9, now)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(10, date)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(11, t)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(12, dt)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.setNull(13)); + // Purposely skip the col15 + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(15, geogPoint)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(16, geogLineString)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(17, geogPolygon)); + // Purposely skip the col19 + EXPECT_EQ(WriteResult::SUCCEEDED, writer1.set(19, du)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer1.finish()); + + RowWriterV2 writer2(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col01", true)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col02", 8)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col03", 16)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col04", 32)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.setValue("Col05", iVal)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col06", pi)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col07", e)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.setValue("Col08", sVal)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col09", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col10", now)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col11", date)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col12", t)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col13", dt)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.setNull("Col14")); + // Purposely skip the col15 + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col16", geogPoint)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col17", geogLineString)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col18", geogPolygon)); + // Purposely skip the col19 + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col20", du)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer2.finish()); + + std::string encoded1 = std::move(writer1).moveEncodedStr(); + std::string encoded2 = writer2.getEncodedStr(); + LOG(INFO) << "encoded1, size=" << encoded1.size() << "content=" << folly::hexlify(encoded1); + LOG(INFO) << "encoded2, size=" << encoded2.size() << "content=" << folly::hexlify(encoded2); + + auto reader1 = RowReaderWrapper::getRowReader(&schema, encoded1); + auto reader2 = RowReaderWrapper::getRowReader(&schema, encoded2); + + // Col01 + Value v1 = reader1->getValueByName("Col01"); + Value v2 = reader2->getValueByIndex(0); + EXPECT_EQ(Value::Type::BOOL, v1.type()); + EXPECT_TRUE(v1.getBool()); + EXPECT_EQ(v1, v2); + + // Col02 + v1 = reader1->getValueByName("Col02"); + v2 = reader2->getValueByIndex(1); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(8, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col03 + v1 = reader1->getValueByName("Col03"); + v2 = reader2->getValueByIndex(2); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(16, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col04 + v1 = reader1->getValueByName("Col04"); + v2 = reader2->getValueByIndex(3); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(32, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col05 + v1 = reader1->getValueByName("Col05"); + v2 = reader2->getValueByIndex(4); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(iVal.getInt(), v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col06 + v1 = reader1->getValueByName("Col06"); + v2 = reader2->getValueByIndex(5); + EXPECT_EQ(Value::Type::FLOAT, v1.type()); + EXPECT_DOUBLE_EQ(pi, v1.getFloat()); + EXPECT_EQ(v1, v2); + + // Col07 + v1 = reader1->getValueByName("Col07"); + v2 = reader2->getValueByIndex(6); + EXPECT_EQ(Value::Type::FLOAT, v1.type()); + EXPECT_DOUBLE_EQ(e, v1.getFloat()); + EXPECT_EQ(v1, v2); + + // Col08 + v1 = reader1->getValueByName("Col08"); + v2 = reader2->getValueByIndex(7); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(sVal.getStr(), v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col09 + v1 = reader1->getValueByName("Col09"); + v2 = reader2->getValueByIndex(8); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(fixed, v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col10 + v1 = reader1->getValueByName("Col10"); + v2 = reader2->getValueByIndex(9); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(now, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col11 + v1 = reader1->getValueByName("Col11"); + v2 = reader2->getValueByIndex(10); + EXPECT_EQ(Value::Type::DATE, v1.type()); + EXPECT_EQ(date, v1.getDate()); + EXPECT_EQ(v1, v2); + + // Col12 + v1 = reader1->getValueByName("Col12"); + v2 = reader2->getValueByIndex(11); + EXPECT_EQ(Value::Type::TIME, v1.type()); + EXPECT_EQ(t, v1.getTime()); + EXPECT_EQ(v1, v2); + + // Col13 + v1 = reader1->getValueByName("Col13"); + v2 = reader2->getValueByIndex(12); + EXPECT_EQ(Value::Type::DATETIME, v1.type()); + EXPECT_EQ(dt, v1.getDateTime()); + EXPECT_EQ(v1, v2); + + // Col14 + v1 = reader1->getValueByName("Col14"); + v2 = reader2->getValueByIndex(13); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col15 + v1 = reader1->getValueByName("Col15"); + v2 = reader2->getValueByIndex(14); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col16 + v1 = reader1->getValueByName("Col16"); + v2 = reader2->getValueByIndex(15); + EXPECT_EQ(Value::Type::GEOGRAPHY, v1.type()); + EXPECT_EQ(geogPoint, v1.getGeography()); + EXPECT_EQ(v1, v2); + + // Col17 + v1 = reader1->getValueByName("Col17"); + v2 = reader2->getValueByIndex(16); + EXPECT_EQ(Value::Type::GEOGRAPHY, v1.type()); + EXPECT_EQ(geogLineString, v1.getGeography()); + EXPECT_EQ(v1, v2); + + // Col18 + v1 = reader1->getValueByName("Col18"); + v2 = reader2->getValueByIndex(17); + EXPECT_EQ(Value::Type::GEOGRAPHY, v1.type()); + EXPECT_EQ(geogPolygon, v1.getGeography()); + EXPECT_EQ(v1, v2); + + // Col19 + v1 = reader1->getValueByName("Col19"); + v2 = reader2->getValueByIndex(18); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col20 + v1 = reader1->getValueByName("Col20"); + v2 = reader2->getValueByIndex(19); + EXPECT_EQ(Value::Type::DURATION, v1.type()); + EXPECT_EQ(du, v1.getDuration()); + EXPECT_EQ(v1, v2); +} + +TEST(RowWriterV2, WithDefaultValue) { + ObjectPool objPool; + auto pool = &objPool; + + meta::NebulaSchemaProvider schema(7 /*Schema version*/); + schema.addField("Col01", PropertyType::BOOL, 0, true); + schema.addField( + "Col02", PropertyType::INT64, 0, false, ConstantExpression::make(pool, 12345)->encode()); + schema.addField( + "Col03", PropertyType::STRING, 0, true, ConstantExpression::make(pool, str)->encode()); + schema.addField("Col04", + PropertyType::FIXED_STRING, + 12, + false, + ConstantExpression::make(pool, fixed)->encode()); + + RowWriterV2 writer(&schema); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + + // Col01 + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col02 + v1 = reader->getValueByName("Col02"); + v2 = reader->getValueByIndex(1); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(12345, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col03 + v1 = reader->getValueByName("Col03"); + v2 = reader->getValueByIndex(2); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(str, v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col04 + v1 = reader->getValueByName("Col04"); + v2 = reader->getValueByIndex(3); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(fixed, v1.getStr()); + EXPECT_EQ(v1, v2); +} + +TEST(RowWriterV2, DoubleSet) { + meta::NebulaSchemaProvider schema(3 /*Schema version*/); + schema.addField("Col01", PropertyType::BOOL, 0, true); + schema.addField("Col02", PropertyType::INT64); + schema.addField("Col03", PropertyType::STRING); + schema.addField("Col04", PropertyType::STRING, 0, true); + schema.addField("Col05", PropertyType::FIXED_STRING, 12); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", false)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", true)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.setNull("Col01")); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col02", 1234567)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col02", 7654321)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col03", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col03", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col04", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col04", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.setNull("Col04")); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col05", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col05", str)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + + // Col01 + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col02 + v1 = reader->getValueByName("Col02"); + v2 = reader->getValueByIndex(1); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(7654321, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col03 + v1 = reader->getValueByName("Col03"); + v2 = reader->getValueByIndex(2); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(fixed, v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col04 + v1 = reader->getValueByName("Col04"); + v2 = reader->getValueByIndex(3); + EXPECT_EQ(Value::Type::NULLVALUE, v1.type()); + EXPECT_EQ(v1, v2); + + // Col05 + v1 = reader->getValueByName("Col05"); + v2 = reader->getValueByIndex(4); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(str, v1.getStr()); + EXPECT_EQ(v1, v2); +} + +TEST(RowWriterV2, Update) { + meta::NebulaSchemaProvider schema(2 /*Schema version*/); + schema.addField("Col01", PropertyType::BOOL, 0, true); + schema.addField("Col02", PropertyType::INT64); + schema.addField("Col03", PropertyType::STRING); + schema.addField("Col04", PropertyType::STRING, 0, true); + schema.addField("Col05", PropertyType::FIXED_STRING, 12); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", true)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col02", 1234567)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col03", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.setNull("Col04")); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col05", fixed)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded1 = writer.moveEncodedStr(); + + auto oldReader = RowReaderWrapper::getRowReader(&schema, encoded1); + + RowWriterV2 updater2(*oldReader); + RowWriterV2 updater1(&schema, std::move(encoded1)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater1.set("Col02", 7654321)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater2.set("Col02", 7654321)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater1.set("Col03", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater2.set("Col03", fixed)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater1.set("Col04", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater2.set("Col04", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater1.set("Col05", str)); + EXPECT_EQ(WriteResult::SUCCEEDED, updater2.set("Col05", str)); + ASSERT_EQ(WriteResult::SUCCEEDED, updater1.finish()); + ASSERT_EQ(WriteResult::SUCCEEDED, updater2.finish()); + + std::string encoded2 = updater1.moveEncodedStr(); + std::string encoded3 = updater2.moveEncodedStr(); + EXPECT_EQ(encoded2.substr(0, encoded2.size() - sizeof(int64_t)), + encoded3.substr(0, encoded3.size() - sizeof(int64_t))); + + auto reader1 = RowReaderWrapper::getRowReader(&schema, encoded2); + auto reader2 = RowReaderWrapper::getRowReader(&schema, encoded3); + + // Col01 + Value v1 = reader1->getValueByName("Col01"); + Value v2 = reader2->getValueByIndex(0); + EXPECT_EQ(Value::Type::BOOL, v1.type()); + EXPECT_TRUE(v1.getBool()); + EXPECT_EQ(v1, v2); + + // Col02 + v1 = reader1->getValueByName("Col02"); + v2 = reader2->getValueByIndex(1); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(7654321, v1.getInt()); + EXPECT_EQ(v1, v2); + + // Col03 + v1 = reader1->getValueByName("Col03"); + v2 = reader2->getValueByIndex(2); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(fixed, v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col04 + v1 = reader1->getValueByName("Col04"); + v2 = reader2->getValueByIndex(3); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(str, v1.getStr()); + EXPECT_EQ(v1, v2); + + // Col05 + v1 = reader1->getValueByName("Col05"); + v2 = reader2->getValueByIndex(4); + EXPECT_EQ(Value::Type::STRING, v1.type()); + EXPECT_EQ(str, v1.getStr()); + EXPECT_EQ(v1, v2); +} + +TEST(RowWriterV2, Timestamp) { + meta::NebulaSchemaProvider schema(20 /*Schema version*/); + schema.addField("Col01", PropertyType::TIMESTAMP); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", 1582183355)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded1 = writer.moveEncodedStr(); + auto reader1 = RowReaderWrapper::getRowReader(&schema, encoded1); + + // Col01 + Value v1 = reader1->getValueByName("Col01"); + EXPECT_EQ(Value::Type::INT, v1.type()); + EXPECT_EQ(now, v1.getInt()); + + // Invalid value test + RowWriterV2 writer2(&schema); + EXPECT_NE(WriteResult::SUCCEEDED, writer2.set("Col01", 9223372037)); + + EXPECT_NE(WriteResult::SUCCEEDED, writer2.set("Col01", -1)); + + EXPECT_NE(WriteResult::SUCCEEDED, writer2.set("Col01", "2020-02-20 15:22:35")); + + std::string dateStr = "3220-02-20 15:22:35"; // NOLINT + EXPECT_NE(WriteResult::SUCCEEDED, writer2.set(1, dateStr)); +} + +TEST(RowWriterV2, EmptyString) { + meta::NebulaSchemaProvider schema(0 /*Schema version*/); + schema.addField("Col01", PropertyType::STRING); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "")); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + + // Col01 + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ("", v1.getStr()); + EXPECT_EQ("", v2.getStr()); +} + +TEST(RowWriterV2, NumericLimit) { + meta::NebulaSchemaProvider schema(1 /*Schema version*/); + schema.addField("Col01", PropertyType::INT8); + schema.addField("Col03", PropertyType::INT16); + schema.addField("Col06", PropertyType::INT32); + schema.addField("Col07", PropertyType::INT64); + schema.addField("Col09", PropertyType::FLOAT); + schema.addField("Col11", PropertyType::DOUBLE); + + { + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(0, static_cast(std::numeric_limits::max()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(1, static_cast(std::numeric_limits::max()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(2, static_cast(std::numeric_limits::max()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(3, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(3, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(3, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(3, static_cast(std::numeric_limits::max()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(4, static_cast(std::numeric_limits::max()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(5, static_cast(std::numeric_limits::min()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(5, static_cast(std::numeric_limits::min()))); + + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(5, static_cast(std::numeric_limits::max()))); + EXPECT_EQ(WriteResult::SUCCEEDED, + writer.set(5, static_cast(std::numeric_limits::max()))); + } + { + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(0, static_cast(0))); + + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(1, static_cast(0))); + + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(2, static_cast(0))); + + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(3, static_cast(0))); + + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(4, static_cast(0))); + + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set(5, static_cast(0))); + } + { + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(-0x81))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(0x80))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(-0x81))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(0x80))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(-0x81))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, static_cast(0x80))); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, std::numeric_limits::max())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(0, std::numeric_limits::max())); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, static_cast(-0x8001))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, static_cast(0x8000))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, static_cast(-0x8001))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, static_cast(0x8000))); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, std::numeric_limits::max())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(1, std::numeric_limits::max())); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, static_cast(-0x80000001L))); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, static_cast(0x80000000L))); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, std::numeric_limits::max())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(2, std::numeric_limits::max())); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(3, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(3, std::numeric_limits::max())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(3, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(3, std::numeric_limits::max())); + + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(4, std::numeric_limits::lowest())); + EXPECT_EQ(WriteResult::OUT_OF_RANGE, writer.set(4, std::numeric_limits::max())); + } +} + +TEST(RowWriterV2, TimestampTest) { + { + meta::NebulaSchemaProvider schema(1); + schema.addField("Col01", PropertyType::STRING); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "")); + auto ts = time::WallClock::fastNowInMicroSec(); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ("", v1.getStr()); + EXPECT_EQ("", v2.getStr()); + auto ret = (reader->getTimestamp() >= ts) && + (reader->getTimestamp() <= time::WallClock::fastNowInMicroSec()); + EXPECT_TRUE(ret); + } + { + meta::NebulaSchemaProvider schema(1); + schema.addField("Col01", PropertyType::STRING); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "")); + // rewrite, for processOutOfSpace test + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "new")); + auto ts = time::WallClock::fastNowInMicroSec(); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ("new", v1.getStr()); + EXPECT_EQ("new", v2.getStr()); + auto ret = (reader->getTimestamp() >= ts) && + (reader->getTimestamp() <= time::WallClock::fastNowInMicroSec()); + EXPECT_TRUE(ret); + } + { + meta::NebulaSchemaProvider schema(1); + schema.addField("Col01", PropertyType::STRING); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "")); + // rewrite, for processOutOfSpace test + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", "abc")); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + std::string encoded = std::move(writer).moveEncodedStr(); + + RowWriterV2 writer2(&schema, encoded); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col01", "abc")); + ASSERT_EQ(WriteResult::SUCCEEDED, writer2.finish()); + std::string encoded2 = std::move(writer2).moveEncodedStr(); + EXPECT_EQ(encoded.substr(0, encoded.size() - sizeof(int64_t)), + encoded2.substr(0, encoded2.size() - sizeof(int64_t))); + } + { + meta::NebulaSchemaProvider schema(1); + schema.addField("Col01", PropertyType::INT64); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", 1)); + // rewrite, for processOutOfSpace test + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", 2)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + std::string encoded = std::move(writer).moveEncodedStr(); + + RowWriterV2 writer2(&schema, encoded); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col01", 2)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer2.finish()); + std::string encoded2 = std::move(writer2).moveEncodedStr(); + EXPECT_EQ(encoded.substr(0, encoded.size() - sizeof(int64_t)), + encoded2.substr(0, encoded2.size() - sizeof(int64_t))); + } + { + // test checkUnsetFields + meta::NebulaSchemaProvider schema(1); + schema.addField("Col01", PropertyType::INT64); + schema.addField("Col02", PropertyType::INT64); + + RowWriterV2 writer(&schema); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col01", 1)); + ASSERT_EQ(WriteResult::FIELD_UNSET, writer.finish()); + EXPECT_EQ(WriteResult::SUCCEEDED, writer.set("Col02", 2)); + auto ts = time::WallClock::fastNowInMicroSec(); + ASSERT_EQ(WriteResult::SUCCEEDED, writer.finish()); + std::string encoded = std::move(writer).moveEncodedStr(); + auto reader = RowReaderWrapper::getRowReader(&schema, encoded); + Value v1 = reader->getValueByName("Col01"); + Value v2 = reader->getValueByIndex(0); + EXPECT_EQ(1, v1.getInt()); + EXPECT_EQ(1, v2.getInt()); + v1 = reader->getValueByName("Col02"); + v2 = reader->getValueByIndex(1); + EXPECT_EQ(2, v1.getInt()); + EXPECT_EQ(2, v2.getInt()); + auto ret = (reader->getTimestamp() >= ts) && + (reader->getTimestamp() <= time::WallClock::fastNowInMicroSec()); + EXPECT_TRUE(ret); + RowWriterV2 writer2(&schema, encoded); + EXPECT_EQ(WriteResult::SUCCEEDED, writer2.set("Col02", 2)); + ASSERT_EQ(WriteResult::SUCCEEDED, writer2.finish()); + std::string encoded2 = std::move(writer2).moveEncodedStr(); + EXPECT_EQ(encoded.substr(0, encoded.size() - sizeof(int64_t)), + encoded2.substr(0, encoded2.size() - sizeof(int64_t))); + } +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt new file mode 100644 index 0000000..bc5bf97 --- /dev/null +++ b/src/common/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_subdirectory(base) +nebula_add_subdirectory(time) +nebula_add_subdirectory(network) +nebula_add_subdirectory(thrift) +nebula_add_subdirectory(fs) +nebula_add_subdirectory(thread) +nebula_add_subdirectory(process) +nebula_add_subdirectory(hdfs) +nebula_add_subdirectory(http) +nebula_add_subdirectory(stats) +nebula_add_subdirectory(charset) +nebula_add_subdirectory(algorithm) +nebula_add_subdirectory(datatypes) +nebula_add_subdirectory(conf) +nebula_add_subdirectory(meta) +nebula_add_subdirectory(expression) +nebula_add_subdirectory(function) +nebula_add_subdirectory(graph) +nebula_add_subdirectory(plugin) +nebula_add_subdirectory(utils) +nebula_add_subdirectory(ssl) +nebula_add_subdirectory(geo) +nebula_add_subdirectory(memory) +nebula_add_subdirectory(id) +nebula_add_subdirectory(log) diff --git a/src/common/algorithm/CMakeLists.txt b/src/common/algorithm/CMakeLists.txt new file mode 100644 index 0000000..44161c7 --- /dev/null +++ b/src/common/algorithm/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + nebula_algo_obj OBJECT + ReservoirSampling.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/common/algorithm/ReservoirSampling.cpp b/src/common/algorithm/ReservoirSampling.cpp new file mode 100644 index 0000000..e08f7f4 --- /dev/null +++ b/src/common/algorithm/ReservoirSampling.cpp @@ -0,0 +1,10 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/algorithm/ReservoirSampling.h" + +namespace nebula { +namespace algorithm {} // namespace algorithm +} // namespace nebula diff --git a/src/common/algorithm/ReservoirSampling.h b/src/common/algorithm/ReservoirSampling.h new file mode 100644 index 0000000..7ad90ed --- /dev/null +++ b/src/common/algorithm/ReservoirSampling.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_ALGORITHM_RESERVOIR_H_ +#define COMMON_ALGORITHM_RESERVOIR_H_ + +#include "common/base/Base.h" +#include "common/time/WallClock.h" + +namespace nebula { +namespace algorithm { +template +class ReservoirSampling final { + public: + explicit ReservoirSampling(uint64_t num) { + num_ = num; + samples_.reserve(num); + } + + explicit ReservoirSampling(uint64_t num, uint64_t count) { + num_ = num; + samples_.reserve(num); + for (uint64_t i = 0; i < count; ++i) { + sampling(i); + } + } + + bool sampling(T&& sample) { + if (cnt_ < num_) { + samples_.emplace_back(std::move(sample)); + ++cnt_; + return true; + } else { + auto index = folly::Random::rand64(cnt_); + if (index < num_) { + samples_[index] = (std::move(sample)); + ++cnt_; + return true; + } + } + ++cnt_; + return false; + } + + std::vector samples() { + auto result = std::move(samples_); + samples_.clear(); + samples_.reserve(num_); + cnt_ = 0; + return result; + } + + private: + std::vector samples_; + uint64_t cnt_{0}; + uint64_t num_{0}; +}; +} // namespace algorithm +} // namespace nebula +#endif diff --git a/src/common/algorithm/test/CMakeLists.txt b/src/common/algorithm/test/CMakeLists.txt new file mode 100644 index 0000000..ea38a32 --- /dev/null +++ b/src/common/algorithm/test/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_test( + NAME reservoir_sampling_test + SOURCES ReservoirSamplingTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) diff --git a/src/common/algorithm/test/ReservoirSamplingTest.cpp b/src/common/algorithm/test/ReservoirSamplingTest.cpp new file mode 100644 index 0000000..0486b47 --- /dev/null +++ b/src/common/algorithm/test/ReservoirSamplingTest.cpp @@ -0,0 +1,44 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/algorithm/ReservoirSampling.h" + +namespace nebula { +namespace algorithm { +TEST(ReservoirSamplingTest, Sample) { + { + ReservoirSampling sampler(5); + std::vector sampleSpace = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + for (auto i : sampleSpace) { + sampler.sampling(std::move(i)); + } + + auto result = sampler.samples(); + EXPECT_EQ(5, result.size()); + for (auto i : result) { + EXPECT_LE(0, i); + EXPECT_GE(9, i); + } + } + { + ReservoirSampling sampler(5); + for (size_t count = 0; count < 10; count++) { + std::vector sampleSpace = {0, 1, 2}; + for (auto i : sampleSpace) { + sampler.sampling(std::move(i)); + } + + auto result = sampler.samples(); + EXPECT_EQ(3, result.size()); + EXPECT_EQ(0, result[0]); + EXPECT_EQ(1, result[1]); + EXPECT_EQ(2, result[2]); + } + } +} +} // namespace algorithm +} // namespace nebula diff --git a/src/common/base/Arena.cpp b/src/common/base/Arena.cpp new file mode 100644 index 0000000..0775438 --- /dev/null +++ b/src/common/base/Arena.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2022 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. + +#include "common/base/Arena.h" + +#include + +namespace nebula { + +void* Arena::allocateAligned(const std::size_t alloc) { + DCHECK_NE(alloc, 0); // don't allow zero sized allocation + // replace the modulo operation by bit and + static_assert(kAlignment && !(kAlignment & (kAlignment - 1)), "Align must be power of 2."); + const std::size_t pad = + kAlignment - (reinterpret_cast(currentPtr_) & (kAlignment - 1)); + const std::size_t consumption = alloc + pad; + if (UNLIKELY(consumption > kMaxChunkSize)) { + DLOG(FATAL) << "Arena can't allocate so large memory."; + return nullptr; + } + if (LIKELY(consumption <= availableSize_)) { + void* ptr = currentPtr_ + pad; + currentPtr_ += consumption; +#ifndef NDEBUG + allocatedSize_ += consumption; +#endif + availableSize_ -= consumption; + return ptr; + } else { + newChunk(std::max(alloc, kMinChunkSize)); + // The new operator will allocate the aligned memory + DCHECK_EQ(reinterpret_cast(currentPtr_) & (kAlignment - 1), 0); + void* ptr = currentPtr_; + currentPtr_ += alloc; +#ifndef NDEBUG + allocatedSize_ += alloc; +#endif + availableSize_ -= alloc; + return ptr; + } +} + +} // namespace nebula diff --git a/src/common/base/Arena.h b/src/common/base/Arena.h new file mode 100644 index 0000000..13e4abe --- /dev/null +++ b/src/common/base/Arena.h @@ -0,0 +1,90 @@ +// Copyright (c) 2022 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/base/Logging.h" +#include "common/cpp/helpers.h" + +namespace nebula { + +// MT-unsafe arena allocator +// It's optimized for many small objects construct/destruct +class Arena : public boost::noncopyable, cpp::NonMovable { + public: + ~Arena() { + while (LIKELY(currentChunk_ != nullptr)) { + auto *prev = currentChunk_->prev; + delete[] currentChunk_; + currentChunk_ = prev; + } +#ifndef NDEBUG + allocatedSize_ = 0; +#endif + availableSize_ = 0; + currentPtr_ = nullptr; + } + + // The CPU access memory with the alignment, + // So construct object from alignment address will reduce the CPU access count then + // speed up read/write + void *allocateAligned(const std::size_t alloc); + +#ifndef NDEBUG + std::size_t allocatedSize() const { + return allocatedSize_; + } +#endif + + std::size_t availableSize() const { + return availableSize_; + } + + private: + static constexpr std::size_t kMinChunkSize = 4096; + static constexpr std::size_t kMaxChunkSize = std::numeric_limits::max(); + static constexpr std::size_t kAlignment = std::alignment_of::value; + + struct Chunk { + explicit Chunk(Chunk *p) : prev{p} {} + + union { + Chunk *prev{nullptr}; + std::byte aligned[kAlignment]; + }; + }; + + // allocate new chunk + // The current pointer will keep alignment + void newChunk(std::size_t size) { + DCHECK_NE(size, 0); + std::byte *ptr = new std::byte[size + sizeof(Chunk)]; + currentChunk_ = new (ptr) Chunk(currentChunk_); + availableSize_ = size; + currentPtr_ = (ptr + sizeof(Chunk)); + } + + Chunk *currentChunk_{nullptr}; +// These are debug info +// Remove to speed up in Release build +#ifndef NDEBUG + // total size allocated + std::size_t allocatedSize_{0}; +#endif + // total size which available to allocate + std::size_t availableSize_{0}; + // The total chunks size + // = allocatedSize_ + availableSize_ + Memory Deprecated (Size can't fit allocation) + // Current pointer to available memory address + std::byte *currentPtr_{nullptr}; +}; + +} // namespace nebula diff --git a/src/common/base/Base.cpp b/src/common/base/Base.cpp new file mode 100644 index 0000000..4afd569 --- /dev/null +++ b/src/common/base/Base.cpp @@ -0,0 +1,55 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/base/Base.h" + +namespace nebula { + +std::string toHexStr(folly::StringPiece str) { + static const char* hex[] = { + "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", + "0A", "0B", "0C", "0D", "0E", "0F", "10", "11", "12", "13", + "14", "15", "16", "17", "18", "19", "1A", "1B", "1C", "1D", + "1E", "1F", "20( )", "21(!)", "22(\")", "23(#)", "24($)", "25(%)", "26(&)", "27(')", + "28(()", "29())", "2A(*)", "2B(+)", "2C(,)", "2D(-)", "2E(.)", "2F(/)", "30(0)", "31(1)", + "32(2)", "33(3)", "34(4)", "35(5)", "36(6)", "37(7)", "38(8)", "39(9)", "3A(:)", "3B(;)", + "3C(<)", "3D(=)", "3E(>)", "3F(?)", "40(@)", "41(A)", "42(B)", "43(C)", "44(D)", "45(E)", + "46(F)", "47(G)", "48(H)", "49(I)", "4A(J)", "4B(K)", "4C(L)", "4D(M)", "4E(N)", "4F(O)", + "50(P)", "51(Q)", "52(R)", "53(S)", "54(T)", "55(U)", "56(V)", "57(W)", "58(X)", "59(Y)", + "5A(Z)", "5B([)", "5C(\\)", "5D(])", "5E(^)", "5F(_)", "60(`)", "61(a)", "62(b)", "63(c)", + "64(d)", "65(e)", "66(f)", "67(g)", "68(h)", "69(i)", "6A(j)", "6B(k)", "6C(l)", "6D(m)", + "6E(n)", "6F(o)", "70(p)", "71(q)", "72(r)", "73(s)", "74(t)", "75(u)", "76(v)", "77(w)", + "78(x)", "79(y)", "7A(z)", "7B({)", "7C(|)", "7D(})", "7E(~)", "7F", "80", "81", + "82", "83", "84", "85", "86", "87", "88", "89", "8A", "8B", + "8C", "8D", "8E", "8F", "90", "91", "92", "93", "94", "95", + "96", "97", "98", "99", "9A", "9B", "9C", "9D", "9E", "9F", + "A0", "A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "A9", + "AA", "AB", "AC", "AD", "AE", "AF", "B0", "B1", "B2", "B3", + "B4", "B5", "B6", "B7", "B8", "B9", "BA", "BB", "BC", "BD", + "BE", "BF", "C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", + "C8", "C9", "CA", "CB", "CC", "CD", "CE", "CF", "D0", "D1", + "D2", "D3", "D4", "D5", "D6", "D7", "D8", "D9", "DA", "DB", + "DC", "DD", "DE", "DF", "E0", "E1", "E2", "E3", "E4", "E5", + "E6", "E7", "E8", "E9", "EA", "EB", "EC", "ED", "EE", "EF", + "F0", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", + "FA", "FB", "FC", "FD", "FE", "FF"}; + + if (str.empty()) { + return std::string(); + } + + std::string buf; + buf.reserve(str.size() * 3 - 1); + + buf.append(hex[static_cast(str[0])]); + for (size_t i = 1; i < str.size(); i++) { + buf.append(" "); + buf.append(hex[static_cast(str[i])]); + } + + return buf; +} + +} // namespace nebula diff --git a/src/common/base/Base.h b/src/common/base/Base.h new file mode 100644 index 0000000..35a72dc --- /dev/null +++ b/src/common/base/Base.h @@ -0,0 +1,167 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_BASE_H_ +#define COMMON_BASE_BASE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__clang__) && defined(__aarch64__) +#if FOLLY_HAVE_EXTRANDOM_SFMT19937 +#undef FOLLY_HAVE_EXTRANDOM_SFMT19937 +#endif +#endif // __clang__ && __aarch64__ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/base/Logging.h" + +#define NG_MUST_USE_RESULT __attribute__((warn_unused_result)) +#define DONT_OPTIMIZE __attribute__((optimize("O0"))) + +#define ALWAYS_INLINE __attribute__((always_inline)) +#define ALWAYS_NO_INLINE __attribute__((noinline)) + +#define BEGIN_NO_OPTIMIZATION _Pragma("GCC push_options") _Pragma("GCC optimize(\"O0\")") +#define END_NO_OPTIMIZATION _Pragma("GCC pop_options") + +#define NEBULA_STRINGIFY(STR) NEBULA_STRINGIFY_X(STR) +#define NEBULA_STRINGIFY_X(STR) #STR + +#ifndef UNUSED +#define UNUSED(x) (void)(x) +#endif // UNUSED + +#ifndef MAYBE_UNUSED +#if (__cplusplus >= 201703L) // c++17 +#include +#define MAYBE_UNUSED FOLLY_MAYBE_UNUSED +#else +#define MAYBE_UNUSED __attribute__((unused)) +#endif +#endif + +#ifndef COMPILER_BARRIER +#define COMPILER_BARRIER() asm volatile("" ::: "memory") +#endif // COMPILER_BARRIER + +// Formatted logging +#define FLOG_FATAL(...) LOG(FATAL) << folly::stringPrintf(__VA_ARGS__) +#define FLOG_ERROR(...) LOG(ERROR) << folly::stringPrintf(__VA_ARGS__) +#define FLOG_WARN(...) LOG(WARNING) << folly::stringPrintf(__VA_ARGS__) +#define FLOG_INFO(...) LOG(INFO) << folly::stringPrintf(__VA_ARGS__) +#define FVLOG1(...) VLOG(1) << folly::stringPrintf(__VA_ARGS__) +#define FVLOG2(...) VLOG(2) << folly::stringPrintf(__VA_ARGS__) +#define FVLOG3(...) VLOG(3) << folly::stringPrintf(__VA_ARGS__) +#define FVLOG4(...) VLOG(4) << folly::stringPrintf(__VA_ARGS__) + +namespace nebula { + +using VariantType = std::variant; + +#ifndef VAR_INT64 +#define VAR_INT64 0 +#endif + +#ifndef VAR_DOUBLE +#define VAR_DOUBLE 1 +#endif + +#ifndef VAR_BOOL +#define VAR_BOOL 2 +#endif + +#ifndef VAR_STR +#define VAR_STR 3 +#endif + +// reserved property names +constexpr char kId[] = "_id"; +constexpr char kVid[] = "_vid"; +constexpr char kTag[] = "_tag"; +constexpr char kSrc[] = "_src"; +constexpr char kType[] = "_type"; +constexpr char kRank[] = "_rank"; +constexpr char kDst[] = "_dst"; +constexpr char kEdgePrefix[] = "_edge"; +constexpr char kStatsPrefix[] = "_stats"; +constexpr char kExprPrefix[] = "_expr"; + +// Useful type traits + +// Tell if `T' is copy-constructible +template +static constexpr auto is_copy_constructible_v = std::is_copy_constructible::value; + +// Tell if `T' is move-constructible +template +static constexpr auto is_move_constructible_v = std::is_move_constructible::value; + +// Tell if `T' is copy or move constructible +template +static constexpr auto is_copy_or_move_constructible_v = + is_copy_constructible_v || is_move_constructible_v; + +// Tell if `T' is constructible from `Args' +template +static constexpr auto is_constructible_v = std::is_constructible::value; + +// Tell if `U' could be convertible to `T' +template +static constexpr auto is_convertible_v = std::is_constructible::value; + +std::string toHexStr(folly::StringPiece str); + +} // namespace nebula + +#endif // COMMON_BASE_BASE_H_ diff --git a/src/common/base/CMakeLists.txt b/src/common/base/CMakeLists.txt new file mode 100644 index 0000000..95539f0 --- /dev/null +++ b/src/common/base/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +if (ENABLE_GDB_SCRIPT_SECTION) + set(gdb_debug_script GdbDebugScript.cpp) +endif() + +nebula_add_library( + base_obj OBJECT + Base.cpp + Cord.cpp + Status.cpp + SanitizerOptions.cpp + SignalHandler.cpp + Arena.cpp + ${gdb_debug_script} +) + +nebula_add_subdirectory(test) diff --git a/src/common/base/CheckPointer.h b/src/common/base/CheckPointer.h new file mode 100644 index 0000000..dcb2564 --- /dev/null +++ b/src/common/base/CheckPointer.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_CHECKPOINTER_H_ +#define COMMON_BASE_CHECKPOINTER_H_ + +namespace nebula { + +template +bool inline checkPointer(const T *lhs, const T *rhs) { + if (lhs == rhs) { + return true; + } else if (lhs != nullptr && rhs != nullptr) { + return *lhs == *rhs; + } else { + return false; + } +} + +template class DeRefAble> +bool inline checkPointer(const DeRefAble &lhs, const DeRefAble &rhs) { + if (lhs == rhs) { + return true; + } else if (lhs != nullptr && rhs != nullptr) { + return *lhs == *rhs; + } else { + return false; + } +} + +} // namespace nebula + +#endif // COMMON_BASE_CHECKPOINTER_H_ diff --git a/src/common/base/CollectNSucceeded-inl.h b/src/common/base/CollectNSucceeded-inl.h new file mode 100644 index 0000000..df20bcd --- /dev/null +++ b/src/common/base/CollectNSucceeded-inl.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_COLLECTNSUCCEEDED_INL_H +#define COMMON_BASE_COLLECTNSUCCEEDED_INL_H + +#include + +namespace nebula { + +// Not thread-safe, all futures need to be on the same executor +template +folly::Future> collectNSucceeded(FutureIter first, + FutureIter last, + size_t n, + ResultEval&& eval) { + using Result = SucceededResultList; + if (n == 0) { + return folly::Future(Result()); + } + + struct Context { + Context(size_t total, ResultEval&& e) : eval(std::forward(e)), nTotal(total) {} + + ResultEval eval; + Result results; + std::atomic numCompleted = {0}; + std::atomic nSucceeded = {0}; + folly::Promise promise; + size_t nTotal; + }; + + size_t total = size_t(std::distance(first, last)); + DCHECK_GE(total, 0U); + + if (total < n) { + return folly::Future( + folly::exception_wrapper(std::runtime_error("Not enough futures"))); + } + + auto ctx = std::make_shared(total, std::forward(eval)); + + // for each succeeded Future, add to the result list, until + // we have required number of futures, at which point we fulfil + // the promise with the result list + for (size_t index = 0; first != last; ++first, ++index) { + first->setCallback_([n, ctx, index](auto, folly::Try>&& t) { + if (!ctx->promise.isFulfilled()) { + if (!t.hasException()) { + if (ctx->eval(index, t.value())) { + ++ctx->nSucceeded; + } + ctx->results.emplace_back(index, std::move(t.value())); + } + if ((++ctx->numCompleted) == ctx->nTotal || ctx->nSucceeded == n) { + // Done + VLOG(2) << "Set Value [completed=" << ctx->numCompleted << ", total=" << ctx->nTotal + << ", Result list size=" << ctx->results.size() << "]"; + ctx->promise.setValue(std::move(ctx->results)); + } + } + }); + } + + return ctx->promise.getFuture(); +} + +} // namespace nebula +#endif diff --git a/src/common/base/CollectNSucceeded.h b/src/common/base/CollectNSucceeded.h new file mode 100644 index 0000000..cf75216 --- /dev/null +++ b/src/common/base/CollectNSucceeded.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_COLLECTNSUCCEEDED_H_ +#define COMMON_BASE_COLLECTNSUCCEEDED_H_ + +#include + +#include "common/base/Base.h" + +namespace nebula { + +/********************************************************************* + * Here is a future aggregator, along with its variation + * + * It is based on folly future libraries. Please read + * folly/futures/README.md for information about folly futures + * + * The aggregator uses an evaluator to evaluate the future results. + * The evaluator can be a lambda function, or std::function, or any + * class/struct which overloads operator(). The evaluator takes two + * parameters. + * + * operator() (size_t idx, T& value) + * + * The first parameter ##idx## is the position of the future being + * evaluated in the future list. The second parameter is the value + * of the future + * + * Only the future having value will be checked by the evaluator. + * Futures having exceptions will be considered as failure + * + * The aggregator is **NOT** thread-safe + * + * The aggregator returns a future with an array of pairs + * . The first element of the pair is the index to the + * given futures, and the second is the value of the succeeded + * future. + * + * The returned future fulfils when the given number of futures + * succeed, or when all given futures fulfil. In either case, an + * array of succeeded values will be returned + * + ********************************************************************/ + +template +using FutureReturnType = typename std::iterator_traits::value_type::value_type; + +// The result pair list. +// The first element in pair represents the index in Requests list. +template +using SucceededResultList = std::vector>>; + +template +folly::Future> collectNSucceeded( + FutureIter first, + FutureIter last, + size_t n, // NUmber of succeeded futures required + ResultEval&& eval); + +// A convenient form of ##collectNSucceeded## +template +auto collectNSucceeded(Collection&& c, size_t n, ResultEval&& eval) + -> decltype(collectNSucceeded(c.begin(), c.end(), n, eval)) { + return collectNSucceeded(c.begin(), c.end(), n, std::forward(eval)); +} + +} // namespace nebula + +#include "common/base/CollectNSucceeded-inl.h" + +#endif // COMMON_BASE_COLLECTNSUCCEEDED_H_ diff --git a/src/common/base/CommonMacro.h b/src/common/base/CommonMacro.h new file mode 100644 index 0000000..2af8ad8 --- /dev/null +++ b/src/common/base/CommonMacro.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_MACOR_H_ +#define COMMON_BASE_MACOR_H_ + +namespace nebula { + +// for fixed_string data type, maximum of 256 bytes are allowed in the index. +// Full text index is similar. +#ifndef MAX_INDEX_TYPE_LENGTH +#define MAX_INDEX_TYPE_LENGTH 256 +#endif + +} // namespace nebula + +#endif // COMMON_BASE_MACOR_H_ diff --git a/src/common/base/ConcurrentLRUCache.h b/src/common/base/ConcurrentLRUCache.h new file mode 100644 index 0000000..f2df211 --- /dev/null +++ b/src/common/base/ConcurrentLRUCache.h @@ -0,0 +1,295 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_CONCURRENTLRUCACHE_H_ +#define COMMON_BASE_CONCURRENTLRUCACHE_H_ + +#include + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/base/StatusOr.h" + +namespace nebula { + +template +class LRU; + +template +class ConcurrentLRUCache final { + FRIEND_TEST(ConcurrentLRUCacheTest, SimpleTest); + + public: + explicit ConcurrentLRUCache(size_t capacity, uint32_t bucketsExp = 4) + : bucketsNum_(1 << bucketsExp), bucketsExp_(bucketsExp) { + CHECK(capacity > bucketsNum_ && bucketsNum_ > 0); + auto capPerBucket = capacity >> bucketsExp; + auto left = capacity; + for (uint32_t i = 0; i < bucketsNum_ - 1; i++) { + buckets_.emplace_back(capPerBucket); + left -= capPerBucket; + } + CHECK_GT(left, 0); + buckets_.emplace_back(left); + } + + bool contains(const K& key, int32_t hint = -1) { + return buckets_[bucketIndex(key, hint)].contains(key); + } + + void insert(K key, V val, int32_t hint = -1) { + buckets_[bucketIndex(key, hint)].insert(std::move(key), std::move(val)); + } + + StatusOr get(const K& key, int32_t hint = -1) { + return buckets_[bucketIndex(key, hint)].get(key); + } + + /** + * Insert the {key, val} if key not existed, and return Status::Inserted. + * Otherwise, just return the value for the existed key. + * */ + StatusOr putIfAbsent(K key, V val, int32_t hint = -1) { + return buckets_[bucketIndex(key, hint)].putIfAbsent(std::move(key), std::move(val)); + } + + void evict(const K& key, int32_t hint = -1) { + buckets_[bucketIndex(key, hint)].evict(key); + } + + void clear() { + for (uint32_t i = 0; i < bucketsNum_; i++) { + buckets_[i].clear(); + } + } + + uint64_t total() { + uint64_t total = 0; + for (uint32_t i = 0; i < bucketsNum_; i++) { + total += buckets_[i].lru_->total(); + } + return total; + } + + uint64_t hits() { + uint64_t hits = 0; + for (uint32_t i = 0; i < bucketsNum_; i++) { + hits += buckets_[i].lru_->hits(); + } + return hits; + } + + uint64_t evicts() { + uint64_t evicts = 0; + for (uint32_t i = 0; i < bucketsNum_; i++) { + evicts += buckets_[i].lru_->evicts(); + } + return evicts; + } + + private: + class Bucket { + public: + explicit Bucket(size_t capacity) : lru_(std::make_unique>(capacity)) {} + + Bucket(Bucket&& b) : lru_(std::move(b.lru_)) {} + + bool contains(const K& key) { + std::lock_guard guard(lock_); + return lru_->contains(key); + } + + void insert(K&& key, V&& val) { + std::lock_guard guard(lock_); + lru_->insert(std::forward(key), std::forward(val)); + } + + StatusOr get(const K& key) { + std::lock_guard guard(lock_); + auto v = lru_->get(key); + if (v == std::nullopt) { + return Status::Error(); + } + return std::move(v).value(); + } + + StatusOr putIfAbsent(K&& key, V&& val) { + std::lock_guard guard(lock_); + auto v = lru_->get(key); + if (v == std::nullopt) { + lru_->insert(std::forward(key), std::forward(val)); + return Status::Inserted(); + } + return std::move(v).value(); + } + + void evict(const K& key) { + std::lock_guard guard(lock_); + lru_->evict(key); + } + + void clear() { + std::lock_guard guard(lock_); + lru_->clear(); + } + + std::mutex lock_; + std::unique_ptr> lru_; + }; + + private: + /** + * If hint is specified, we could use it to cal the bucket index directly + * without hash key. + * */ + uint32_t bucketIndex(const K& key, int32_t hint = -1) { + return hint >= 0 ? (hint & ((1 << bucketsExp_) - 1)) + : (std::hash()(key) & ((1 << bucketsExp_) - 1)); + } + + private: + std::vector buckets_; + uint32_t bucketsNum_ = 1; + uint32_t bucketsExp_ = 0; +}; + +/** + It is copied from boost::compute::detail::LRU. + The differences: + 1. Add method evict(const K& key); + 2. Instead std::map with std::unordered_map + 3. Update the code style. + 4. Add stats + 5. Avoid some extra copies + 6. Support right reference for insert. +*/ +template +class LRU { + public: + typedef Key key_type; + typedef Value value_type; + typedef std::list list_type; + typedef std::unordered_map> + map_type; + + explicit LRU(size_t capacity) : capacity_(capacity) {} + + ~LRU() = default; + + size_t size() const { + return map_.size(); + } + + size_t capacity() const { + return capacity_; + } + + bool empty() const { + return map_.empty(); + } + + bool contains(const key_type& key) { + return map_.find(key) != map_.end(); + } + + void insert(key_type&& key, value_type&& value) { + typename map_type::iterator it = map_.find(key); + if (it == map_.end()) { + // insert item into the cache, but first check if it is full + if (size() >= capacity_) { + VLOG(3) << "Size:" << size() << ", capacity " << capacity_; + // cache is full, evict the least recently used item + evict(); + } + // insert the new item + list_.push_front(key); + map_.emplace(std::forward(key), + std::make_tuple(std::forward(value), list_.begin())); + } else { + // Overwrite the value + std::get<0>(it->second) = std::move(value); + typename list_type::iterator j = std::get<1>(it->second); + list_.splice(list_.begin(), list_, j); + } + } + + std::optional get(const key_type& key) { + // lookup value in the cache + total_++; + typename map_type::iterator i = map_.find(key); + if (i == map_.end()) { + // value not in cache + return std::nullopt; + } + + // return the value, but first update its place in the most + // recently used list + const value_type& value = std::get<0>(i->second); + typename list_type::iterator j = std::get<1>(i->second); + CHECK(key == *j); + if (j != list_.begin()) { + // move item to the front of the most recently used list + list_.splice(list_.begin(), list_, j); + } + hits_++; + return value; + } + + /** + * evict the key if exist. + * */ + void evict(const key_type& key) { + auto it = map_.find(key); + if (it != map_.end()) { + list_.erase(std::get<1>(it->second)); + map_.erase(it); + evicts_++; + } + } + + void clear() { + map_.clear(); + list_.clear(); + total_ = 0; + hits_ = 0; + evicts_ = 0; + } + + uint64_t total() { + return total_; + } + + uint64_t hits() { + return hits_; + } + + uint64_t evicts() { + return evicts_; + } + + private: + void evict() { + // evict item from the end of most recently used list + typename list_type::iterator i = --list_.end(); + map_.erase(*i); + list_.erase(i); + evicts_++; + } + + private: + map_type map_; + list_type list_; + size_t capacity_; + uint64_t total_{0}; + uint64_t hits_{0}; + uint64_t evicts_{0}; +}; + +} // namespace nebula + +#endif // COMMON_BASE_CONCURRENTLRUCACHE_H_ diff --git a/src/common/base/Cord.cpp b/src/common/base/Cord.cpp new file mode 100644 index 0000000..4ecf154 --- /dev/null +++ b/src/common/base/Cord.cpp @@ -0,0 +1,210 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/base/Cord.h" + +#include "common/base/Logging.h" + +namespace nebula { + +Cord::Cord(int32_t blockSize) + : blockSize_(blockSize), + blockContentSize_(blockSize_ - sizeof(char*)), + blockPt_(blockContentSize_) {} + +Cord::~Cord() { + clear(); +} + +void Cord::allocateBlock() { + DCHECK_EQ(blockPt_, blockContentSize_); + char* blk = reinterpret_cast(malloc(blockSize_ * sizeof(char))); + CHECK(blk) << "Out of memory"; + + if (tail_) { + // Link the tail to the new block + memcpy(tail_ + blockPt_, reinterpret_cast(&blk), sizeof(char*)); + } + tail_ = blk; + blockPt_ = 0; + + if (!head_) { + head_ = blk; + } +} + +size_t Cord::size() const noexcept { + return len_; +} + +bool Cord::empty() const noexcept { + return len_ == 0; +} + +void Cord::clear() { + if (head_) { + DCHECK(tail_); + + // Need to release all blocks + char* p = head_; + while (p != tail_) { + char* next; + memcpy(reinterpret_cast(&next), p + blockContentSize_, sizeof(char*)); + free(p); + p = next; + } + // Free the last block + free(p); + } + + blockPt_ = blockContentSize_; + len_ = 0; + + head_ = nullptr; + tail_ = nullptr; +} + +bool Cord::applyTo(std::function visitor) const { + if (empty()) { + return true; + } + + char* next = head_; + while (next != tail_) { + if (!visitor(next, blockContentSize_)) { + // stop visiting further + return false; + } + // Get the pointer to the next block + memcpy(reinterpret_cast(&next), next + blockContentSize_, sizeof(char*)); + } + + // Last block + return visitor(tail_, blockPt_); +} + +size_t Cord::appendTo(std::string& str) const { + if (empty()) { + return 0; + } + + char* next = head_; + while (next != tail_) { + str.append(next, blockContentSize_); + // Get the pointer to the next block + memcpy(reinterpret_cast(&next), next + blockContentSize_, sizeof(char*)); + } + + // Last block + str.append(tail_, blockPt_); + + return len_; +} + +std::string Cord::str() const { + std::string buf; + buf.reserve(len_); + appendTo(buf); + + return buf; +} + +Cord& Cord::write(const char* value, size_t len) { + if (len == 0) { + return *this; + } + + size_t bytesToWrite = std::min(len, static_cast(blockContentSize_ - blockPt_)); + if (bytesToWrite == 0) { + allocateBlock(); + bytesToWrite = std::min(len, static_cast(blockContentSize_)); + } + memcpy(tail_ + blockPt_, value, bytesToWrite); + blockPt_ += bytesToWrite; + len_ += bytesToWrite; + + if (bytesToWrite < len) { + return write(value + bytesToWrite, len - bytesToWrite); + } else { + return *this; + } +} + +/********************** + * + * Stream operator + * + *********************/ +Cord& Cord::operator<<(int8_t value) { + return write(reinterpret_cast(&value), sizeof(int8_t)); +} + +Cord& Cord::operator<<(uint8_t value) { + return write(reinterpret_cast(&value), sizeof(uint8_t)); +} + +Cord& Cord::operator<<(int16_t value) { + return write(reinterpret_cast(&value), sizeof(int16_t)); +} + +Cord& Cord::operator<<(uint16_t value) { + return write(reinterpret_cast(&value), sizeof(uint16_t)); +} + +Cord& Cord::operator<<(int32_t value) { + return write(reinterpret_cast(&value), sizeof(int32_t)); +} + +Cord& Cord::operator<<(uint32_t value) { + return write(reinterpret_cast(&value), sizeof(uint32_t)); +} + +Cord& Cord::operator<<(int64_t value) { + return write(reinterpret_cast(&value), sizeof(int64_t)); +} + +Cord& Cord::operator<<(uint64_t value) { + return write(reinterpret_cast(&value), sizeof(uint64_t)); +} + +Cord& Cord::operator<<(char value) { + return write(&value, sizeof(char)); +} + +Cord& Cord::operator<<(bool value) { + return write(reinterpret_cast(&value), sizeof(bool)); +} + +Cord& Cord::operator<<(float value) { + return write(reinterpret_cast(&value), sizeof(float)); +} + +Cord& Cord::operator<<(double value) { + return write(reinterpret_cast(&value), sizeof(double)); +} + +Cord& Cord::operator<<(const std::string& value) { + return write(value.data(), value.size()); +} + +Cord& Cord::operator<<(const char* value) { + return write(value, strlen(value)); +} + +Cord& Cord::operator<<(const Cord& rhs) { + char* next = rhs.head_; + while (next != rhs.tail_) { + write(next, blockContentSize_); + // Get the pointer to the next block + memcpy(reinterpret_cast(&next), next + blockContentSize_, sizeof(char*)); + } + + // Last block + write(rhs.tail_, rhs.blockPt_); + + return *this; +} + +} // namespace nebula diff --git a/src/common/base/Cord.h b/src/common/base/Cord.h new file mode 100644 index 0000000..6a990bf --- /dev/null +++ b/src/common/base/Cord.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_CORD_H_ +#define COMMON_BASE_CORD_H_ + +#include +#include +#include +#include + +namespace nebula { + +class Cord { + public: + Cord() = default; + explicit Cord(int32_t blockSize); + virtual ~Cord(); + + size_t size() const noexcept; + bool empty() const noexcept; + + // Apply each block to the visitor until the end or the visitor + // returns false + bool applyTo(std::function visitor) const; + + // Append the cord content to the given string + size_t appendTo(std::string& str) const; + // Convert the cord content to a new string + std::string str() const; + + void clear(); + + Cord& write(const char* value, size_t len); + + Cord& operator<<(int8_t value); + Cord& operator<<(uint8_t value); + + Cord& operator<<(int16_t value); + Cord& operator<<(uint16_t value); + + Cord& operator<<(int32_t value); + Cord& operator<<(uint32_t value); + + Cord& operator<<(int64_t value); + Cord& operator<<(uint64_t value); + + Cord& operator<<(char value); + Cord& operator<<(bool value); + + Cord& operator<<(float value); + Cord& operator<<(double value); + + Cord& operator<<(const std::string& value); + Cord& operator<<(const char* value); + + Cord& operator<<(const Cord& rhs); + + private: + const int32_t blockSize_ = 1024; + const int32_t blockContentSize_ = 1024 - sizeof(char*); + int32_t blockPt_ = blockContentSize_; + size_t len_ = 0; + + char* head_ = nullptr; + char* tail_ = nullptr; + + void allocateBlock(); +}; + +} // namespace nebula +#endif // COMMON_BASE_CORD_H_ diff --git a/src/common/base/EitherOr.h b/src/common/base/EitherOr.h new file mode 100644 index 0000000..d832196 --- /dev/null +++ b/src/common/base/EitherOr.h @@ -0,0 +1,477 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_EITHEROR_H_ +#define COMMON_BASE_EITHEROR_H_ + +#include + +#include "common/base/Base.h" + +namespace nebula { + +using LeftType = std::true_type; +using RightType = std::false_type; + +enum class State : int16_t { + VOID = 0, + LEFT_TYPE = 1, + RIGHT_TYPE = 2, +}; + +static constexpr LeftType* kConstructLeft = nullptr; +static constexpr RightType* kConstructRight = nullptr; + +/** + * EitherOr<> is a type to hold a value of either LEFT type or RIGHT type. + * The only constrain is the LEFT type cannot be same as the RIGHT type + * + * vs. boost::variant<>: Variant is an old design which tries to solve the + * problem that only POD can be allowed in `union` + * (https://www.boost.org/doc/libs/1_69_0/doc/html/variant.html#variant.abstract). + * The problem boost:::variant tries to solve does not exist any more since + * C++11 (https://en.cppreference.com/w/cpp/language/union). EitherOr is + * designed based on the new `union` in C++11, so it means they are more + * efficient + * + * vs. StatusOr<>: EitherOr is a more generic form of StatusOr. EitherOr + * supports any two types, as long as the two types are different. StatusOr + * supports Status and another type. So StatusOr is a special case of EitherOr. + * Later we would rewrite StatusOr based on EitherOr + * + * EitherOr<> provides multiple constructors. Besides the default constructor + * and copy constructors, EitherOr<> will use the parameters to decide which + * type should be instantiated. If the LEFT and RIGHT types are similar enough + * and EitherOr<> cannot decide which type to instantiate, we need to use the + * constructors with the type Tag. For example + * + * EitherOr v1(kConstructLeft, 12); + * EXPECT_TRUE(v1.isLeftType()); + * + * EitherOr v1(kConstructRight, 12); + * EXPECT_TRUE(v1.isRightType()); + */ +template +class EitherOr { + private: + // Make friends with other compatible EitherOr + template + friend class EitherOr; + + static_assert(!std::is_same::value, + "The left type and right type of EitherOr<> cannot be same"); + + template + struct TypeConverter { + TypeConverter() = delete; + + template + static constexpr typename std::enable_if_t::value && + !std::is_constructible::value, + LeftType>* + sfinae() { + return kConstructLeft; + } + + template + static constexpr typename std::enable_if_t::value && + std::is_constructible::value, + RightType>* + sfinae() { + return kConstructRight; + } + + template + static constexpr typename std::enable_if_t::value && + std::is_constructible::value>* + sfinae() { + static_assert(std::is_constructible::value && + std::is_constructible::value, + "The arguments can be converted into either" + " LEFT or RIGHT, so please use the constructor" + " with an explicit tag"); + } + + static constexpr State getState() { + auto* ptr = sfinae(); + if (std::is_same::value) { + return State::LEFT_TYPE; + } + if (std::is_same::value) { + return State::RIGHT_TYPE; + } + return State::VOID; + } + + static constexpr auto* type = sfinae(); + static constexpr State state = getState(); + }; + + template + static constexpr decltype(TypeConverter::type) convert_to_t{}; + + template + static constexpr State convert_to_s = TypeConverter::state; + + public: + virtual ~EitherOr() { + destruct(); + } + + /*********************************************** + * + * Constructors + * + **********************************************/ + EitherOr() noexcept {}; + + EitherOr(const EitherOr& rhs) noexcept { + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, rhs.val_.left_); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, rhs.val_.right_); + state_ = State::RIGHT_TYPE; + break; + } + } + + EitherOr(EitherOr&& rhs) noexcept { + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, std::move(rhs).left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, std::move(rhs).right()); + state_ = State::RIGHT_TYPE; + break; + } + } + + template ::value && + std::is_constructible::value>> + EitherOr(const EitherOr& rhs) noexcept { + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, rhs.val_.left_); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, rhs.val_.right_); + state_ = State::RIGHT_TYPE; + break; + } + } + + template ::value && + std::is_constructible::value>> + EitherOr(EitherOr&& rhs) noexcept { + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, std::move(rhs).left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, std::move(rhs).right()); + state_ = State::RIGHT_TYPE; + break; + } + } + + EitherOr(const LEFT& v) noexcept { // NOLINT + new (&val_) Variant(kConstructLeft, v); + state_ = State::LEFT_TYPE; + } + + EitherOr(LEFT&& v) noexcept { // NOLINT + new (&val_) Variant(kConstructLeft, std::move(v)); + state_ = State::LEFT_TYPE; + } + + EitherOr(const RIGHT& v) noexcept { // NOLINT + new (&val_) Variant(kConstructRight, v); + state_ = State::RIGHT_TYPE; + } + + EitherOr(RIGHT&& v) noexcept { // NOLINT + new (&val_) Variant(kConstructRight, std::move(v)); + state_ = State::RIGHT_TYPE; + } + + // Construct from a list of values which can only construct either + // LEFT or RIGHT, not both + template ::value || + std::is_constructible::value>> + EitherOr(Args&&... v) noexcept { // NOLINT + new (&val_) Variant(convert_to_t, std::forward(v)...); + state_ = convert_to_s; + } + + // Construct from a value which can construct both LEFT and RIGHT + // So we use a type tag to force selecting LEFT + template ::value && + std::is_constructible::value>> + EitherOr(const LeftType*, U&& v) noexcept { + new (&val_) Variant(kConstructLeft, std::forward(v)); + state_ = State::LEFT_TYPE; + } + + // Construct from a value which can construct both LEFT and RIGHT + // So we use a type tag to force selecting RIGHT + template ::value && + std::is_constructible::value>> + EitherOr(const RightType*, U&& v) noexcept { + new (&val_) Variant(kConstructRight, std::forward(v)); + state_ = State::RIGHT_TYPE; + } + + /*********************************************** + * + * Assignments + * + **********************************************/ + EitherOr& operator=(const LEFT& v) noexcept { + reset(); + new (&val_) Variant(kConstructLeft, v); + state_ = State::LEFT_TYPE; + return *this; + } + + EitherOr& operator=(LEFT&& v) noexcept { + reset(); + new (&val_) Variant(kConstructLeft, std::move(v)); + state_ = State::LEFT_TYPE; + return *this; + } + + EitherOr& operator=(const RIGHT& v) noexcept { + reset(); + new (&val_) Variant(kConstructRight, v); + state_ = State::RIGHT_TYPE; + return *this; + } + + EitherOr& operator=(RIGHT&& v) noexcept { + reset(); + new (&val_) Variant(kConstructRight, std::move(v)); + state_ = State::RIGHT_TYPE; + return *this; + } + + // Assign from a value which can only construct either LEFT or RIGHT, + // but not both + template + typename std::enable_if_t::value || + std::is_constructible::value, + EitherOr>& + operator=(U&& v) noexcept { + reset(); + new (&val_) Variant(convert_to_t, std::forward(v)); + state_ = convert_to_s; + return *this; + } + + EitherOr& operator=(const EitherOr& rhs) noexcept { + // Avoid self-assignment + if (&rhs == this) { + return *this; + } + + reset(); + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, rhs.left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, rhs.right()); + state_ = State::RIGHT_TYPE; + break; + } + return *this; + } + + EitherOr& operator=(EitherOr&& rhs) noexcept { + // Avoid self-assignment + if (&rhs == this) { + return *this; + } + + reset(); + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, std::move(rhs).left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, std::move(rhs).right()); + state_ = State::RIGHT_TYPE; + break; + } + return *this; + } + + template + typename std::enable_if_t::value && + std::is_constructible::value, + EitherOr>& + operator=(const EitherOr& rhs) noexcept { + reset(); + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, rhs.left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, rhs.right()); + state_ = State::RIGHT_TYPE; + break; + } + return *this; + } + + template + typename std::enable_if_t::value && + std::is_constructible::value, + EitherOr>& + operator=(EitherOr&& rhs) noexcept { + reset(); + switch (rhs.state_) { + case State::VOID: + break; + case State::LEFT_TYPE: + new (&val_) Variant(kConstructLeft, std::move(rhs).left()); + state_ = State::LEFT_TYPE; + break; + case State::RIGHT_TYPE: + new (&val_) Variant(kConstructRight, std::move(rhs).right()); + state_ = State::RIGHT_TYPE; + break; + } + return *this; + } + + /*********************************************** + * + * State check + * + **********************************************/ + bool isVoid() const { + return state_ == State::VOID; + } + + bool isLeftType() const { + return state_ == State::LEFT_TYPE; + } + + bool isRightType() const { + return state_ == State::RIGHT_TYPE; + } + + /*********************************************** + * + * Access values + * + **********************************************/ + LEFT& left() & { + CHECK(isLeftType()); + return val_.left_; + } + + const LEFT& left() const& { + CHECK(isLeftType()); + return val_.left_; + } + + LEFT left() && { + CHECK(isLeftType()); + auto v = std::move(val_.left_); + val_.left_.~LEFT(); + state_ = State::VOID; + return v; + } + + RIGHT& right() & { + CHECK(isRightType()); + return val_.right_; + } + + const RIGHT& right() const& { + CHECK(isRightType()); + return val_.right_; + } + + RIGHT right() && { + CHECK(isRightType()); + auto v = std::move(val_.right_); + val_.right_.~RIGHT(); + state_ = State::VOID; + return v; + } + + private: + void destruct() { + switch (state_) { + case State::VOID: + return; + case State::LEFT_TYPE: + val_.left_.~LEFT(); + return; + case State::RIGHT_TYPE: + val_.right_.~RIGHT(); + } + } + + void reset() { + destruct(); + state_ = State::VOID; + } + + private: + union Variant { + Variant() {} + + template + Variant(const LeftType*, Args&&... v) : left_(std::forward(v)...) {} + + template + Variant(const RightType*, Args&&... v) : right_(std::forward(v)...) {} + + ~Variant() {} + + LEFT left_; + RIGHT right_; + }; + + Variant val_; + State state_{State::VOID}; +}; + +} // namespace nebula +#endif // COMMON_BASE_EITHEROR_H_ diff --git a/src/common/base/ErrorOr.h b/src/common/base/ErrorOr.h new file mode 100644 index 0000000..d3d63fd --- /dev/null +++ b/src/common/base/ErrorOr.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_ERROROR_H_ +#define COMMON_BASE_ERROROR_H_ + +#include "common/base/Base.h" +#include "common/base/EitherOr.h" + +namespace nebula { + +/** + * ErrorOr<> is a convenient type to allow a method to return either an error + * code or a user defined result + * + * vs. StatusOr<>: ErrorOr<> is an alias of EitherOr with a few accessory + * methods. The left type of the ErrorOr has to be an integral type or enum, + * while the left type of StatusOr<> is a Status, which is a more complicate + * object. This is pretty much the only difference between ErrorOr<> and + * StatusOr<> + * + * So in the scenario that the caller only cares about the error code (or result + * code), ErrorOr<> would be more suitable. If the caller also needs an error + * message along with the error code from the callee, then StatusOr<> should be + * used + */ +template ::value || + std::is_enum::value>> +using ErrorOr = EitherOr; + +/*********************************************** + * + * Accessary methods + * + **********************************************/ +// We treat void ErrorOr objects as succeeded +template +bool ok(const ErrorOr& err) { + return !err.isLeftType(); +} + +template +E error(const ErrorOr& err) { + return err.left(); +} + +template +bool hasValue(const ErrorOr& err) { + return err.isRightType(); +} + +template +R& value(ErrorOr& err) { + return err.right(); +} + +template +const R& value(const ErrorOr& err) { + return err.right(); +} + +template +R value(ErrorOr&& err) { + return std::move(err).right(); +} + +} // namespace nebula +#endif // COMMON_BASE_ERROROR_H_ diff --git a/src/common/base/GdbDebugScript.cpp b/src/common/base/GdbDebugScript.cpp new file mode 100644 index 0000000..1e677e7 --- /dev/null +++ b/src/common/base/GdbDebugScript.cpp @@ -0,0 +1,19 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +asm(".pushsection \".debug_gdb_scripts\", \"MS\",@progbits,1\n" + ".byte 4" + "\n" + ".ascii \"gdb.inlined-script\\n\"\n" + ".ascii \"import glob\\n\"\n" + ".ascii \"path = glob.glob('/usr/share/gcc-*/python')\\n\"\n" + ".ascii \"sys.path = path + sys.path\\n\"\n" + ".ascii \"try:\\n\"\n" + ".ascii \" from libstdcxx.v6 import register_libstdcxx_printers \\n\"\n" + ".ascii \" register_libstdcxx_printers(gdb.current_objfile()) \\n\"\n" + ".ascii \"except ImportError: \\n\"\n" + ".ascii \" print('No pretty printer found for libstdc++') \\n\"\n" + ".byte 0\n" + ".popsection\n"); diff --git a/src/common/base/ICord.h b/src/common/base/ICord.h new file mode 100644 index 0000000..7f97cfe --- /dev/null +++ b/src/common/base/ICord.h @@ -0,0 +1,250 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_ICORD_H +#define COMMON_BASE_ICORD_H + +#include "common/base/Base.h" + +namespace nebula { + +template +class ICord { + public: + static_assert(kBlockContentSize && !(kBlockContentSize & (kBlockContentSize - 1)), + "kBlockContentSize must be power of 2"); + ICord() : head_(inlineBlock_), tail_(head_) {} + + virtual ~ICord() { + clear(); + } + + size_t size() const noexcept { + return len_; + } + + bool empty() const noexcept { + return len_ == 0; + } + + void clear() { + auto alloc = next(head_); + if (alloc) { + DCHECK(tail_); + + // Need to release all blocks + char* p = alloc; + while (p != tail_) { + char* n = next(p); + free(p); + p = n; + } + // Free the last block + free(p); + } + + len_ = 0; + + head_ = nullptr; + tail_ = nullptr; + } + + // Apply each block to the visitor until the end or the visitor + // returns false + bool applyTo(std::function visitor) const { + if (empty()) { + return true; + } + + char* n = head_; + while (n != tail_) { + if (!visitor(n, kBlockContentSize)) { + // stop visiting further + return false; + } + // Get the pointer to the next block + n = next(n); + } + + // Last block + return visitor(tail_, lengthMod()); + } + + // Append the cord content to the given string + size_t appendTo(std::string& str) const { + if (empty()) { + return 0; + } + + char* n = head_; + while (n != tail_) { + str.append(n, kBlockContentSize); + // Get the pointer to the next block + n = next(n); + } + + // Last block + std::size_t lengthModSize = lengthMod(); + str.append(tail_, lengthModSize == 0 ? kBlockContentSize : lengthModSize); + + return len_; + } + + // Convert the cord content to a new string + std::string str() const { + std::string buf; + buf.reserve(len_); + appendTo(buf); + + return buf; + } + + ICord& write(const char* value, size_t len) { + if (len == 0) { + return *this; + } + + std::size_t lengthModSize = lengthMod(); + std::size_t bytesToWrite = + std::min(len, static_cast(kBlockContentSize - lengthModSize)); + if (len_ != 0 && lengthModSize == 0) { // is full filled. + allocateBlock(); + bytesToWrite = std::min(len, static_cast(kBlockContentSize)); + } + memcpy(tail_ + lengthModSize, value, bytesToWrite); + len_ += bytesToWrite; + + if (bytesToWrite < len) { + return write(value + bytesToWrite, len - bytesToWrite); + } else { + return *this; + } + } + + // stream + ICord& operator<<(int8_t value) { + return write(reinterpret_cast(&value), sizeof(int8_t)); + } + + ICord& operator<<(uint8_t value) { + return write(reinterpret_cast(&value), sizeof(uint8_t)); + } + + ICord& operator<<(int16_t value) { + return write(reinterpret_cast(&value), sizeof(int16_t)); + } + + ICord& operator<<(uint16_t value) { + return write(reinterpret_cast(&value), sizeof(uint16_t)); + } + + ICord& operator<<(int32_t value) { + return write(reinterpret_cast(&value), sizeof(int32_t)); + } + + ICord& operator<<(uint32_t value) { + return write(reinterpret_cast(&value), sizeof(uint32_t)); + } + + ICord& operator<<(int64_t value) { + return write(reinterpret_cast(&value), sizeof(int64_t)); + } + + ICord& operator<<(uint64_t value) { + return write(reinterpret_cast(&value), sizeof(uint64_t)); + } + + ICord& operator<<(char value) { + return write(&value, sizeof(char)); + } + + ICord& operator<<(bool value) { + return write(reinterpret_cast(&value), sizeof(bool)); + } + + ICord& operator<<(float value) { + return write(reinterpret_cast(&value), sizeof(float)); + } + + ICord& operator<<(double value) { + return write(reinterpret_cast(&value), sizeof(double)); + } + + ICord& operator<<(const std::string& value) { + return write(value.data(), value.size()); + } + + ICord& operator<<(const char* value) { + std::string _tmp = std::string(value); + return write(_tmp.data(), _tmp.size()); + } + + ICord& operator<<(const ICord& rhs) { + char* n = rhs.head_; + while (n != rhs.tail_) { + write(n, kBlockContentSize); + // Get the pointer to the next block + n = next(n); + } + + // Last block + write(rhs.tail_, rhs.lengthMod()); + + return *this; + } + + private: + // Disable dynamic allocation + void* operator new(std::size_t) = delete; + + // Is the capacity full filled + bool isFull() const { + return len_ != 0 && lengthMod() == 0; + } + + // Used size in last block + std::size_t lengthMod() const { + return len_ & (kBlockContentSize - 1); + } + + // Is there only inline allocation + bool isInline() const { + return len_ < kBlockContentSize; + } + + // return next block pointer + char* next(char* p) const { + if (p == tail_) { + return nullptr; + } else { + return *reinterpret_cast(p + kBlockContentSize); + } + } + + void allocateBlock() { + DCHECK(isFull()); + char* blk = reinterpret_cast(malloc(kBlockSize * sizeof(char))); + + if (tail_) { + // Link the tail to the new block + memcpy(tail_ + kBlockContentSize, reinterpret_cast(&blk), sizeof(char*)); + } + tail_ = blk; + + if (!head_) { + head_ = blk; + } + } + + static constexpr std::size_t kBlockSize = kBlockContentSize + sizeof(char*); + + size_t len_ = 0; + char* head_; + char* tail_; + char inlineBlock_[kBlockSize]; +}; + +} // namespace nebula +#endif diff --git a/src/common/base/Logging.h b/src/common/base/Logging.h new file mode 100644 index 0000000..cf65dfc --- /dev/null +++ b/src/common/base/Logging.h @@ -0,0 +1,199 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_LOGGING_H_ +#define COMMON_BASE_LOGGING_H_ + +#include + +#define _MINLOGLEVEL_GT_INFO (FLAGS_minloglevel > google::GLOG_INFO) + +#define _MINLOGLEVEL_GT_WARNING (FLAGS_minloglevel > google::GLOG_WARNING) + +#define _MINLOGLEVEL_GT_ERROR (FLAGS_minloglevel > google::GLOG_ERROR) + +#define _MINLOGLEVEL_GT_FATAL (FLAGS_minloglevel > google::GLOG_FATAL) + +#ifdef NDEBUG +// Treat DFATAL as ERROR in non-debug mode +#define _MINLOGLEVEL_GT_DFATAL (FLAGS_minloglevel > google::GLOG_ERROR) +#else +// Treat DFATAL as FATAL in debug mode +#define _MINLOGLEVEL_GT_DFATAL (FLAGS_minloglevel > google::GLOG_FATAL) +#endif + +/***************************************** + * + * Normal loggings + * + ****************************************/ +#define _LOG_INFO \ + _MINLOGLEVEL_GT_INFO \ + ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_INFO.stream() + +#define _LOG_WARNING \ + _MINLOGLEVEL_GT_WARNING \ + ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_WARNING.stream() + +#define _LOG_ERROR \ + _MINLOGLEVEL_GT_ERROR \ + ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_ERROR.stream() + +#define _LOG_FATAL COMPACT_GOOGLE_LOG_FATAL.stream() + +#define _LOG_DFATAL \ + _MINLOGLEVEL_GT_DFATAL \ + ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_DFATAL.stream() + +#undef LOG +#define LOG(severity) _LOG_##severity + +#undef PLOG +#define PLOG(severity) \ + _MINLOGLEVEL_GT_##severity ? (void)0 \ + : google::LogMessageVoidify() & GOOGLE_PLOG(severity, 0).stream() + +#undef SYSLOG +#define SYSLOG(severity) \ + _MINLOGLEVEL_GT_##severity ? (void)0 : google::LogMessageVoidify() & SYSLOG_##severity(0).stream() + +#undef LOG_TO_STRING +#define LOG_TO_STRING(severity, message) \ + _MINLOGLEVEL_GT_##severity \ + ? (void)0 \ + : google::LogMessageVoidify() & \ + LOG_TO_STRING_##severity(static_cast(message)).stream() + +#undef LOG_STRING +#define LOG_STRING(severity, outvec) \ + _MINLOGLEVEL_GT_##severity \ + ? (void)0 \ + : google::LogMessageVoidify() & \ + LOG_TO_STRING_##severity(static_cast*>(outvec)).stream() + +/***************************************** + * + * Conditional loggings + * + ****************************************/ +#undef LOG_IF +#define LOG_IF(severity, condition) \ + (!(condition) || _MINLOGLEVEL_GT_##severity) \ + ? (void)0 \ + : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_##severity.stream() + +#undef PLOG_IF +#define PLOG_IF(severity, condition) \ + (!(condition) || _MINLOGLEVEL_GT_##severity) \ + ? (void)0 \ + : google::LogMessageVoidify() & GOOGLE_PLOG(severity, 0).stream() + +#undef SYSLOG_IF +#define SYSLOG_IF(severity, condition) \ + (!(condition) || _MINLOGLEVEL_GT_##severity) \ + ? (void)0 \ + : google::LogMessageVoidify() & SYSLOG_##severity(0).stream() + +/***************************************** + * + * Sampling loggings + * + ****************************************/ +#undef SOME_KIND_OF_LOG_EVERY_N +#define SOME_KIND_OF_LOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (!(_MINLOGLEVEL_GT_##severity) && LOG_OCCURRENCES_MOD_N == 1) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_##severity, LOG_OCCURRENCES, &what_to_do) \ + .stream() + +#undef SOME_KIND_OF_LOG_IF_EVERY_N +#define SOME_KIND_OF_LOG_IF_EVERY_N(severity, condition, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (!(_MINLOGLEVEL_GT_##severity) && (condition) && LOG_OCCURRENCES_MOD_N == 1) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_##severity, LOG_OCCURRENCES, &what_to_do) \ + .stream() + +#undef SOME_KIND_OF_PLOG_EVERY_N +#define SOME_KIND_OF_PLOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (!(_MINLOGLEVEL_GT_##severity) && LOG_OCCURRENCES_MOD_N == 1) \ + google::ErrnoLogMessage( \ + __FILE__, __LINE__, google::GLOG_##severity, LOG_OCCURRENCES, &what_to_do) \ + .stream() + +#undef SOME_KIND_OF_LOG_FIRST_N +#define SOME_KIND_OF_LOG_FIRST_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0; \ + if (LOG_OCCURRENCES <= n) ++LOG_OCCURRENCES; \ + if (!(_MINLOGLEVEL_GT_##severity) && LOG_OCCURRENCES <= n) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_##severity, LOG_OCCURRENCES, &what_to_do) \ + .stream() + +#undef LOG_EVERY_N +#define LOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), google::LogMessage::SendToLog) + +#undef LOG_IF_EVERY_N +#define LOG_IF_EVERY_N(severity, condition, n) \ + SOME_KIND_OF_LOG_IF_EVERY_N(severity, (condition), (n), google::LogMessage::SendToLog) + +#undef PLOG_EVERY_N +#define PLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_PLOG_EVERY_N(severity, (n), google::LogMessage::SendToLog) + +#undef SYSLOG_EVERY_N +#define SYSLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), google::LogMessage::SendToSyslogAndLog) + +#undef LOG_FIRST_N +#define LOG_FIRST_N(severity, n) \ + SOME_KIND_OF_LOG_FIRST_N(severity, (n), google::LogMessage::SendToLog) + +/***************************************** + * + * Debugging loggings + * + * Only works when compiled with NDEBUG=1 + * + ****************************************/ +#undef DLOG +#ifndef NDEBUG +#define DLOG(severity) LOG(severity) +#else +#define DLOG(severity) \ + true ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_##severity.stream() +#endif + +#undef DLOG_IF +#ifndef NDEBUG +#define DLOG_IF(severity, condition) LOG_IF(severity, condition) +#else +#define DLOG_IF(severity, condition) \ + true ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_##severity.stream() +#endif + +#undef DLOG_EVERY_N +#ifndef NDEBUG +#define DLOG_EVERY_N(severity, n) LOG_EVERY_N(severity, n) +#else +#define DLOG_EVERY_N(severity, n) \ + true ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_##severity.stream() +#endif + +#undef DLOG_IF_EVERY_N +#ifndef NDEBUG +#define DLOG_IF_EVERY_N(severity, condition, n) LOG_IF_EVERY_N(severity, condition, n) +#else +#define DLOG_IF_EVERY_N(severity, condition, n) \ + true ? (void)0 : google::LogMessageVoidify() & COMPACT_GOOGLE_LOG_##severity.stream() +#endif + +#endif // COMMON_BASE_LOGGING_H_ diff --git a/src/common/base/MurmurHash2.h b/src/common/base/MurmurHash2.h new file mode 100644 index 0000000..f9e6c6e --- /dev/null +++ b/src/common/base/MurmurHash2.h @@ -0,0 +1,125 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#ifndef COMMON_BASE_MURMURHASH2_H_ +#define COMMON_BASE_MURMURHASH2_H_ + +#include +#include +#include +#include +#include + +namespace nebula { + +/** + * This is an implementation of MurmurHash2, + * which is identical to `std::hash'(at least until GCC 8.1). + * This one is more performant on short strings, because: + * 1. It could be inlined. + * 2. It utilizes the loop unrolling trick. + * Besides, it works with the plain old raw bytes array! + */ +class MurmurHash2 { + template + static constexpr bool is_char_v = + std::is_same::value || std::is_same::value || + std::is_same::value; + + public: + // std::string + size_t operator()(const std::string &str) const noexcept { + return this->operator()(str.data(), str.length()); + } + + // null-terminated C-style string + template >> + size_t operator()(const T *&str) const noexcept { + return this->operator()(str, ::strlen(str)); + } + + // raw bytes array + template >> + size_t operator()(const T *str, size_t size) const noexcept { + uint64_t seed = 0xc70f6907UL; + const uint64_t m = 0xc6a4a7935bd1e995; + const uint32_t r = 47; + uint64_t h = seed ^ (size * m); + const uint64_t *data = (const uint64_t *)str; + const uint64_t *end = data + (size / 8); + while (data != end) { + uint64_t k = *data++; + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + const unsigned char *data2 = (const unsigned char *)data; + switch (size & 7) { + case 7: + h ^= uint64_t(data2[6]) << 48; // fallthrough + case 6: + h ^= uint64_t(data2[5]) << 40; // fallthrough + case 5: + h ^= uint64_t(data2[4]) << 32; // fallthrough + case 4: + h ^= uint64_t(data2[3]) << 24; // fallthrough + case 3: + h ^= uint64_t(data2[2]) << 16; // fallthrough + case 2: + h ^= uint64_t(data2[1]) << 8; // fallthrough + case 1: + h ^= uint64_t(data2[0]); + h *= m; // fallthrough + } + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; + } + + // std::thread::id + size_t operator()(std::thread::id id) const noexcept { + return std::hash()(id); + } + + // literal string(without decay) + template >> + size_t operator()(const T (&str)[N]) const noexcept { + return this->operator()(str, N - 1); + } + + // integer + template + std::enable_if_t::value, size_t> operator()(T key) const noexcept { + return static_cast(key); + } + + // pointers + template + std::enable_if_t, size_t> operator()(const T *ptr) const noexcept { + return reinterpret_cast(ptr); + } + + // std::shared_ptr + template + size_t operator()(const std::shared_ptr &ptr) const noexcept { + return reinterpret_cast(ptr.get()); + } + + // std::unique_ptr + template + size_t operator()(const std::unique_ptr &ptr) const noexcept { + return reinterpret_cast(ptr.get()); + } +}; + +} // namespace nebula + +#endif // COMMON_BASE_MURMURHASH2_H_ diff --git a/src/common/base/ObjectPool.h b/src/common/base/ObjectPool.h new file mode 100644 index 0000000..ce0f420 --- /dev/null +++ b/src/common/base/ObjectPool.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_OBJECTPOOL_H_ +#define COMMON_BASE_OBJECTPOOL_H_ + +#include + +#include +#include +#include +#include + +#include "common/base/Arena.h" +#include "common/base/Logging.h" +#include "common/cpp/helpers.h" + +namespace nebula { + +class Expression; + +typedef std::lock_guard SLGuard; + +class ObjectPool final : private boost::noncopyable, private cpp::NonMovable { + public: + ObjectPool() {} + + ~ObjectPool() { + clear(); + } + + void clear() { + SLGuard g(lock_); + objects_.clear(); + } + + template + T *makeAndAdd(Args &&... args) { + void *ptr; + { + // alloc happens here(may throw bad_alloc), use guard to guarantee unlock + SLGuard g(lock_); + ptr = arena_.allocateAligned(sizeof(T)); + } + return add(new (ptr) T(std::forward(args)...)); + } + + bool empty() const { + return objects_.empty(); + } + + private: + // Holder the ownership of the any object + class OwnershipHolder { + public: + template + explicit OwnershipHolder(T *obj) + : obj_(obj), deleteFn_([](void *p) { reinterpret_cast(p)->~T(); }) {} + + ~OwnershipHolder() { + deleteFn_(obj_); + } + + private: + void *obj_; + std::function deleteFn_; + }; + + template + T *add(T *obj) { + SLGuard g(lock_); + objects_.emplace_back(obj); + return obj; + } + + std::list objects_; + Arena arena_; + + folly::SpinLock lock_; +}; + +} // namespace nebula + +#endif // COMMON_BASE_OBJECTPOOL_H_ diff --git a/src/common/base/SanitizerOptions.cpp b/src/common/base/SanitizerOptions.cpp new file mode 100644 index 0000000..7449a39 --- /dev/null +++ b/src/common/base/SanitizerOptions.cpp @@ -0,0 +1,69 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include "common/base/Base.h" + +#ifdef BUILT_WITH_SANITIZER + +// Following hook functions are required by the sanitizer runtime library. +// So make them exported. +// Besides, keep these hooks outside the consideration of sanitizer +#define SANITIZER_HOOK_ATTRIBUTES __attribute__((visibility("default"))) +// __attribute__((no_sanitize("address", "thread", "undefined"))) + +extern "C" { + +SANITIZER_HOOK_ATTRIBUTES +const char* __asan_default_options() { + // You could add new or update existing options via ASAN_OPTIONS at runtime. + return "" + "fast_unwind_on_malloc=0 \n" + "detect_stack_use_after_return=1 \n" + "alloc_dealloc_mismatch=1 \n" + // todo(doodle): Reopen when https://github.com/vesoft-inc/nebula/issues/3690 + // addressed throughly + "new_delete_type_mismatch=0 \n" + "strict_init_order=1 \n" + "intercept_tls_get_addr=1 \n" + "symbolize_inline_frames=1 \n" + "strict_string_checks=1 \n" + "detect_container_overflow=1 \n" + "strip_path_prefix=" NEBULA_STRINGIFY(NEBULA_HOME) "/ \n" + ""; +} + +SANITIZER_HOOK_ATTRIBUTES +const char* __asan_default_suppressions() { + return "" + "" + ""; +} + +SANITIZER_HOOK_ATTRIBUTES +const char* __lsan_default_options() { + return "" + "" + ""; +} + +SANITIZER_HOOK_ATTRIBUTES +const char* __lsan_default_suppressions() { + // NOTE Don't add extra spaces around the suppression patterns, unless you + // intend to. + return "" + "leak:folly::SingletonVault::destroyInstances\n" + "leak:__cxa_thread_atexit\n" + ""; +} + +SANITIZER_HOOK_ATTRIBUTES +const char* __ubsan_default_options() { + return "print_stacktrace=1 \n"; +} + +} // extern "C" + +#undef SANITIZER_HOOK_ATTRIBUTES + +#endif diff --git a/src/common/base/SignalHandler.cpp b/src/common/base/SignalHandler.cpp new file mode 100644 index 0000000..f99c6e5 --- /dev/null +++ b/src/common/base/SignalHandler.cpp @@ -0,0 +1,145 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/base/SignalHandler.h" + +#include "common/base/Base.h" + +namespace nebula { + +SignalHandler::SignalHandler() {} + +Status SignalHandler::init() { + // Ignore SIGPIPE and SIGHUP + auto ignored = {SIGPIPE, SIGHUP}; + for (auto sig : ignored) { + struct sigaction act; + ::memset(&act, 0, sizeof(act)); + act.sa_handler = SIG_IGN; + if (0 != ::sigaction(sig, &act, nullptr)) { + return Status::Error("Register signal %d failed: %s", sig, ::strerror(errno)); + } + } + + return Status::OK(); +} + +SignalHandler &SignalHandler::get() { + static SignalHandler instance; + return instance; +} + +// static +Status SignalHandler::install(int sig, Handler handler) { + static bool init_ = false; + if (!init_) { + auto status = get().init(); + if (!status.ok()) { + return status; + } + init_ = true; + } + + CHECK(sig >= 1 && sig <= 64); + CHECK(sig != SIGKILL) << "SIGKILL cannot be handled"; + CHECK(sig != SIGSTOP) << "SIGKSTOP cannot be handled"; + return get().installInternal(sig, std::move(handler)); +} + +// static +Status SignalHandler::install(std::initializer_list sigs, Handler handler) { + auto status = Status::OK(); + for (auto sig : sigs) { + status = install(sig, handler); + if (!status.ok()) { + break; + } + } + return status; +} + +Status SignalHandler::installInternal(int sig, Handler handler) { + struct sigaction act; + ::memset(&act, 0, sizeof(act)); + act.sa_sigaction = &handlerHook; + act.sa_flags |= SA_SIGINFO; + if (::sigaction(sig, &act, nullptr) != 0) { + return Status::Error("Register signal %d failed: %s", sig, ::strerror(errno)); + } + auto index = sig - 1; + if (handlers_[index]) { + LOG(WARNING) << "Register signal " << sig << " twice!"; + } + handlers_[index] = std::move(handler); + return Status::OK(); +} + +// static +void SignalHandler::handlerHook(int sig, siginfo_t *info, void *uctx) { + get().doHandle(sig, info, uctx); +} + +void SignalHandler::doHandle(int sig, siginfo_t *info, void *uctx) { + switch (sig) { + case SIGSEGV: // segment fault + case SIGABRT: // abort + case SIGILL: // ill instruction + case SIGFPE: // floating point error, e.g. divide by zero + case SIGBUS: // I/O error in mmapped memory, mce error, etc. + handleFatalSignal(sig, info, uctx); + break; + case SIGCHLD: + // TODO(dutor) Since we rarely use this signal, we regard it as a general + // one for now. + handleGeneralSignal(sig, info); + break; + } + + handleGeneralSignal(sig, info); +} + +void SignalHandler::handleGeneralSignal(int sig, siginfo_t *info) { + auto index = sig - 1; + GeneralSignalInfo siginfo(info); + handlers_[index](&siginfo); +} + +void SignalHandler::handleFatalSignal(int sig, siginfo_t *info, void *uctx) { + auto index = sig - 1; + FatalSignalInfo siginfo(info, uctx); + handlers_[index](&siginfo); + // Restore the signal handler to its default to make the program crash + ::signal(sig, SIG_DFL); + ::raise(sig); +} + +SignalHandler::GeneralSignalInfo::GeneralSignalInfo(const siginfo_t *info) { + pid_ = info->si_pid; + uid_ = info->si_uid; + sig_ = info->si_signo; +} + +const char *SignalHandler::GeneralSignalInfo::toString() const { + static thread_local char buffer[1024]; + snprintf(buffer, + sizeof(buffer), + "sig[%d], name[%s], pid[%d], uid(%d)", + sig_, + ::strsignal(sig_), + pid_, + uid_); + return buffer; +} + +SignalHandler::FatalSignalInfo::FatalSignalInfo(const siginfo_t *info, void *uctx) + : GeneralSignalInfo(info) { + UNUSED(uctx); +} + +const char *SignalHandler::FatalSignalInfo::toString() const { + return GeneralSignalInfo::toString(); +} + +} // namespace nebula diff --git a/src/common/base/SignalHandler.h b/src/common/base/SignalHandler.h new file mode 100644 index 0000000..80598b8 --- /dev/null +++ b/src/common/base/SignalHandler.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_SIGNALHANDLER_H_ +#define COMMON_BASE_SIGNALHANDLER_H_ + +#include + +#include "common/base/Base.h" +#include "common/base/Status.h" + +/** + * SignalHandler is a singleton to do the basic signal handling, + * mainly used in a daemon executable. + * + * By default, it ignores SIGPIPE and SIGHUP as we usually do. + * + * In the signal handler's context, we try our best to avoid memory allocation, + * since this is the major reason which results in async-signal-unsafe. + * + * We strongly suggest not to handle fatal signals by yourself, + * just let the program die and generate a coredump file. + */ + +namespace nebula { + +class SignalHandler final { + public: + ~SignalHandler() = default; + + /** + * To install one or several signals to handle. + * Upon any signal arrives, the corresponding handler would be invoked, + * with an argument holding the information about the signal and the sender. + * The handler typically prints out the info and do some other things, + * e.g. stop the process on SIGTERM. + */ + class GeneralSignalInfo; + using Handler = std::function; + static Status install(int sig, Handler handler); + static Status install(std::initializer_list sigs, Handler handler); + + class GeneralSignalInfo { + public: + explicit GeneralSignalInfo(const siginfo_t *info); + virtual ~GeneralSignalInfo() = default; + virtual const char *toString() const; + int sig() const { + return sig_; + } + pid_t pid() const { + return pid_; + } + uid_t uid() const { + return uid_; + } + + protected: + int sig_{0}; + pid_t pid_{0}; + uid_t uid_{0}; + }; + + // TODO(dutor) Retrieve the stacktrace of the thread who raises the fatal + // signal + class FatalSignalInfo final : public GeneralSignalInfo { + public: + FatalSignalInfo(const siginfo_t *info, void *uctx); + ~FatalSignalInfo() override = default; + const char *toString() const override; + }; + + private: + SignalHandler(); + SignalHandler(const SignalHandler &) = delete; + SignalHandler &operator=(const SignalHandler &) = delete; + SignalHandler(SignalHandler &&) = delete; + SignalHandler &operator=(SignalHandler &&) = delete; + + // Get the singleton + static SignalHandler &get(); + + // The handler we use to register to the operating system. + static void handlerHook(int sig, siginfo_t *info, void *uctx); + + // init SIGPIPE and SIGHUB + Status init(); + + // Invoked by handlerHook, and dispatch signals to handleGeneralSignal or + // handleFatalSignal + void doHandle(int sig, siginfo_t *info, void *uctx); + + // To handle general signals like SIGINT, SIGTERM, etc. + void handleGeneralSignal(int sig, siginfo_t *info); + + // To handle fatal signals like SIGSEGV, SIGABRT, etc. + void handleFatalSignal(int sig, siginfo_t *info, void *uctx); + + Status installInternal(int sig, Handler handler); + + private: + // The POSIX signal number ranges from 1 to 64, so we use (sig - 1) as index + std::array handlers_; +}; + +inline std::ostream &operator<<(std::ostream &os, const SignalHandler::GeneralSignalInfo &info) { + return os << info.toString(); +} + +} // namespace nebula + +#endif // COMMON_BASE_SIGNALHANDLER_H_ diff --git a/src/common/base/Status.cpp b/src/common/base/Status.cpp new file mode 100644 index 0000000..51d4f86 --- /dev/null +++ b/src/common/base/Status.cpp @@ -0,0 +1,111 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/base/Status.h" + +#include "common/base/Base.h" + +namespace nebula { + +Status::Status(Code code, folly::StringPiece msg) { + const uint16_t size = msg.size(); + auto state = std::unique_ptr(new char[size + kHeaderSize]); + auto *header = reinterpret_cast
(state.get()); + header->size_ = size; + header->code_ = code; + ::memcpy(&state[kHeaderSize], msg.data(), size); + state_ = std::move(state); +} + +std::string Status::message() const { + if (state_ == nullptr) return ""; + return std::string(&state_[kHeaderSize], size()); +} + +std::string Status::toString() const { + Code code = this->code(); + if (code == kOk) { + return "OK"; + } + std::string result(toString(code)); + result.append(&state_[kHeaderSize], size()); + return result; +} + +std::unique_ptr Status::copyState(const char *state) { + const auto size = *reinterpret_cast(state); + const auto total = size + kHeaderSize; + auto result = std::unique_ptr(new char[total]); + ::memcpy(&result[0], state, total); + return result; +} + +std::string Status::format(const char *fmt, va_list args) { + char result[256]; + vsnprintf(result, sizeof(result), fmt, args); + return result; +} + +// static +const char *Status::toString(Code code) { + switch (code) { + case kOk: + return "OK"; + case kInserted: + return "Inserted: "; + case kError: + return ""; + case kNoSuchFile: + return "NoSuchFile: "; + case kNotSupported: + return "NotSupported: "; + case kSyntaxError: + return "SyntaxError: "; + case kStatementEmpty: + return "StatementEmpty: "; + case kSemanticError: + return "SemanticError: "; + case kGraphMemoryExceeded: + return "GraphMemoryExceeded: "; + case kKeyNotFound: + return "KeyNotFound: "; + case kPartialSuccess: + return "PartialSuccess: "; + case kStorageMemoryExceeded: + return "StorageMemoryExceeded: "; + case kSpaceNotFound: + return "SpaceNotFound: "; + case kHostNotFound: + return "HostNotFound: "; + case kTagNotFound: + return "TagNotFound: "; + case kEdgeNotFound: + return "EdgeNotFound: "; + case kGroupNotFound: + return "GroupNotFound: "; + case kZoneNotFound: + return "ZoneNotFound: "; + case kUserNotFound: + return "UserNotFound: "; + case kLeaderChanged: + return "LeaderChanged: "; + case kBalanced: + return "Balanced: "; + case kIndexNotFound: + return "IndexNotFound: "; + case kPartNotFound: + return "PartNotFound: "; + case kPermissionError: + return "PermissionError: "; + case kListenerNotFound: + return "ListenerNotFound"; + case kSessionNotFound: + return "SessionNotFound"; + } + DLOG(FATAL) << "Invalid status code: " << static_cast(code); + return ""; +} + +} // namespace nebula diff --git a/src/common/base/Status.h b/src/common/base/Status.h new file mode 100644 index 0000000..598320a --- /dev/null +++ b/src/common/base/Status.h @@ -0,0 +1,256 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_STATUS_H_ +#define COMMON_BASE_STATUS_H_ + +#include "common/base/Base.h" + +/** + * Status is modeled on the one from levelDB, beyond that, + * this one adds support on move semantics and formatted error messages. + * + * Status is as cheap as raw pointers in the successful case, + * without any heap memory allocations. + */ + +namespace nebula { + +template +class StatusOr; + +class Status final { + public: + Status() = default; + + ~Status() = default; + + Status(const Status &rhs) { + state_ = (rhs.state_ == nullptr ? nullptr : copyState(rhs.state_.get())); + } + + Status &operator=(const Status &rhs) { + // `state_ == rhs.state_' means either `this == &rhs', + // or both `*this' and `rhs' are OK + if (state_ != rhs.state_) { + state_ = rhs.state_ == nullptr ? nullptr : copyState(rhs.state_.get()); + } + return *this; + } + + Status(Status &&rhs) noexcept { + state_ = std::move(rhs.state_); + } + + Status &operator=(Status &&rhs) noexcept { + // `state_ == rhs.state_' means either `this == &rhs', + // or both `*this' and `rhs' are OK + if (state_ != rhs.state_) { + state_ = std::move(rhs.state_); + } + return *this; + } + + static Status from(const Status &s) { + return s; + } + + template + static Status from(StatusOr &&s) { + return std::move(s).status(); + } + + template + static Status from(const StatusOr &s) { + return s.status(); + } + + bool operator==(const Status &rhs) const { + // `state_ == rhs.state_' means either `this == &rhs', + // or both `*this' and `rhs' are OK + if (state_ == rhs.state_) { + return true; + } + return code() == rhs.code(); + } + + bool operator!=(const Status &rhs) const { + return !(*this == rhs); + } + + bool ok() const { + return state_ == nullptr; + } + + static Status OK() { + return Status(); + } + +#define STATUS_GENERATOR(ERROR) \ + static Status ERROR() { \ + return Status(k##ERROR, ""); \ + } \ + \ + static Status ERROR(folly::StringPiece msg) { \ + return Status(k##ERROR, msg); \ + } \ + \ + static Status ERROR(const char *fmt, ...) __attribute__((format(printf, 1, 2))) { \ + va_list args; \ + va_start(args, fmt); \ + auto msg = format(fmt, args); \ + va_end(args); \ + return Status(k##ERROR, msg); \ + } \ + \ + bool is##ERROR() const { \ + return code() == k##ERROR; \ + } + // Some succeeded codes + STATUS_GENERATOR(Inserted); + + // General errors + STATUS_GENERATOR(Error); + STATUS_GENERATOR(NoSuchFile); + STATUS_GENERATOR(NotSupported); + + // Graph engine errors + STATUS_GENERATOR(SyntaxError); + STATUS_GENERATOR(SemanticError); + STATUS_GENERATOR(GraphMemoryExceeded); + + // Nothing is executed When command is comment + STATUS_GENERATOR(StatementEmpty); + + // Storage engine errors + STATUS_GENERATOR(KeyNotFound); + STATUS_GENERATOR(PartialSuccess); + STATUS_GENERATOR(StorageMemoryExceeded); + + // Meta engine errors + // TODO(dangleptr) we could use ErrorOr to replace SpaceNotFound here. + STATUS_GENERATOR(SpaceNotFound); + STATUS_GENERATOR(HostNotFound); + STATUS_GENERATOR(TagNotFound); + STATUS_GENERATOR(EdgeNotFound); + STATUS_GENERATOR(UserNotFound); + STATUS_GENERATOR(IndexNotFound); + STATUS_GENERATOR(GroupNotFound); + STATUS_GENERATOR(ZoneNotFound); + STATUS_GENERATOR(LeaderChanged); + STATUS_GENERATOR(Balanced); + STATUS_GENERATOR(PartNotFound); + STATUS_GENERATOR(ListenerNotFound); + STATUS_GENERATOR(SessionNotFound); + // User or permission errors + STATUS_GENERATOR(PermissionError); + +#undef STATUS_GENERATOR + + std::string toString() const; + + friend std::ostream &operator<<(std::ostream &os, const Status &status); + + // If some kind of error really needs to be distinguished with others using a + // specific code, other than a general code and specific msg, you could add a + // new code below, e.g. kSomeError, and add the corresponding + // STATUS_GENERATOR(SomeError) + enum Code : uint16_t { + // OK + kOk = 0, + kInserted = 1, + // 1xx, for general errors + kError = 101, + kNoSuchFile = 102, + kNotSupported = 103, + // 2xx, for graph engine errors + kSyntaxError = 201, + kStatementEmpty = 202, + kSemanticError = 203, + kGraphMemoryExceeded = 204, + // 3xx, for storage engine errors + kKeyNotFound = 301, + kPartialSuccess = 302, + kStorageMemoryExceeded = 303, + // 4xx, for meta service errors + kSpaceNotFound = 404, + kHostNotFound = 405, + kTagNotFound = 406, + kEdgeNotFound = 407, + kUserNotFound = 408, + kLeaderChanged = 409, + kBalanced = 410, + kIndexNotFound = 411, + kPartNotFound = 412, + kGroupNotFound = 413, + kZoneNotFound = 414, + kListenerNotFound = 415, + kSessionNotFound = 416, + // 5xx for user or permission error + kPermissionError = 501, + }; + + Code code() const { + if (state_ == nullptr) { + return kOk; + } + return reinterpret_cast(state_.get())->code_; + } + + std::string message() const; + + private: + // REQUIRES: stat_ != nullptr + uint16_t size() const { + return reinterpret_cast(state_.get())->size_; + } + + Status(Code code, folly::StringPiece msg); + + static std::unique_ptr copyState(const char *state); + + static std::string format(const char *fmt, va_list args); + + static const char *toString(Code code); + + private: + struct Header { + uint16_t size_; + Code code_; + }; + static constexpr auto kHeaderSize = sizeof(Header); + // state_ == nullptr indicates OK + // otherwise, the buffer layout is: + // state_[0..1] length of the error msg, i.e. size() - kHeaderSize + // state_[2..3] code + // state_[4...] verbose error message + std::unique_ptr state_; +}; + +inline std::ostream &operator<<(std::ostream &os, const Status &status) { + return os << status.toString(); +} + +} // namespace nebula + +#define NG_RETURN_IF_ERROR(s) \ + do { \ + const auto &__s = (s); \ + if (UNLIKELY(!__s.ok())) { \ + return ::nebula::Status::from(__s); \ + } \ + } while (0) + +#define NG_LOG_AND_RETURN_IF_ERROR(s) \ + do { \ + const auto &__s = (s); \ + if (UNLIKELY(!__s.ok())) { \ + ::nebula::Status __status = ::nebula::Status::from(__s); \ + LOG(ERROR) << __status; \ + return __status; \ + } \ + } while (0) + +#endif // COMMON_BASE_STATUS_H_ diff --git a/src/common/base/StatusOr.h b/src/common/base/StatusOr.h new file mode 100644 index 0000000..605f94b --- /dev/null +++ b/src/common/base/StatusOr.h @@ -0,0 +1,358 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_BASE_STATUSOR_H_ +#define COMMON_BASE_STATUSOR_H_ + +#include + +#include "common/base/Base.h" +#include "common/base/Status.h" + +namespace nebula { + +template +class StatusOr final { + public: + // Make friends with other compatible StatusOr + template + friend class StatusOr; + + // Convenience type traits + template + static constexpr bool is_status_v = std::is_same>::value; + + // Tell if `U' is of type `StatusOr' + template + struct is_status_or { + static auto sfinae(...) -> uint8_t { + return 0; + } + template + static auto sfinae(const StatusOr &) -> uint16_t { + return 0; + } + static constexpr auto value = (sizeof(sfinae(std::declval())) == sizeof(uint16_t)); + }; + + template + static constexpr bool is_status_or_v = is_status_or::value; + + // Tell if `T' is initializable from `U'. + // We simply use `is_constructible_v' for now, + // because we are only utilizing direct-initializations. + // Besides, we must exclude `Status' and `StatusOr' from `U'. + // + // TODO(dutor) we may take other cases into account in future, + // e.g. convertible but not constructible. + template + static constexpr bool is_initializable_v = + is_constructible_v &&std::is_convertible::value && !is_status_or_v && + !is_status_v; + + // Assert that `T' must be copy/move constructible + static_assert(is_copy_or_move_constructible_v, "`T' must be copy/move constructible"); + + // Assert that `T' must not be of type reference + static_assert(!std::is_reference::value, "`T' must not be of type reference"); + + // Assert that `T' must not be of type `Status' + static_assert(!is_status_v, "`T' must not be of type `Status'"); + + // Assert that `T' must not be of type `StatusOr' + static_assert(!is_status_or_v, "`T' must not be of type `StatusOr'"); + + // `StatusOr' contains neither a Status nor a value + // in the default-constructed case. + // From the semantics aspect, it must have been associated with + // a Status or value eventually before being used. + StatusOr() { + state_ = kVoid; + } + + // Destruct the `Status' or value if it's holding one. + ~StatusOr() { + destruct(); + } + + // Copy/move construct from `Status' + // Not explicit to allow construct from a `Status', e.g. in the `return' + // statement + template + StatusOr(U &&status, std::enable_if_t> * = nullptr) // NOLINT + : variant_(std::forward(status)) { + state_ = kStatus; + } + + // Copy/move construct with a value of any compatible type + // Not explicit to allow construct from a value, e.g. in the `return' + // statement + template >> + StatusOr(U &&value) // NOLINT + : variant_(std::forward(value)) { + state_ = kValue; + } + + StatusOr(T &&value) // NOLINT + : variant_(std::move(value)) { + state_ = kValue; + } + + // Copy constructor + StatusOr(const StatusOr &rhs) : state_(rhs.state_) { + if (hasValue()) { + new (&variant_) Variant(rhs.variant_.value_); + } else if (hasStatus()) { + new (&variant_) Variant(rhs.variant_.status_); + } + } + + // Copy construct from a lvalue of `StatusOr' + template >> + StatusOr(const StatusOr &rhs) : state_(rhs.state_) { + if (hasValue()) { + new (&variant_) Variant(rhs.variant_.value_); + } else if (hasStatus()) { + new (&variant_) Variant(rhs.variant_.status_); + } + } + + // Copy assignment operator + StatusOr &operator=(const StatusOr &rhs) { + if (&rhs == this) { + return *this; + } + reset(); + if (rhs.hasValue()) { + new (&variant_) Variant(rhs.variant_.value_); + state_ = kValue; + } else if (rhs.hasStatus()) { + new (&variant_) Variant(rhs.variant_.status_); + state_ = kStatus; + } + return *this; + } + + // Move constructor + StatusOr(StatusOr &&rhs) noexcept : state_(rhs.state_) { + if (hasValue()) { + new (&variant_) Variant(std::move(rhs.variant_.value_)); + rhs.resetValue(); + } else if (rhs.hasStatus()) { + new (&variant_) Variant(std::move(rhs.variant_.status_)); + rhs.resetStatus(); + } + } + + // Move construct from a rvalue of StatusOr + template >> + StatusOr(StatusOr &&rhs) noexcept : state_(rhs.state_) { + if (hasValue()) { + new (&variant_) Variant(std::move(rhs.variant_.value_)); + rhs.resetValue(); + } else if (rhs.hasStatus()) { + new (&variant_) Variant(std::move(rhs.variant_.status_)); + rhs.resetStatus(); + } else { + // do nothing on void + } + } + + // Move assignment operator + StatusOr &operator=(StatusOr &&rhs) noexcept { + if (&rhs == this) { + return *this; + } + reset(); + if (rhs.hasValue()) { + new (&variant_) Variant(std::move(rhs.variant_.value_)); + rhs.resetValue(); + state_ = kValue; + } else if (rhs.hasStatus()) { + new (&variant_) Variant(std::move(rhs.variant_.status_)); + rhs.resetStatus(); + state_ = kStatus; + } else { + // do nothing on void, since it's already reset + } + return *this; + } + + // Move assignment operator from a rvalue of `StatusOr' + template >> + StatusOr &operator=(StatusOr &&rhs) noexcept { + reset(); + if (rhs.hasValue()) { + new (&variant_) Variant(std::move(rhs.variant_.value_)); + rhs.resetValue(); + state_ = kValue; + } else if (rhs.hasStatus()) { + new (&variant_) Variant(std::move(rhs.variant_.status_)); + rhs.resetStatus(); + state_ = kStatus; + } else { + // do nothing on void, since it's already reset + } + return *this; + } + + // Move assignment operator from a rvalue of any compatible type with `T' + template >> + StatusOr &operator=(U &&value) noexcept { + destruct(); + new (&variant_) Variant(std::forward(value)); + state_ = kValue; + return *this; + } + + // Copy assign from a lvalue of `Status' + StatusOr &operator=(const Status &status) { + destruct(); + new (&variant_) Variant(status); + state_ = kStatus; + return *this; + } + + // Move assign from a rvalue of `Status' + StatusOr &operator=(Status &&status) noexcept { + destruct(); + new (&variant_) Variant(std::move(status)); + state_ = kStatus; + return *this; + } + + // Tell if `*this' contains a value + bool ok() const { + return hasValue(); + } + + // Type conversion operator, i.e. alias of `ok()' + operator bool() const { + return ok(); + } + + // Return the associated `Status' if and only if it has one, + // + Status status() const & { + CHECK(hasStatus()); + return variant_.status_; + } + + Status status() && { + CHECK(hasStatus()); + auto status = std::move(variant_.status_); + resetStatus(); + return status; + } + + // Return the non-const lvalue reference to the associated value + // `ok()' is DCHECK'd + T &value() & { + DCHECK(ok()); + return variant_.value_; + } + + // Return the const lvalue reference to the associated value + const T &value() const & { + DCHECK(ok()); + return variant_.value_; + } + + // Move the associated value out. + // `*this' must be a rvalue + T value() && { + DCHECK(ok()); + auto value = std::move(variant_.value_); + resetValue(); + return value; + } + + private: + bool hasValue() const { + return state_ == kValue; + } + + bool hasStatus() const { + return state_ == kStatus; + } + + bool isVoid() const { + return state_ == kVoid; + } + + void destructValue() { + variant_.value_.~T(); + } + + void destructStatus() { + variant_.status_.~Status(); + } + + void resetValue() { + destructValue(); + state_ = kVoid; + } + + void resetStatus() { + destructStatus(); + state_ = kVoid; + } + + void destruct() { + if (isVoid()) { + return; + } + if (hasValue()) { + destructValue(); + } else { + destructStatus(); + } + } + + void reset() { + destruct(); + state_ = kVoid; + } + + private: + // Variant type to hold a `Status', or a `T', or nothing + union Variant { + Variant() {} + Variant(const Status &status) : status_(status) {} + Variant(Status &&status) : status_(std::move(status)) {} + template >> + Variant(U &&value) : value_(std::forward(value)) {} + ~Variant() {} + + Status status_; + T value_; + }; + + static constexpr uint8_t kVoid = 0; + static constexpr uint8_t kStatus = 1; + static constexpr uint8_t kValue = 2; + + Variant variant_; + uint8_t state_; +}; + +namespace internal { +template +struct StatusOrValueType { + using type = T; +}; + +template +struct StatusOrValueType> { + using type = std::remove_cv_t>; +}; +} // namespace internal + +template +using status_or_value_t = typename internal::StatusOrValueType::type; + +} // namespace nebula + +#endif // COMMON_BASE_STATUSOR_H_ diff --git a/src/common/base/test/ArenaBenchmark.cpp b/src/common/base/test/ArenaBenchmark.cpp new file mode 100644 index 0000000..5bab2fe --- /dev/null +++ b/src/common/base/test/ArenaBenchmark.cpp @@ -0,0 +1,88 @@ +// Copyright (c) 2022 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. + +#include +#include +#include + +#include +#include + +#include "common/base/Arena.h" +#include "common/expression/LabelExpression.h" + +namespace nebula { + +class TestExpr : public LabelExpression { + public: + explicit TestExpr(const std::string &name = "") + : LabelExpression(reinterpret_cast(1), name) {} +}; + +BENCHMARK(DefaultAllocator, iters) { + std::size_t round = iters * 1000; + for (std::size_t _ = 0; _ < round; ++_) { + auto *expr = new TestExpr("Label"); + delete expr; + } +} + +BENCHMARK_RELATIVE(ArenaAllocator, iters) { + std::size_t round = iters * 1000; + Arena a; + for (std::size_t _ = 0; _ < round; ++_) { + auto *ptr = a.allocateAligned(sizeof(TestExpr)); + auto *expr = new (ptr) TestExpr("Label"); + expr->~TestExpr(); + } +} + +BENCHMARK_RELATIVE(FollyArenaAllocator, iters) { + std::size_t round = iters * 1000; + folly::SysArena a; + for (std::size_t _ = 0; _ < round; ++_) { + auto *ptr = a.allocate(sizeof(TestExpr)); + auto *expr = new (ptr) TestExpr("Label"); + expr->~TestExpr(); + } +} + +BENCHMARK_DRAW_LINE(); + +} // namespace nebula + +int main(int argc, char **argv) { + folly::init(&argc, &argv, true); + + folly::runBenchmarks(); + return 0; +} + +// CPU info +// Brand Raw: Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz +// Hz Advertised Friendly: 3.0000 GHz +// Hz Actual Friendly: 3.2942 GHz +// Hz Advertised: (3000000000, 0) +// Hz Actual: (3294220000, 0) +// Arch: X86_64 +// Bits: 64 +// Count: 40 +// Arch String Raw: x86_64 +// L1 Data Cache Size: 32768 +// L1 Instruction Cache Size: 32768 +// L2 Cache Size: 262144 +// L2 Cache Line Size: 256 +// L2 Cache Associativity: 6 +// L3 Cache Size: 26214400 +// +// Build in Release mode +// +// ============================================================================ +// /home/shylock.huang/nebula/src/common/base/test/ArenaBenchmark.cpprelative time/iter iters/s +// ============================================================================ +// DefaultAllocator 36.59us 27.33K +// ArenaAllocator 145.89% 25.08us 39.87K +// FollyArenaAllocator 138.96% 26.33us 37.98K +// ---------------------------------------------------------------------------- +// ============================================================================ diff --git a/src/common/base/test/ArenaTest.cpp b/src/common/base/test/ArenaTest.cpp new file mode 100644 index 0000000..47dc327 --- /dev/null +++ b/src/common/base/test/ArenaTest.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2022 vesoft inc. All rights reserved. +// +// This source code is licensed under Apache 2.0 License. + +#include + +#include + +#include "common/base/Arena.h" + +namespace nebula { + +TEST(ArenaTest, Basic) { + Arena a; + + for (int i = 1; i < 4096; i += 8) { + void *ptr = a.allocateAligned(i); + EXPECT_EQ(reinterpret_cast(ptr) % std::alignment_of::value, 0); + } +} + +TEST(ArenaTest, Construct) { + Arena a; + { + void *ptr = a.allocateAligned(sizeof(std::string)); + auto *obj = new (ptr) std::string("Hello World!"); + EXPECT_EQ(*obj, "Hello World!"); + obj->~basic_string(); + } + { + void *ptr = a.allocateAligned(sizeof(int)); + auto *obj = new (ptr) int(3); // NOLINT + EXPECT_EQ(*obj, 3); + } + { + for (std::size_t i = 0; i < 1024; ++i) { + void *ptr = a.allocateAligned(sizeof(int)); + auto *obj = new (ptr) int(i); // NOLINT + EXPECT_EQ(*obj, i); + } + } +} + +} // namespace nebula diff --git a/src/common/base/test/CMakeLists.txt b/src/common/base/test/CMakeLists.txt new file mode 100644 index 0000000..e644906 --- /dev/null +++ b/src/common/base/test/CMakeLists.txt @@ -0,0 +1,133 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_test( + NAME cord_test + SOURCES CordTest.cpp + OBJECTS $ + LIBRARIES gtest +) + +nebula_add_test( + NAME icord_test + SOURCES ICordTest.cpp + OBJECTS $ + LIBRARIES gtest +) + +nebula_add_executable( + NAME icord_bm + SOURCES ICordBenchmark.cpp + OBJECTS $ + LIBRARIES follybenchmark boost_regex +) + +nebula_add_executable( + NAME logging_bm + SOURCES LoggingBenchmark.cpp + OBJECTS $ + LIBRARIES follybenchmark boost_regex +) + +nebula_add_test( + NAME murmurhash2_test + SOURCES MurmurHash2Test.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_test( + NAME status_test + SOURCES StatusTest.cpp + OBJECTS $ $ + LIBRARIES gtest gtest_main +) + +nebula_add_test( + NAME status_or_test + SOURCES StatusOrTest.cpp + OBJECTS $ $ + LIBRARIES gtest gtest_main +) + +nebula_add_test( + NAME either_or_test + SOURCES EitherOrTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_test( + NAME error_or_test + SOURCES ErrorOrTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_executable( + NAME hash_bm + SOURCES HashBenchmark.cpp + OBJECTS $ + LIBRARIES follybenchmark boost_regex +) + +nebula_add_test( + NAME signal_handler_test + SOURCES SignalHandlerTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_test( + NAME lru_test + SOURCES ConcurrentLRUCacheTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_executable( + NAME range_vs_transform_bm + SOURCES RangeVsTransformBenchmark.cpp + OBJECTS $ + LIBRARIES follybenchmark boost_regex +) +target_compile_options(range_vs_transform_bm PRIVATE -O3) + +nebula_add_test( + NAME object_pool_test + SOURCES ObjectPoolTest.cpp + OBJECTS $ + LIBRARIES gtest gtest_main +) + +nebula_add_executable( + NAME arena_bm + SOURCES ArenaBenchmark.cpp + OBJECTS + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + LIBRARIES + follybenchmark + ${THRIFT_LIBRARIES} +) + +nebula_add_test( + NAME arena_test + SOURCES ArenaTest.cpp + OBJECTS + $ + LIBRARIES + gtest + gtest_main +) diff --git a/src/common/base/test/ConcurrentLRUCacheTest.cpp b/src/common/base/test/ConcurrentLRUCacheTest.cpp new file mode 100644 index 0000000..f109c82 --- /dev/null +++ b/src/common/base/test/ConcurrentLRUCacheTest.cpp @@ -0,0 +1,167 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/ConcurrentLRUCache.h" + +namespace nebula { + +TEST(ConcurrentLRUCacheTest, SimpleTest) { + ConcurrentLRUCache cache(1024); + cache.insert(10, "ten"); + { + auto v = cache.get(10); + EXPECT_TRUE(v.ok()); + EXPECT_EQ("ten", v.value()); + } + + { + auto v = cache.get(5); + EXPECT_FALSE(v.ok()); + } + + EXPECT_EQ(5, cache.bucketIndex(100, 5)); + EXPECT_EQ(11, cache.bucketIndex(100, 11)); + EXPECT_EQ(0, cache.bucketIndex(100, 0)); + EXPECT_EQ(1024 % 16, cache.bucketIndex(100, 1024)); + EXPECT_EQ(std::hash()(100) % 16, cache.bucketIndex(100, -1)); + + EXPECT_EQ(0, cache.evicts()); + EXPECT_EQ(1, cache.hits()); + EXPECT_EQ(2, cache.total()); + + for (auto i = 0; i < 100; i++) { + auto v = cache.get(10); + EXPECT_TRUE(v.ok()); + EXPECT_EQ("ten", v.value()); + } + EXPECT_EQ(0, cache.evicts()); + EXPECT_EQ(101, cache.hits()); + EXPECT_EQ(102, cache.total()); +} + +TEST(ConcurrentLRUCacheTest, PutIfAbsentTest) { + ConcurrentLRUCache cache(1024); + { + auto v = cache.putIfAbsent(10, "ten"); + EXPECT_EQ(Status::Inserted(), v.status()); + } + + { + auto v = cache.putIfAbsent(10, "ele"); + EXPECT_TRUE(v.ok()); + EXPECT_EQ("ten", v.value()); + } + + EXPECT_EQ(0, cache.evicts()); + EXPECT_EQ(1, cache.hits()); + EXPECT_EQ(2, cache.total()); +} + +TEST(ConcurrentLRUCacheTest, EvictTest) { + ConcurrentLRUCache cache(1000, 0); + for (auto j = 0; j < 1000; j++) { + cache.insert(j, folly::stringPrintf("%d_str", j)); + } + for (auto i = 0; i < 1000; i++) { + auto v = cache.get(i); + EXPECT_TRUE(v.ok()); + EXPECT_EQ(folly::stringPrintf("%d_str", i), v.value()); + } + for (auto j = 1000; j < 2000; j++) { + cache.insert(j, folly::stringPrintf("%d_str", j)); + } + for (auto i = 1000; i < 2000; i++) { + auto v = cache.get(i); + EXPECT_TRUE(v.ok()); + EXPECT_EQ(folly::stringPrintf("%d_str", i), v.value()); + } + for (auto i = 0; i < 1000; i++) { + auto v = cache.get(i); + EXPECT_FALSE(v.ok()); + } + EXPECT_EQ(1000, cache.evicts()); + EXPECT_EQ(2000, cache.hits()); + EXPECT_EQ(3000, cache.total()); +} + +TEST(ConcurrentLRUCacheTest, EvictKeyTest) { + ConcurrentLRUCache cache(1024, 4); + for (auto j = 0; j < 1000; j++) { + cache.insert(j, folly::stringPrintf("%d_str", j)); + } + for (auto i = 0; i < 1000; i++) { + auto v = cache.get(i); + EXPECT_TRUE(v.ok()); + EXPECT_EQ(folly::stringPrintf("%d_str", i), v.value()); + } + for (auto i = 1; i < 1000; i += 2) { + cache.evict(i); + } + for (auto i = 0; i < 1000; i++) { + auto v = cache.get(i); + if (i % 2 != 0) { + EXPECT_FALSE(v.ok()); + } else { + EXPECT_TRUE(v.ok()); + } + } + + EXPECT_EQ(500, cache.evicts()); + EXPECT_EQ(1500, cache.hits()); + EXPECT_EQ(2000, cache.total()); +} + +TEST(ConcurrentLRUCacheTest, MultiThreadsTest) { + ConcurrentLRUCache cache(1024 * 1024); + std::vector threads; + for (auto i = 0; i < 10; i++) { + threads.emplace_back([&cache, i]() { + for (auto j = i * 1000; j < (i + 1) * 1000; j++) { + cache.insert(j, folly::stringPrintf("%d_str", j)); + } + }); + } + for (auto i = 0; i < 10; i++) { + threads[i].join(); + } + for (auto i = 0; i < 10000; i++) { + auto v = cache.get(i); + EXPECT_TRUE(v.ok()); + EXPECT_EQ(folly::stringPrintf("%d_str", i), v.value()); + } + + EXPECT_EQ(0, cache.evicts()); + EXPECT_EQ(10000, cache.hits()); + EXPECT_EQ(10000, cache.total()); +} + +TEST(ConcurrentLRUCacheTest, OverwriteTest) { + ConcurrentLRUCache cache(1024); + cache.insert(10, "ten"); + { + auto v = cache.get(10); + EXPECT_TRUE(v.ok()); + EXPECT_EQ("ten", v.value()); + } + cache.insert(10, "ten_v1"); + { + auto v = cache.get(10); + EXPECT_TRUE(v.ok()); + EXPECT_EQ("ten_v1", v.value()); + } +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/common/base/test/CordTest.cpp b/src/common/base/test/CordTest.cpp new file mode 100644 index 0000000..267e149 --- /dev/null +++ b/src/common/base/test/CordTest.cpp @@ -0,0 +1,30 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/Cord.h" + +namespace nebula { + +TEST(CordTest, multipleBlocks) { + Cord cord(128); + + std::string buf; + for (int i = 0; i < 100; i++) { + buf.append("Hello World!"); + } + + cord.write(buf.data(), buf.size()); + + EXPECT_EQ(buf.size(), cord.size()); + EXPECT_EQ(buf.size(), cord.str().size()); + EXPECT_EQ(buf, cord.str()); +} + +} // namespace nebula + +#include "common/base/test/CordTestT.cpp" diff --git a/src/common/base/test/CordTestT.cpp b/src/common/base/test/CordTestT.cpp new file mode 100644 index 0000000..0dce606 --- /dev/null +++ b/src/common/base/test/CordTestT.cpp @@ -0,0 +1,190 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" + +// The testing template for Cord/ICord together + +namespace nebula { + +TEST(CordTest, empty) { + Cord cord; + std::string a; + std::string b; + EXPECT_TRUE(cord.empty()); + EXPECT_EQ(0, cord.size()); + EXPECT_EQ(a, cord.str()); + cord.appendTo(b); + EXPECT_EQ(a, b); + EXPECT_TRUE( + cord.applyTo([](const char* s, int32_t len) -> bool { return s == nullptr && len == 0; })); + cord.clear(); +} + +TEST(CordTest, write) { + Cord cord; + + cord.write("cord", 4).write("-", 1).write("test", 4); + + EXPECT_EQ(9, cord.size()); + EXPECT_EQ(9, cord.str().size()); + EXPECT_EQ("cord-test", cord.str()); + + int64_t iVal = 1234567890L; + cord.write(reinterpret_cast(&iVal), sizeof(int64_t)); + + EXPECT_EQ(9 + sizeof(int64_t), cord.size()); + EXPECT_EQ(9 + sizeof(int64_t), cord.str().size()); + iVal = 0; + memcpy(reinterpret_cast(&iVal), cord.str().data() + 9, sizeof(int64_t)); + EXPECT_EQ(1234567890L, iVal); +} + +TEST(CordTest, byteStream) { + Cord cord1; + + cord1 << static_cast('A') << static_cast('b') << 'C' << true; + + EXPECT_EQ(4, cord1.size()); + EXPECT_EQ(4, cord1.str().size()); + EXPECT_EQ("AbC", cord1.str().substr(0, 3)); + + bool bVal = false; + memcpy(reinterpret_cast(&bVal), cord1.str().data() + 3, sizeof(bool)); + EXPECT_EQ(true, bVal); + + uint8_t bytes[] = {0x00, + 0x11, + 0x22, + 0x33, + 0x44, + 0x55, + 0x66, + 0x77, + 0x88, + 0x99, + 0xAA, + 0xBB, + 0xCC, + 0xDD, + 0xEE, + 0xFF}; + Cord cord2; + + cord2.write(reinterpret_cast(&bytes[0]), sizeof(bytes)); + std::string str = cord2.str(); + + EXPECT_EQ(sizeof(bytes), str.size()); + const char* p = str.data(); + for (auto i = 0UL; i < str.size(); i++) { + EXPECT_EQ(bytes[i], uint8_t(p[i])); + } +} + +TEST(CordTest, integerStream) { + Cord cord; + + cord << static_cast(16) << static_cast(0x8080) << static_cast(32) + << static_cast(0xFF00FF00) << static_cast(64) + << static_cast(0xFF11FF22FF33FF44UL); + + EXPECT_EQ(sizeof(int16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(uint32_t) + + sizeof(int64_t) + sizeof(uint64_t), + cord.size()); + EXPECT_EQ(sizeof(int16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(uint32_t) + + sizeof(int64_t) + sizeof(uint64_t), + cord.str().size()); + + int16_t sVal; + uint16_t usVal; + int32_t iVal; + uint32_t uiVal; + int64_t lVal; + uint64_t ulVal; + + std::string str = cord.str(); + memcpy(reinterpret_cast(&sVal), str.data(), sizeof(int16_t)); + memcpy(reinterpret_cast(&usVal), str.data() + sizeof(int16_t), sizeof(uint16_t)); + memcpy(reinterpret_cast(&iVal), + str.data() + sizeof(int16_t) + sizeof(uint16_t), + sizeof(int32_t)); + memcpy(reinterpret_cast(&uiVal), + str.data() + sizeof(int16_t) + sizeof(uint16_t) + sizeof(int32_t), + sizeof(uint32_t)); + memcpy(reinterpret_cast(&lVal), + str.data() + sizeof(int16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(uint32_t), + sizeof(int64_t)); + memcpy(reinterpret_cast(&ulVal), + str.data() + sizeof(int16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(uint32_t) + + sizeof(int64_t), + sizeof(uint64_t)); + + EXPECT_EQ(16, sVal); + EXPECT_EQ(0x8080, usVal); + EXPECT_EQ(32, iVal); + EXPECT_EQ(0xFF00FF00, uiVal); + EXPECT_EQ(64, lVal); + EXPECT_EQ(0xFF11FF22FF33FF44UL, ulVal); +} + +TEST(CordTest, floatStream) { + Cord cord; + + cord << static_cast(1.234) << static_cast(9.876); + + EXPECT_EQ(sizeof(float) + sizeof(double), cord.size()); + EXPECT_EQ(sizeof(float) + sizeof(double), cord.str().size()); + + float fVal; + double dVal; + + std::string str = cord.str(); + memcpy(reinterpret_cast(&fVal), str.data(), sizeof(float)); + memcpy(reinterpret_cast(&dVal), str.data() + sizeof(float), sizeof(double)); + + EXPECT_FLOAT_EQ(1.234, fVal); + EXPECT_DOUBLE_EQ(9.876, dVal); +} + +TEST(CordTest, stringStream) { + std::string str1("Hello"); + char str2[] = "Beautiful"; + std::string str3("World"); + + Cord cord; + + cord << str1 << str2; + cord.write(str3.data(), str3.size()); + + EXPECT_EQ(str1.size() + strlen(str2) + str3.size(), cord.size()); + EXPECT_EQ(str1.size() + strlen(str2) + str3.size(), cord.str().size()); + EXPECT_EQ(str1 + str2 + str3, cord.str()); +} + +TEST(CordTest, cordStream) { + Cord c1; + Cord c2; + + std::string str1("Hello world!"); + std::string str2("Welcome to the future!"); + + c2 << str2; + c1 << str1 << c2; + + EXPECT_EQ(str1.size() + str2.size(), c1.size()); + EXPECT_EQ(str1 + str2, c1.str()); +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/common/base/test/EitherOrTest.cpp b/src/common/base/test/EitherOrTest.cpp new file mode 100644 index 0000000..db985f4 --- /dev/null +++ b/src/common/base/test/EitherOrTest.cpp @@ -0,0 +1,369 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/EitherOr.h" + +namespace nebula { + +TEST(EitherOr, ConstructFromDefault) { + { + EitherOr result; + ASSERT_TRUE(result.isVoid()); + } + { + EitherOr result; + ASSERT_TRUE(result.isVoid()); + } + { + EitherOr result; + ASSERT_TRUE(result.isVoid()); + } +} + +TEST(EitherOr, ConstructFromValue) { + { + EitherOr result(true); + ASSERT_FALSE(result.isVoid()); + ASSERT_TRUE(result.isLeftType()); + EXPECT_TRUE(result.left()); + } + { + EitherOr result("Hello World"); + ASSERT_FALSE(result.isVoid()); + ASSERT_FALSE(result.isLeftType()); + EXPECT_EQ("Hello World", result.right()); + } + { + static char str[] = "Hello World"; + EitherOr result(str); + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ(str, result.left()); + } + { + EitherOr result(6, 'a'); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("aaaaaa", result.right()); + } +} + +TEST(EitherOr, ConstructFromTaggedValue) { + { + // This will cause compile failure + // EitherOr result(1); + EitherOr result(kConstructLeft, 1); + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ(1, result.left()); + } + { + EitherOr result(kConstructRight, 65535); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ(65535U, result.right()); + } +} + +TEST(EitherOr, ReturnFromFunctionCall) { + { + auto foo = []() -> EitherOr { return "Hello World"; }; + + auto result = foo(); + ASSERT_FALSE(result.isVoid()); + ASSERT_FALSE(result.isLeftType()); + EXPECT_EQ("Hello World", result.right()); + } + { + auto foo = []() -> EitherOr { return 101; }; + + auto result = foo(); + ASSERT_FALSE(result.isVoid()); + ASSERT_FALSE(result.isRightType()); + EXPECT_EQ(101, result.left()); + } + { + auto foo = []() -> EitherOr { return static_cast(101); }; + + auto result = foo(); + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ(101, result.left()); + } +} + +TEST(EitherOr, ReturnFromMoveOnlyValue) { + // return move only from anonymous value + { + auto foo = []() -> EitherOr> { + return std::make_unique("SomeValue"); + }; + auto result = foo(); + ASSERT_FALSE(result.isVoid()); + ASSERT_FALSE(result.isLeftType()); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("SomeValue", *result.right()); + } + // return move only from named value + { + auto foo = []() -> EitherOr> { + auto ptr = std::make_unique("SomeValue"); + return ptr; + }; + auto result = foo(); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("SomeValue", *result.right()); + } +} + +TEST(EitherOr, CopyConstructFromDefault) { + EitherOr result1; + auto result2 = result1; + ASSERT_TRUE(result2.isVoid()); +} + +TEST(EitherOr, CopyConstructFromValue) { + { + EitherOr result1(101); + auto result2 = result1; + ASSERT_FALSE(result1.isVoid()); + ASSERT_FALSE(result2.isVoid()); + ASSERT_EQ(101, result1.left()); + ASSERT_EQ(101, result2.left()); + } + { + EitherOr result1("Something"); + auto result2 = result1; + ASSERT_FALSE(result1.isVoid()); + ASSERT_FALSE(result2.isVoid()); + ASSERT_EQ("Something", result1.right()); + ASSERT_EQ("Something", result2.right()); + } + { + EitherOr r1("Hello World"); + EitherOr r2 = r1; + ASSERT_TRUE(r1.isRightType()); + ASSERT_TRUE(r2.isRightType()); + EXPECT_EQ(r1.right(), r2.right()); + } + { + EitherOr r1(256); + EitherOr r2 = r1; + ASSERT_TRUE(r1.isLeftType()); + ASSERT_TRUE(r2.isLeftType()); + EXPECT_EQ(r1.left(), r2.left()); + } +} + +TEST(EitherOr, CopyAssignFromDefault) { + EitherOr result1; + decltype(result1) result2; + result2 = result1; + ASSERT_TRUE(result1.isVoid()); + ASSERT_TRUE(result2.isVoid()); +} + +TEST(EitherOr, CopyAssignFromValue) { + { + EitherOr result1(101); + decltype(result1) result2; + result2 = result1; + ASSERT_FALSE(result1.isVoid()); + ASSERT_FALSE(result2.isVoid()); + ASSERT_TRUE(result1.isLeftType()); + ASSERT_TRUE(result2.isLeftType()); + EXPECT_EQ(101, result1.left()); + EXPECT_EQ(101, result2.left()); + } + { + EitherOr result1("SomeValue"); + decltype(result1) result2; + result2 = result1; + ASSERT_TRUE(result1.isRightType()); + ASSERT_TRUE(result2.isRightType()); + ASSERT_EQ("SomeValue", result1.right()); + ASSERT_EQ("SomeValue", result2.right()); + } + { + EitherOr r1("Hello World"); + EitherOr r2; + r2 = r1; + ASSERT_TRUE(r1.isRightType()); + ASSERT_TRUE(r2.isRightType()); + EXPECT_EQ(r1.right(), r2.right()); + + r1 = 256; + r2 = r1; + ASSERT_TRUE(r1.isLeftType()); + ASSERT_TRUE(r2.isLeftType()); + EXPECT_EQ(r1.left(), r2.left()); + } +} + +TEST(EitherOr, MoveConstructFromDefault) { + EitherOr result1; + auto result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_TRUE(result2.isVoid()); +} + +TEST(EitherOr, MoveConstructFromValue) { + { + EitherOr result1(101); + auto result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_FALSE(result2.isVoid()); + ASSERT_TRUE(result2.isLeftType()); + EXPECT_EQ(101, result2.left()); + } + { + EitherOr result1("SomeValue"); + auto result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_TRUE(result2.isRightType()); + EXPECT_EQ("SomeValue", result2.right()); + } + { + EitherOr r1("Hello World"); + EitherOr r2 = std::move(r1); + ASSERT_TRUE(r1.isVoid()); + ASSERT_TRUE(r2.isRightType()); + EXPECT_EQ("Hello World", r2.right()); + } + { + EitherOr r1(256); + EitherOr r2 = std::move(r1); + ASSERT_TRUE(r1.isVoid()); + ASSERT_TRUE(r2.isLeftType()); + EXPECT_EQ(256, r2.left()); + } +} + +TEST(EitherOr, MoveAssignFromDefault) { + EitherOr result1; + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_TRUE(result2.isVoid()); +} + +TEST(EitherOr, MoveAssignFromValue) { + { + EitherOr result1(101); + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_FALSE(result2.isVoid()); + ASSERT_TRUE(result2.isLeftType()); + EXPECT_EQ(101, result2.left()); + } + { + EitherOr result1("SomeValue"); + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_TRUE(result1.isVoid()); + ASSERT_TRUE(result2.isRightType()); + EXPECT_EQ("SomeValue", result2.right()); + } + { + EitherOr r1("Hello World"); + EitherOr r2; + r2 = std::move(r1); + ASSERT_TRUE(r1.isVoid()); + ASSERT_TRUE(r2.isRightType()); + EXPECT_EQ("Hello World", r2.right()); + + r1 = 256; + r2 = std::move(r1); + ASSERT_TRUE(r1.isVoid()); + ASSERT_TRUE(r2.isLeftType()); + EXPECT_EQ(256, r2.left()); + } +} + +TEST(EitherOr, AssignFromValue) { + { + EitherOr result; + result = 101; + ASSERT_FALSE(result.isVoid()); + EXPECT_EQ(101, result.left()); + + result = "SomeValue"; + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("SomeValue", result.right()); + } + { + EitherOr result; + result = "SomeValue"; + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("SomeValue", result.right()); + + result = 101; + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ(101, result.left()); + } + { + EitherOr, std::string> result; + auto val = std::make_unique("Hello World"); + result = std::move(val); + ASSERT_TRUE(!val); + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ("Hello World", *result.left()); + + auto str = std::string("SomeValue"); + result = std::move(str); + ASSERT_TRUE(str.empty()); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ("SomeValue", result.right()); + } + { + EitherOr result; + result = static_cast(101); + ASSERT_TRUE(result.isLeftType()); + EXPECT_EQ(101, result.left()); + + result = static_cast(202); + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ(202, result.right()); + } +} + +TEST(EitherOr, MoveOutValue) { + EitherOr result("SomeValue"); + ASSERT_FALSE(result.isVoid()); + EXPECT_EQ("SomeValue", result.right()); + auto str = std::move(result).right(); + ASSERT_TRUE(result.isVoid()); + EXPECT_EQ("SomeValue", str); +} + +TEST(EitherOr, SelfAssignment) { + { + EitherOr r("SomeValue"); + r = r; + ASSERT_TRUE(r.isRightType()); + EXPECT_EQ("SomeValue", r.right()); + } + { + EitherOr r("SomeValue"); + r = std::move(r); + ASSERT_TRUE(r.isRightType()); + EXPECT_EQ("SomeValue", r.right()); + } +} + +TEST(EitherOr, Destruct) { + auto ptr = std::make_shared("SomeValue"); + ASSERT_EQ(1UL, ptr.use_count()); + { + EitherOr> result; + result = ptr; + ASSERT_TRUE(result.isRightType()); + EXPECT_EQ(2UL, ptr.use_count()); + auto ptr2 = result.right(); + ASSERT_EQ(3UL, ptr.use_count()); + } + ASSERT_EQ(1UL, ptr.use_count()); +} + +} // namespace nebula diff --git a/src/common/base/test/ErrorOrTest.cpp b/src/common/base/test/ErrorOrTest.cpp new file mode 100644 index 0000000..36014e1 --- /dev/null +++ b/src/common/base/test/ErrorOrTest.cpp @@ -0,0 +1,67 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/ErrorOr.h" + +namespace nebula { + +enum class ErrorCode { + E_1, + E_2, + E_3, + E_4, + E_5, +}; + +TEST(ErrorOr, VerifyOk) { + { + ErrorOr e; + ASSERT_TRUE(ok(e)); + ASSERT_FALSE(hasValue(e)); + } + { + ErrorOr e(ErrorCode::E_2); + ASSERT_FALSE(ok(e)); + ASSERT_FALSE(hasValue(e)); + EXPECT_EQ(ErrorCode::E_2, error(e)); + } +} + +TEST(ErrorOr, RetrieveValue) { + { + const ErrorOr e("Hello World"); + ASSERT_TRUE(ok(e)); + ASSERT_TRUE(hasValue(e)); + ASSERT_TRUE(std::is_const::value); + EXPECT_EQ("Hello World", value(e)); + } + { + ErrorOr e(10); + ASSERT_TRUE(ok(e)); + ASSERT_TRUE(hasValue(e)); + EXPECT_EQ(10, value(e)); + + value(e) = 20; + ASSERT_TRUE(ok(e)); + ASSERT_TRUE(hasValue(e)); + EXPECT_EQ(20, value(e)); + } + { + ErrorOr e(10); + ASSERT_TRUE(ok(e)); + ASSERT_TRUE(hasValue(e)); + EXPECT_EQ(10, value(e)); + + auto res = value(std::move(e)); + EXPECT_EQ(10, res); + ASSERT_TRUE(ok(e)); // void is considered as succeeded + ASSERT_FALSE(hasValue(e)); + } +} + +} // namespace nebula diff --git a/src/common/base/test/HashBenchmark.cpp b/src/common/base/test/HashBenchmark.cpp new file mode 100644 index 0000000..c38da7f --- /dev/null +++ b/src/common/base/test/HashBenchmark.cpp @@ -0,0 +1,143 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include + +#include "common/base/Base.h" +#include "common/base/MurmurHash2.h" + +using nebula::MurmurHash2; + +std::string makeString(size_t size) { + std::string str; + str.resize(size); + for (auto &c : str) { + c = folly::Random::rand32() % (0x7E /*~*/ - 0x21 /*!*/) + 0x21; + } + return str; +} + +size_t StdHashTest(size_t iters, size_t size) { + constexpr size_t ops = 1000000UL; + + std::hash hash; + auto str = makeString(size); + auto i = 0UL; + while (i++ < ops * iters) { + auto hv = hash(str); + folly::doNotOptimizeAway(hv); + } + + return iters * ops; +} + +size_t MurmurHash2Test(size_t iters, size_t size) { + constexpr size_t ops = 1000000UL; + + MurmurHash2 hash; + auto str = makeString(size); + auto i = 0UL; + while (i++ < ops * iters) { + auto hv = hash(str); + folly::doNotOptimizeAway(hv); + } + + return iters * ops; +} + +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 1Byte, 1UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 1Byte, 1UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 2Byte, 2UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 2Byte, 2UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 3Byte, 3UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 3Byte, 3UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 4Byte, 4UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 4Byte, 4UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 5Byte, 5UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 5Byte, 5UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 6Byte, 6UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 6Byte, 6UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 7Byte, 7UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 7Byte, 7UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 8Byte, 8UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 8Byte, 8UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 9Byte, 9UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 9Byte, 9UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 10Byte, 10UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 10Byte, 10UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 64Byte, 64UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 64Byte, 64UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 256Byte, 256UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 256Byte, 256UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 1024Byte, 1024UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 1024Byte, 1024UL) +BENCHMARK_DRAW_LINE(); +BENCHMARK_NAMED_PARAM_MULTI(StdHashTest, 4096Byte, 4096UL) +BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(MurmurHash2Test, 4096Byte, 4096UL) + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + folly::runBenchmarks(); + return 0; +} + +/* Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz +============================================================================ +src/common/base/test/HashBenchmark.cpp relative time/iter iters/s +============================================================================ +StdHashTest(1Byte) 4.53ns 220.75M +MurmurHash2Test(1Byte) 293.53% 1.54ns 647.95M +---------------------------------------------------------------------------- +StdHashTest(2Byte) 5.05ns 198.20M +MurmurHash2Test(2Byte) 271.92% 1.86ns 538.93M +---------------------------------------------------------------------------- +StdHashTest(3Byte) 5.86ns 170.75M +MurmurHash2Test(3Byte) 278.04% 2.11ns 474.76M +---------------------------------------------------------------------------- +StdHashTest(4Byte) 6.65ns 150.37M +MurmurHash2Test(4Byte) 291.51% 2.28ns 438.34M +---------------------------------------------------------------------------- +StdHashTest(5Byte) 7.21ns 138.77M +MurmurHash2Test(5Byte) 270.39% 2.67ns 375.23M +---------------------------------------------------------------------------- +StdHashTest(6Byte) 7.91ns 126.38M +MurmurHash2Test(6Byte) 264.52% 2.99ns 334.30M +---------------------------------------------------------------------------- +StdHashTest(7Byte) 8.75ns 114.26M +MurmurHash2Test(7Byte) 258.92% 3.38ns 295.85M +---------------------------------------------------------------------------- +StdHashTest(8Byte) 4.61ns 216.97M +MurmurHash2Test(8Byte) 173.35% 2.66ns 376.11M +---------------------------------------------------------------------------- +StdHashTest(9Byte) 5.56ns 179.73M +MurmurHash2Test(9Byte) 187.64% 2.97ns 337.25M +---------------------------------------------------------------------------- +StdHashTest(10Byte) 6.13ns 163.24M +MurmurHash2Test(10Byte) 196.97% 3.11ns 321.53M +---------------------------------------------------------------------------- +StdHashTest(64Byte) 12.76ns 78.40M +MurmurHash2Test(64Byte) 117.30% 10.87ns 91.96M +---------------------------------------------------------------------------- +StdHashTest(256Byte) 48.69ns 20.54M +MurmurHash2Test(256Byte) 108.87% 44.72ns 22.36M +---------------------------------------------------------------------------- +StdHashTest(1024Byte) 204.19ns 4.90M +MurmurHash2Test(1024Byte) 98.92% 206.43ns 4.84M +---------------------------------------------------------------------------- +StdHashTest(4096Byte) 825.29ns 1.21M +MurmurHash2Test(4096Byte) 98.29% 839.61ns 1.19M +============================================================================ + */ diff --git a/src/common/base/test/ICordBenchmark.cpp b/src/common/base/test/ICordBenchmark.cpp new file mode 100644 index 0000000..38627d0 --- /dev/null +++ b/src/common/base/test/ICordBenchmark.cpp @@ -0,0 +1,143 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/Cord.h" +#include "common/base/ICord.h" + +using nebula::Cord; +using nebula::ICord; + +BENCHMARK(sstream_10k_string, iters) { + for (auto i = 0u; i < iters; i++) { + std::stringstream ss; + for (int j = 0; j < 1000; j++) { + ss << "abcdefghij"; + } + std::string str = ss.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(icord_10k_string, iters) { + for (auto i = 0u; i < iters; i++) { + ICord<> cord; + for (int j = 0; j < 1000; j++) { + cord << "abcdefghij"; + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(cord_10k_string, iters) { + for (auto i = 0u; i < iters; i++) { + Cord cord; + for (int j = 0; j < 1000; j++) { + cord << "abcdefghij"; + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(sstream_1k_mix, iters) { + for (auto i = 0u; i < iters; i++) { + std::stringstream ss; + for (int j = 0; j < 50; j++) { + ss << "abcdefg" << 1234567890L << true << 1.23456789; + } + std::string str = ss.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(icord_1k_mix, iters) { + for (auto i = 0u; i < iters; i++) { + ICord<> cord; + for (int j = 0; j < 50; j++) { + cord << "abcdefg" << folly::to(1234567890L) << folly::to(true) + << folly::to(1.23456789); + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(cord_1k_mix, iters) { + for (auto i = 0u; i < iters; i++) { + Cord cord; + for (int j = 0; j < 50; j++) { + cord << "abcdefg" << folly::to(1234567890L) << folly::to(true) + << folly::to(1.23456789); + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(sstream_512_mix, iters) { + for (auto i = 0u; i < iters; i++) { + std::stringstream ss; + for (int j = 0; j < 25; j++) { + ss << "abcdefg" << 1234567890L << true << 1.23456789; + } + std::string str = ss.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(icord_512_mix, iters) { + for (auto i = 0u; i < iters; i++) { + ICord<> cord; + for (int j = 0; j < 25; j++) { + cord << "abcdefg" << folly::to(1234567890L) << folly::to(true) + << folly::to(1.23456789); + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} +BENCHMARK_RELATIVE(cord_512_mix, iters) { + for (auto i = 0u; i < iters; i++) { + Cord cord; + for (int j = 0; j < 25; j++) { + cord << "abcdefg" << folly::to(1234567890L) << folly::to(true) + << folly::to(1.23456789); + } + std::string str = cord.str(); + folly::doNotOptimizeAway(&str); + } +} + +int main(int argc, char** argv) { + folly::init(&argc, &argv, true); + + folly::runBenchmarks(); + return 0; +} + +/* +// In Intel(R) Xeon(R) Platinum 8260M CPU @ 2.30GHz +============================================================================ +/root/nebula/src/common/base/test/ICordBenchmark.cpprelative time/iter iters/s +============================================================================ +sstream_10k_string 12.53us 79.81K +icord_10k_string 348.85% 3.59us 278.41K +cord_10k_string 181.23% 6.91us 144.63K +---------------------------------------------------------------------------- +sstream_1k_mix 15.13us 66.09K +icord_1k_mix 54.54% 27.74us 36.05K +cord_1k_mix 53.59% 28.24us 35.42K +---------------------------------------------------------------------------- +sstream_512_mix 7.77us 128.68K +icord_512_mix 55.98% 13.88us 72.04K +cord_512_mix 55.08% 14.11us 70.89K +============================================================================ +*/ + +// In summary, compare to cord improve about 2-3 x performance +// and avoid dynamic allocation when write a little data diff --git a/src/common/base/test/ICordTest.cpp b/src/common/base/test/ICordTest.cpp new file mode 100644 index 0000000..8edfcc2 --- /dev/null +++ b/src/common/base/test/ICordTest.cpp @@ -0,0 +1,34 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/ICord.h" + +#define Cord ICord<> + +namespace nebula { + +TEST(CordTest, multipleBlocks) { + // Disabled + // auto c = new Cord<>; + + ICord<128> cord; + + std::string buf; + for (int i = 0; i < 100; i++) { + buf.append("Hello World!"); + } + + cord.write(buf.data(), buf.size()); + + EXPECT_EQ(buf.size(), cord.size()); + EXPECT_EQ(buf.size(), cord.str().size()); + EXPECT_EQ(buf, cord.str()); +} + +} // namespace nebula + +#include "common/base/test/CordTestT.cpp" diff --git a/src/common/base/test/LoggingBenchmark.cpp b/src/common/base/test/LoggingBenchmark.cpp new file mode 100644 index 0000000..bf13506 --- /dev/null +++ b/src/common/base/test/LoggingBenchmark.cpp @@ -0,0 +1,144 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#define LOG_SOMETHING(iters) \ + for (int64_t i = 0; i < iters; i++) { \ + LOG(INFO) << "Hello" \ + << " " \ + << "World" \ + << "123"; \ + } + +#define XLOG_SOMETHING(iters) \ + for (int64_t i = 0; i < iters; i++) { \ + XLOG(INFO) << "Hello" \ + << " " \ + << "World" \ + << "123"; \ + } + +class XlogInit { + public: + explicit XlogInit(folly::StringPiece config) { + folly::initLogging(config); + } +}; + +static void xlogRegistFileHandler() { + folly::LoggerDB::get().registerHandlerFactory(std::make_unique()); + // Since glog outputs the logs to /tmp by default, so we explicitly set a file handler for + // xlog and output logs to /tmp. + folly::initLogging(";default=file:path=/tmp/logging_bm.log"); +} + +/*************************** + * + * native + * + **************************/ +#include +void loggingUsingGlog(int64_t iters) { + LOG_SOMETHING(iters); +} + +/*************************** + * + * Optimized version + * + **************************/ +#include "common/base/Logging.h" +void loggingOptimized(int64_t iters) { + LOG_SOMETHING(iters); +} + +#include +void loggingUsingXlog(int64_t iters) { + XLOG_SOMETHING(iters); +} + +/*************************** + * + * Run benchmarks + * + **************************/ +BENCHMARK(glog_output_logs, iters) { + FLAGS_minloglevel = 0; + loggingUsingGlog(iters); +} + +BENCHMARK_RELATIVE(optimized_output_logs, iters) { + FLAGS_minloglevel = 0; + loggingOptimized(iters); +} + +BENCHMARK_RELATIVE(xlog_output_logs, iters) { + BENCHMARK_SUSPEND { + static XlogInit init(".=INFO:default"); + } + loggingUsingXlog(iters); +} + +BENCHMARK_RELATIVE(xlog_output_logs_async, iters) { + BENCHMARK_SUSPEND { + static XlogInit init(".=INFO:default;default:async=true,max_buffer_size=4096"); + } + loggingUsingXlog(iters); +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(glog_skip_logs, iters) { + FLAGS_minloglevel = 1; + loggingUsingGlog(iters); +} + +BENCHMARK_RELATIVE(optimized_skip_logs, iters) { + FLAGS_minloglevel = 1; + loggingOptimized(iters); +} + +BENCHMARK_RELATIVE(xlog_skip_logs, iters) { + BENCHMARK_SUSPEND { + static XlogInit init(".=WARN:default"); + } + loggingUsingXlog(iters); +} +/*************************** + * + * main() + * + **************************/ +int main(int argc, char** argv) { + folly::init(&argc, &argv, true); + xlogRegistFileHandler(); + + folly::runBenchmarks(); + return 0; +} + +/* +Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz +============================================================================ +src/common/base/test/LoggingBenchmark.cpp relative time/iter iters/s +============================================================================ +glog_output_logs 1.86us 536.82K +optimized_output_logs 100.26% 1.86us 538.24K +xlog_output_logs 52.73% 3.53us 283.09K +xlog_output_logs_async 53.40% 3.49us 286.68K +---------------------------------------------------------------------------- +glog_skip_logs 1.27us 789.36K +optimized_skip_logs 94753.03% 1.34ns 747.94M +xlog_skip_logs 5215.83% 24.29ns 41.17M +============================================================================ +*/ diff --git a/src/common/base/test/MurmurHash2Test.cpp b/src/common/base/test/MurmurHash2Test.cpp new file mode 100644 index 0000000..b1f6294 --- /dev/null +++ b/src/common/base/test/MurmurHash2Test.cpp @@ -0,0 +1,89 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/MurmurHash2.h" + +namespace nebula { + +TEST(MurmurHash2, Basic) { + MurmurHash2 hash; + // string + { +#define LITERAL "Another one bites the dust" + const char *cstr = LITERAL; + std::string str = cstr; + auto hv1 = hash(LITERAL); + auto hv2 = hash(cstr); + auto hv3 = hash(str); + ASSERT_EQ(hv1, hv2); + ASSERT_EQ(hv2, hv3); + ASSERT_EQ(hv3, std::hash()(str)); + } + // integer + { + bool rand8 = folly::Random::rand64(); + unsigned char rand8_2 = folly::Random::rand64(); + int16_t rand16 = folly::Random::rand64(); + int32_t rand32 = folly::Random::rand64(); + int64_t rand64 = folly::Random::rand64(); + ASSERT_EQ(static_cast(rand8), hash(rand8)); + ASSERT_EQ(static_cast(rand8_2), hash(rand8_2)); + ASSERT_EQ(static_cast(rand16), hash(rand16)); + ASSERT_EQ(static_cast(rand32), hash(rand32)); + ASSERT_EQ(static_cast(rand64), hash(rand64)); + } + // pointer + {{auto *ptr = new MurmurHash2(); + ASSERT_EQ(reinterpret_cast(ptr), hash(ptr)); + delete ptr; +} +{ + auto *ptr = new std::string(); + ASSERT_EQ(reinterpret_cast(ptr), hash(ptr)); + delete ptr; +} +{ + auto *ptr = new int(); + ASSERT_EQ(reinterpret_cast(ptr), hash(ptr)); + delete ptr; +} +} // namespace nebula +// shared_ptr +{{auto ptr = std::make_shared(); +ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +{ + auto ptr = std::make_shared(); + ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +{ + auto ptr = std::make_shared(); + ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +} +// unique_ptr +{{auto ptr = std::make_unique(); +ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +{ + auto ptr = std::make_unique(); + ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +{ + auto ptr = std::make_unique(); + ASSERT_EQ(reinterpret_cast(ptr.get()), hash(ptr)); +} +} +// std::thread::id +{ + auto id = std::this_thread::get_id(); + ASSERT_EQ(std::hash()(id), hash(id)); +} +} + +} // namespace nebula diff --git a/src/common/base/test/ObjectPoolTest.cpp b/src/common/base/test/ObjectPoolTest.cpp new file mode 100644 index 0000000..1bc9fd6 --- /dev/null +++ b/src/common/base/test/ObjectPoolTest.cpp @@ -0,0 +1,36 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/ObjectPool.h" + +namespace nebula { + +static int instances = 0; + +class MyClass { + public: + MyClass() { + instances++; + } + ~MyClass() { + instances--; + } +}; + +TEST(ObjectPoolTest, TestPooling) { + ASSERT_EQ(instances, 0); + + ObjectPool pool; + ASSERT_NE(pool.makeAndAdd(), nullptr); + ASSERT_NE(pool.makeAndAdd(), nullptr); + ASSERT_EQ(instances, 2); + + pool.clear(); + ASSERT_EQ(instances, 0); +} + +} // namespace nebula diff --git a/src/common/base/test/RangeVsTransformBenchmark.cpp b/src/common/base/test/RangeVsTransformBenchmark.cpp new file mode 100644 index 0000000..288e5d5 --- /dev/null +++ b/src/common/base/test/RangeVsTransformBenchmark.cpp @@ -0,0 +1,124 @@ +/* Copyright (c) 2019 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include +#include + +#include +#include + +BENCHMARK(Test1_RangeTestStr) { + std::vector from; + std::vector to; + BENCHMARK_SUSPEND { + from.resize(1000, 0); + to.resize(1000); + } + int32_t index = 0; + for (auto& i : from) { + to[index++] = std::to_string(i); + } + folly::doNotOptimizeAway(to); +} +BENCHMARK_RELATIVE(Test1_TransformStr) { + std::vector from; + std::vector to; + BENCHMARK_SUSPEND { + from.resize(1000, 0); + to.resize(1000); + } + std::transform( + from.begin(), from.end(), to.begin(), [](const auto& e) { return std::to_string(e); }); + folly::doNotOptimizeAway(to); +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(Test2_RangeTestInt) { + std::vector from; + std::vector to; + BENCHMARK_SUSPEND { + from.resize(1000, 0); + to.resize(1000); + } + int32_t index = 0; + for (auto& i : from) { + to[index++] = i; + } + folly::doNotOptimizeAway(to); +} +BENCHMARK_RELATIVE(Test2_TransformInt) { + std::vector from; + std::vector to; + BENCHMARK_SUSPEND { + from.resize(1000, 0); + to.resize(1000); + } + std::transform(from.begin(), from.end(), to.begin(), [](const auto& e) { return e; }); + folly::doNotOptimizeAway(to); +} + +BENCHMARK_DRAW_LINE(); + +BENCHMARK(Test3_RangeTestInt) { + std::vector v; + BENCHMARK_SUSPEND { + v.resize(1000, 0); + } + for (auto& i : v) { + (void)(i); + int a{0}; + folly::doNotOptimizeAway(a); + } +} + +BENCHMARK_RELATIVE(Test3_ForEachInt) { + std::vector v; + BENCHMARK_SUSPEND { + v.resize(1000, 0); + } + std::for_each(v.begin(), v.end(), [](const auto&) { + int a{0}; + folly::doNotOptimizeAway(a); + }); +} + +int main(int argc, char** argv) { + folly::init(&argc, &argv, true); + folly::runBenchmarks(); + return 0; +} + +/* +Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz* + +-O2 +============================================================================ +RangeVsTransformBenchmark.cpprelative time/iter iters/s +============================================================================ +Test1_RangeTestStr 80.60us 12.41K +Test1_TransformStr 100.47% 80.22us 12.47K +---------------------------------------------------------------------------- +Test2_RangeTestInt 713.67ns 1.40M +Test2_TransformInt 99.99% 713.72ns 1.40M +---------------------------------------------------------------------------- +Test3_RangeTestInt 383.15ns 2.61M +Test3_ForEachInt 100.63% 380.75ns 2.63M +============================================================================ + +-O3 +============================================================================ +RangeVsTransformBenchmark.cpprelative time/iter iters/s +============================================================================ +Test1_RangeTestStr 84.19us 11.88K +Test1_TransformStr 100.01% 84.18us 11.88K +---------------------------------------------------------------------------- +Test2_RangeTestInt 222.70ns 4.49M +Test2_TransformInt 98.90% 225.18ns 4.44M +---------------------------------------------------------------------------- +Test3_RangeTestInt 395.93ns 2.53M +Test3_ForEachInt 102.41% 386.61ns 2.59M +============================================================================ + +*/ diff --git a/src/common/base/test/SignalHandlerTest.cpp b/src/common/base/test/SignalHandlerTest.cpp new file mode 100644 index 0000000..30992f7 --- /dev/null +++ b/src/common/base/test/SignalHandlerTest.cpp @@ -0,0 +1,39 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/SignalHandler.h" + +namespace nebula { + +TEST(SignalHandler, Term) { + auto handler = [](auto *info) { + ASSERT_EQ(SIGTERM, info->sig()); + ASSERT_EQ(::getpid(), info->pid()); + ASSERT_EQ(::getuid(), info->uid()); + }; + SignalHandler::install({SIGTERM}, handler); + ::raise(SIGTERM); +} + +TEST(SignalHandler, Pipe) { + // SIGPIPE has been ignored + ::raise(SIGPIPE); +} + +TEST(SignalHandler, Overwrite) { + auto handler = [](auto *info) { + ASSERT_EQ(SIGHUP, info->sig()); + ASSERT_EQ(::getpid(), info->pid()); + ASSERT_EQ(::getuid(), info->uid()); + }; + SignalHandler::install({SIGHUP}, handler); + SignalHandler::install({SIGHUP}, handler); + ::raise(SIGHUP); +} + +} // namespace nebula diff --git a/src/common/base/test/StatusOrTest.cpp b/src/common/base/test/StatusOrTest.cpp new file mode 100644 index 0000000..001f7ce --- /dev/null +++ b/src/common/base/test/StatusOrTest.cpp @@ -0,0 +1,437 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/StatusOr.h" + +namespace nebula { + +TEST(StatusOr, ConstructFromDefault) { + StatusOr result; + ASSERT_FALSE(result.ok()); +} + +TEST(StatusOr, ConstructFromStatus) { + { + StatusOr result(Status::OK()); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().ok()); + ASSERT_EQ("OK", result.status().toString()); + } + { + StatusOr result(Status::Error("SomeError")); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } +} + +TEST(StatusOr, ConstructFromValue) { + StatusOr result("SomeValue"); + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", result.value()); +} + +TEST(StatusOr, ReturnFromStatus) { + { + auto foo = []() -> StatusOr { return Status::Error("SomeError"); }; + + auto result = foo(); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } + { + auto foo = []() -> StatusOr { + auto status = Status::Error("SomeError"); + return status; + }; + + auto result = foo(); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } +} + +TEST(StatusOr, ReturnFromValue) { + auto foo = []() -> StatusOr { return "SomeValue"; }; + auto result = foo(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", result.value()); +} + +TEST(StatusOr, ReturnFromMoveOnlyValue) { + // return move only from anonymous value + { + auto foo = []() -> StatusOr> { + return std::make_unique("SomeValue"); + }; + auto result = foo(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", *result.value()); + } + // return move only from named value + { + auto foo = []() -> StatusOr> { + // TODO(dutor) It seems that GCC before 8.0 has some issues to compile + // the following code, in the combination of NVRO and implicit conversion. + // We use `std::move' explicitly for now + auto ptr = std::make_unique("SomeValue"); + return ptr; + }; + auto result = foo(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", *result.value()); + } + + /* + // return move only from compatible named value + { + auto foo = [] () -> StatusOr> { + auto ptr = std::make_unique("SomeValue"); + // TODO(dutor) It seems that GCC before 8.0 has some issues to compile + // the following code, in the combination of NVRO and implicit + conversion. + // We use `std::move' explicitly for now + return std::move(ptr); + }; + auto result = foo(); + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", *result.value()); + } + */ +} + +TEST(StatusOr, CopyConstructFromDefault) { + StatusOr result1; + auto result2 = result1; + ASSERT_FALSE(result2.ok()); +} + +TEST(StatusOr, CopyConstructFromStatus) { + StatusOr result1(Status::Error("SomeError")); + auto result2 = result1; + ASSERT_FALSE(result1.ok()); + ASSERT_FALSE(result1.status().ok()); + ASSERT_FALSE(result2.ok()); + ASSERT_FALSE(result2.status().ok()); + ASSERT_EQ("SomeError", result1.status().toString()); + ASSERT_EQ("SomeError", result2.status().toString()); +} + +TEST(StatusOr, CopyConstructFromValue) { + StatusOr result1("SomeValue"); + auto result2 = result1; + ASSERT_TRUE(result1.ok()); + ASSERT_TRUE(result2.ok()); + ASSERT_EQ("SomeValue", result1.value()); + ASSERT_EQ("SomeValue", result2.value()); +} + +TEST(StatusOr, CopyAssignFromDefault) { + StatusOr result1; + decltype(result1) result2; + result2 = result1; + ASSERT_FALSE(result2.ok()); +} + +TEST(StatusOr, CopyAssignFromStatus) { + StatusOr result1(Status::Error("SomeError")); + decltype(result1) result2; + result2 = result1; + ASSERT_FALSE(result1.ok()); + ASSERT_FALSE(result1.status().ok()); + ASSERT_FALSE(result2.ok()); + ASSERT_FALSE(result2.status().ok()); + ASSERT_EQ("SomeError", result1.status().toString()); + ASSERT_EQ("SomeError", result2.status().toString()); +} + +TEST(StatusOr, CopyAssignFromValue) { + StatusOr result1("SomeValue"); + decltype(result1) result2; + result2 = result1; + ASSERT_TRUE(result1.ok()); + ASSERT_TRUE(result2.ok()); + ASSERT_EQ("SomeValue", result1.value()); + ASSERT_EQ("SomeValue", result2.value()); +} + +TEST(StatusOr, MoveConstructFromDefault) { + StatusOr result1; + auto result2 = std::move(result1); + ASSERT_FALSE(result2.ok()); +} + +TEST(StatusOr, MoveConstructFromStatus) { + StatusOr result1(Status::Error("SomeError")); + auto result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_FALSE(result2.ok()); + ASSERT_FALSE(result2.status().ok()); + ASSERT_EQ("SomeError", result2.status().toString()); +} + +TEST(StatusOr, MoveConstructFromValue) { + StatusOr result1("SomeValue"); + auto result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_TRUE(result2.ok()); + ASSERT_EQ("SomeValue", result2.value()); +} + +TEST(StatusOr, MoveAssignFromDefault) { + StatusOr result1; + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_FALSE(result2.ok()); +} + +TEST(StatusOr, MoveAssignFromStatus) { + StatusOr result1(Status::Error("SomeError")); + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_FALSE(result2.ok()); + ASSERT_FALSE(result2.status().ok()); + ASSERT_EQ("SomeError", result2.status().toString()); +} + +TEST(StatusOr, MoveAssignFromValue) { + StatusOr result1("SomeValue"); + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_TRUE(result2.ok()); + ASSERT_EQ("SomeValue", result2.value()); +} + +TEST(StatusOr, AssignFromStatus) { + { + StatusOr result; + result = Status::OK(); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().ok()); + + result = Status::Error("SomeError"); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } + { + StatusOr result; + Status status = Status::OK(); + result = status; + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().ok()); + + status = Status::Error("SomeError"); + result = status; + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } + { + StatusOr result; + Status status = Status::OK(); + result = std::move(status); + ASSERT_TRUE(status.ok()); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().ok()); + + status = Status::Error("SomeError"); + result = std::move(status); + ASSERT_TRUE(status.ok()); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ("SomeError", result.status().toString()); + } +} + +TEST(StatusOr, AssignFromValue) { + { + StatusOr result; + result = "SomeValue"; + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", result.value()); + + result = "SomeOtherValue"; + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeOtherValue", result.value()); + } + { + StatusOr result; + std::string value = "SomeValue"; + result = value; + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeValue", result.value()); + + value = "SomeOtherValue"; + result = value; + ASSERT_TRUE(result.ok()); + ASSERT_EQ("SomeOtherValue", result.value()); + } +} + +TEST(StatusOr, CopyConstructFromCompatibleTypes) { + { + StatusOr from("SomeValue"); + StatusOr to(from); + ASSERT_TRUE(to.ok()); + ASSERT_EQ("SomeValue", to.value()); + } + { + { + StatusOr from(0); + ASSERT_EQ(0, from.value()); + StatusOr to(from); + ASSERT_TRUE(to.ok()); + ASSERT_FALSE(to.value()); + } + { + StatusOr from(123); + StatusOr to(from); + ASSERT_TRUE(to.ok()); + ASSERT_TRUE(to.value()); + } + } +} + +TEST(StatusOr, CopyAssignFromCompatibleTypes) { + { + StatusOr from("SomeValue"); + StatusOr to; + to = from; + ASSERT_TRUE(to.ok()); + ASSERT_EQ("SomeValue", to.value()); + } + { + StatusOr from(123); + StatusOr to(false); + ASSERT_TRUE(to.ok()); + ASSERT_FALSE(to.value()); + to = from; + ASSERT_TRUE(to.ok()); + ASSERT_TRUE(to.value()); + } +} + +TEST(StatusOr, MoveConstructFromCompatibleTypes) { + StatusOr> from(std::make_unique("SomeValue")); + ASSERT_TRUE(from.ok()); + ASSERT_EQ("SomeValue", *from.value()); + + StatusOr> to(std::move(from)); + ASSERT_FALSE(from.ok()); + ASSERT_TRUE(to.ok()); + ASSERT_EQ("SomeValue", *to.value()); +} + +TEST(StatusOr, MoveAssignFromCompatibleTypes) { + StatusOr> from(std::make_unique("SomeValue")); + ASSERT_TRUE(from.ok()); + ASSERT_EQ("SomeValue", *from.value()); + + StatusOr> to(std::make_shared("SomeOtherValue")); + ASSERT_TRUE(to.ok()); + ASSERT_EQ("SomeOtherValue", *to.value()); + + to = std::move(from); + ASSERT_FALSE(from.ok()); + ASSERT_TRUE(to.ok()); + ASSERT_EQ("SomeValue", *to.value()); +} + +TEST(StatusOr, MoveOnlyValue) { + { + StatusOr> result1(std::make_unique("SomeValue")); + auto result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_TRUE(result2.ok()); + auto ptr = std::move(result2).value(); + ASSERT_NE(nullptr, ptr.get()); + ASSERT_EQ("SomeValue", *ptr); + ASSERT_FALSE(result2.ok()); + } + { + StatusOr> result1(std::make_unique("SomeValue")); + decltype(result1) result2; + result2 = std::move(result1); + ASSERT_FALSE(result1.ok()); + ASSERT_TRUE(result2.ok()); + auto ptr = std::move(result2).value(); + ASSERT_NE(nullptr, ptr.get()); + ASSERT_EQ("SomeValue", *ptr); + ASSERT_FALSE(result2.ok()); + } +} + +TEST(StatusOr, MoveOutStatus) { + StatusOr result(Status::Error("SomeError")); + ASSERT_FALSE(result.ok()); + ASSERT_FALSE(result.status().ok()); + ASSERT_EQ(Status::Error("SomeError"), result.status()); + auto status = std::move(result).status(); + ASSERT_FALSE(result.ok()); + ASSERT_EQ(Status::Error("SomeError"), status); +} + +TEST(StatusOr, Destruct) { + auto ptr = std::make_shared("SomeValue"); + ASSERT_EQ(1UL, ptr.use_count()); + { + StatusOr> result; + result = ptr; + ASSERT_TRUE(result.ok()); + ASSERT_EQ(2UL, ptr.use_count()); + auto ptr2 = result.value(); + ASSERT_EQ(3UL, ptr.use_count()); + } + ASSERT_EQ(1UL, ptr.use_count()); +} + +/* +TEST(StatusOr, ConstructibleButNotConvertible) { + struct ConstructibleButNotConvertible { + explicit ConstructibleButNotConvertible(double) {} + }; + StatusOr x(1.); +} + + +TEST(StatusOr, ConvertibleButNotConstructible) { + struct ConvertibleButNotConstructible { + explicit ConvertibleButNotConstructible(double) = delete; + ConvertibleButNotConstructible(int) {} + }; + StatusOr x(1.); +} +*/ + +TEST(StatusOr, ReturnIfError) { + auto value = []() -> StatusOr { return 1; }; + auto error = []() -> StatusOr { return Status::Error("error"); }; + auto returnOk = [=]() { + NG_RETURN_IF_ERROR(value()); + return Status::OK(); + }; + auto returnError = [=]() { + NG_RETURN_IF_ERROR(error()); + return Status::OK(); + }; + auto returnError2 = [=]() { + auto err = error(); + NG_RETURN_IF_ERROR(err); + return Status::OK(); + }; + EXPECT_TRUE(returnOk().ok()); + EXPECT_FALSE(returnError().ok()); + EXPECT_FALSE(returnError2().ok()); +} + +} // namespace nebula diff --git a/src/common/base/test/StatusTest.cpp b/src/common/base/test/StatusTest.cpp new file mode 100644 index 0000000..9301948 --- /dev/null +++ b/src/common/base/test/StatusTest.cpp @@ -0,0 +1,120 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/base/Status.h" + +namespace nebula { + +TEST(Status, Basic) { + ASSERT_TRUE(Status().ok()); + ASSERT_TRUE(Status::OK().ok()); + ASSERT_FALSE(Status::Error("Error").ok()); + ASSERT_EQ(8UL, sizeof(Status)); +} + +TEST(Status, toString) { + ASSERT_EQ("OK", Status().toString()); + ASSERT_EQ("OK", Status::OK().toString()); + ASSERT_EQ("Error", Status::Error("Error").toString()); + ASSERT_EQ("SomeError", Status::Error("SomeError").toString()); + ASSERT_EQ("SomeError(-1)", Status::Error("%s(%d)", "SomeError", -1).toString()); + ASSERT_EQ("SyntaxError: message", Status::SyntaxError("message").toString()); +} + +TEST(Status, StreamOperator) { + { + auto result = testing::AssertionSuccess(); + result << Status::OK(); + ASSERT_STREQ("OK", result.message()); + } + { + auto result = testing::AssertionSuccess(); + result << Status::Error("SomeError"); + ASSERT_STREQ("SomeError", result.message()); + } + { + std::ostringstream os; + os << Status(); + ASSERT_EQ("OK", os.str()); + } + { + std::ostringstream os; + os << Status::OK(); + ASSERT_EQ("OK", os.str()); + } + { + std::ostringstream os; + os << Status::Error("SomeError"); + ASSERT_EQ("SomeError", os.str()); + } +} + +TEST(Status, Copy) { + { + Status ok; + auto copy = ok; + ASSERT_TRUE(ok.ok()); + ASSERT_TRUE(copy.ok()); + } + { + auto error = Status::Error("SomeError"); + auto copy = error; + ASSERT_FALSE(error.ok()); + ASSERT_FALSE(copy.ok()); + ASSERT_EQ("SomeError", error.toString()); + ASSERT_EQ("SomeError", copy.toString()); + } +} + +TEST(Status, Move) { + { + Status ok; + auto move = std::move(ok); + ASSERT_TRUE(ok.ok()); + ASSERT_TRUE(move.ok()); + } + { + auto error = Status::Error("SomeError"); + ASSERT_FALSE(error.ok()); + ASSERT_EQ("SomeError", error.toString()); + auto move = std::move(error); + ASSERT_TRUE(error.ok()); + ASSERT_EQ("OK", error.toString()); + ASSERT_FALSE(move.ok()); + ASSERT_EQ("SomeError", move.toString()); + } +} + +TEST(Status, ReturnIfError) { + auto testReturnIfError = []() { + NG_RETURN_IF_ERROR(Status::Error("error")); + return Status::OK(); + }; + auto testReturnOK = []() { + NG_RETURN_IF_ERROR(Status::OK()); + return Status::OK(); + }; + EXPECT_FALSE(testReturnIfError().ok()); + EXPECT_TRUE(testReturnOK().ok()); +} + +TEST(Status, Message) { + Status err = Status::Error("error"); + EXPECT_EQ(err.message(), "error"); + Status syntaxError = Status::SyntaxError(err.message()); + EXPECT_EQ(err.message(), "error"); + EXPECT_EQ(syntaxError.message(), "error"); + EXPECT_EQ(syntaxError.toString(), "SyntaxError: error"); + EXPECT_EQ("some reason", Status::Error("some reason").message()); + EXPECT_EQ("", Status::OK().message()); + std::string msg; + { msg = Status::Error("reason").message(); } + EXPECT_EQ(msg, "reason"); +} + +} // namespace nebula diff --git a/src/common/charset/CMakeLists.txt b/src/common/charset/CMakeLists.txt new file mode 100644 index 0000000..bbc308a --- /dev/null +++ b/src/common/charset/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + charset_obj OBJECT + Charset.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/common/charset/Charset.cpp b/src/common/charset/Charset.cpp new file mode 100644 index 0000000..09fb925 --- /dev/null +++ b/src/common/charset/Charset.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/charset/Charset.h" + +namespace nebula { + +Status CharsetInfo::isSupportCharset(const std::string& charsetName) { + if (supportCharsets_.find(charsetName) == supportCharsets_.end()) { + return Status::Error("Charset `%s' not support", charsetName.c_str()); + } + return Status::OK(); +} + +Status CharsetInfo::isSupportCollate(const std::string& collateName) { + if (supportCollations_.find(collateName) == supportCollations_.end()) { + return Status::Error("Collation `%s' not support", collateName.c_str()); + } + return Status::OK(); +} + +Status CharsetInfo::charsetAndCollateMatch(const std::string& charsetName, + const std::string& collateName) { + auto iter = charsetDesc_.find(charsetName); + if (iter != charsetDesc_.end()) { + for (auto& sc : iter->second.supportColls_) { + if (!sc.compare(collateName)) { + return Status::OK(); + } + } + } + return Status::Error( + "Charset `%s' and Collation `%s' not match", charsetName.c_str(), collateName.c_str()); +} + +StatusOr CharsetInfo::getDefaultCollationbyCharset(const std::string& charsetName) { + auto iter = charsetDesc_.find(charsetName); + if (iter != charsetDesc_.end()) { + return iter->second.defaultColl_; + } + return Status::Error("Charset `%s' not support", charsetName.c_str()); +} + +StatusOr CharsetInfo::getCharsetbyCollation(const std::string& collationName) { + for (auto& cset : charsetDesc_) { + for (auto& coll : cset.second.supportColls_) { + if (!coll.compare(collationName)) { + return cset.first; + } + } + } + return Status::Error("Collation `%s' not support", collationName.c_str()); +} + +} // namespace nebula diff --git a/src/common/charset/Charset.h b/src/common/charset/Charset.h new file mode 100644 index 0000000..1a0d4fa --- /dev/null +++ b/src/common/charset/Charset.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_CHARSET_CHARSET_H_ +#define COMMON_CHARSET_CHARSET_H_ + +#include "common/base/Base.h" +#include "common/base/StatusOr.h" + +namespace nebula { + +struct CharsetDesc { + // Charset name + std::string charsetName_; + // Charset default collation + std::string defaultColl_; + // All collations supported by this charset + std::vector supportColls_; + // Charset description info + std::string desc_; + // Maximum byte number of a character by this charset + int32_t maxLen_; +}; + +class CharsetInfo final { + public: + static CharsetInfo* instance() { + static std::unique_ptr charsetInfo(new CharsetInfo()); + return charsetInfo.get(); + } + + /** + * Check if charset is supported + */ + Status isSupportCharset(const std::string& charsetName); + + /** + * Check if collation is supported + */ + Status isSupportCollate(const std::string& collateName); + + /** + * Check if charset and collation match + */ + Status charsetAndCollateMatch(const std::string& charsetName, const std::string& collateName); + + /** + * Get the corresponding collation according to charset + */ + StatusOr getDefaultCollationbyCharset(const std::string& charsetName); + + /** + * Get the corresponding charset according to collation + */ + StatusOr getCharsetbyCollation(const std::string& collationName); + + /** + * Get all supported charsets description information + */ + std::unordered_map getCharsetDesc() { + return charsetDesc_; + } + + private: + CharsetInfo() { + charsetDesc_["utf8"] = {"utf8", "utf8_bin", {"utf8_bin"}, "UTF-8 Unicode", 4}; + } + + /** + * List of supported charsets + */ + std::unordered_set supportCharsets_ = {"utf8"}; + + /** + * List of supported collations + */ + std::unordered_set supportCollations_ = {"utf8_bin"}; + + /** + * Description information of supported charsets + */ + std::unordered_map charsetDesc_; +}; + +} // namespace nebula + +#endif // COMMON_CHARSET_CHARSET_H_ diff --git a/src/common/charset/test/CMakeLists.txt b/src/common/charset/test/CMakeLists.txt new file mode 100644 index 0000000..3bad906 --- /dev/null +++ b/src/common/charset/test/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_test( + NAME + charset_test + SOURCES + CharsetTest.cpp + OBJECTS + $ + $ + LIBRARIES + gtest + gtest_main +) diff --git a/src/common/charset/test/CharsetTest.cpp b/src/common/charset/test/CharsetTest.cpp new file mode 100644 index 0000000..631e834 --- /dev/null +++ b/src/common/charset/test/CharsetTest.cpp @@ -0,0 +1,96 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/charset/Charset.h" + +namespace nebula { + +TEST(CharsetInfo, isSupportCharset) { + auto* charsetInfo = CharsetInfo::instance(); + { + auto status = charsetInfo->isSupportCharset("utf8"); + ASSERT_TRUE(status.ok()) << status; + } + { + auto status = charsetInfo->isSupportCharset("gbk"); + ASSERT_FALSE(status.ok()); + } +} + +TEST(CharsetInfo, isSupportCollate) { + auto* charsetInfo = CharsetInfo::instance(); + { + auto status = charsetInfo->isSupportCollate("utf8_bin"); + ASSERT_TRUE(status.ok()) << status; + } + { + auto status = charsetInfo->isSupportCollate("utf8"); + ASSERT_FALSE(status.ok()); + } + { + auto status = charsetInfo->isSupportCollate("gbk_bin"); + ASSERT_FALSE(status.ok()); + } +} + +TEST(CharsetInfo, charsetAndCollateMatch) { + auto* charsetInfo = CharsetInfo::instance(); + { + auto status = charsetInfo->charsetAndCollateMatch("utf8", "utf8_bin"); + ASSERT_TRUE(status.ok()) << status; + } + { + auto status = charsetInfo->charsetAndCollateMatch("utf8", "utf8_general_ci"); + ASSERT_FALSE(status.ok()); + } + { + auto status = charsetInfo->charsetAndCollateMatch("gbk", "utf8_bin"); + ASSERT_FALSE(status.ok()); + } +} + +TEST(CharsetInfo, getDefaultCollationbyCharset) { + auto* charsetInfo = CharsetInfo::instance(); + { + auto result = charsetInfo->getDefaultCollationbyCharset("utf8"); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_EQ("utf8_bin", result.value()); + } + { + auto result = charsetInfo->getDefaultCollationbyCharset("utf8mb4"); + ASSERT_FALSE(result.ok()); + } + { + auto result = charsetInfo->getDefaultCollationbyCharset("gbk"); + ASSERT_FALSE(result.ok()); + } +} + +TEST(CharsetInfo, getCharsetbyCollation) { + auto* charsetInfo = CharsetInfo::instance(); + { + auto result = charsetInfo->getCharsetbyCollation("utf8_bin"); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_EQ("utf8", result.value()); + } + { + auto result = charsetInfo->getCharsetbyCollation("utf8mb4_bin"); + ASSERT_FALSE(result.ok()); + } + { + auto result = charsetInfo->getCharsetbyCollation("gbk_bin"); + ASSERT_FALSE(result.ok()); + } +} + +TEST(CharsetInfo, getCharsetDesc) { + auto* charsetInfo = CharsetInfo::instance(); + auto result = charsetInfo->getCharsetDesc(); + EXPECT_EQ(1, result.size()); +} +} // namespace nebula diff --git a/src/common/conf/CMakeLists.txt b/src/common/conf/CMakeLists.txt new file mode 100644 index 0000000..7610486 --- /dev/null +++ b/src/common/conf/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + conf_obj OBJECT + Configuration.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/common/conf/Configuration.cpp b/src/common/conf/Configuration.cpp new file mode 100644 index 0000000..c5d3d7b --- /dev/null +++ b/src/common/conf/Configuration.cpp @@ -0,0 +1,281 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include "common/conf/Configuration.h" + +#include "common/base/Base.h" + +namespace nebula { +namespace conf { + +Configuration::Configuration() { + content_ = std::make_unique(folly::dynamic::object()); +} + +Configuration::Configuration(folly::dynamic content) { + CHECK(content.isObject()) << "The content is not a valid configuration"; + content_ = std::make_unique(std::move(content)); +} + +Status Configuration::parseFromFile(const std::string &filename) { + auto fd = ::open(filename.c_str(), O_RDONLY); + auto status = Status::OK(); + std::string content; + do { + if (fd == -1) { + if (errno == ENOENT) { + status = Status::NoSuchFile("File \"%s\" found", filename.c_str()); + break; + } + if (errno == EPERM) { + status = Status::Error("No permission to read file \"%s\"", filename.c_str()); + break; + } + status = Status::Error("Unknown error"); + break; + } + // get the file size + auto len = ::lseek(fd, 0, SEEK_END); + if (len == 0) { + status = Status::Error("File \"%s\"is empty", filename.c_str()); + break; + } + ::lseek(fd, 0, SEEK_SET); + // read the whole content + // TODO(dutor) ::read might be interrupted by signals + auto buffer = std::make_unique(len + 1); + auto charsRead = ::read(fd, buffer.get(), len); + UNUSED(charsRead); + buffer[len] = '\0'; + // strip off all comments + static const std::regex comment("//.*|#.*"); + content = std::regex_replace(buffer.get(), comment, ""); + } while (false); + + if (fd != -1) { + ::close(fd); + } + + if (!status.ok()) { + return status; + } + + return parseFromString(content); +} + +Status Configuration::parseFromString(const std::string &content) { + try { + auto json = folly::parseJson(content); + content_ = std::make_unique(std::move(json)); + } catch (std::exception &e) { + LOG(ERROR) << e.what(); + return Status::Error("Illegal format (%s)", e.what()); + } + return Status::OK(); +} + +std::string Configuration::dumpToString() const { + std::string json; + if (content_ != nullptr) { + json = folly::toJson(*content_); + } + return json; +} + +std::string Configuration::dumpToPrettyString() const { + std::string json; + if (content_ != nullptr) { + json = folly::toPrettyJson(*content_); + } + return json; +} + +Status Configuration::fetchAsInt(const char *key, int64_t &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isInt()) { + return Status::Error("Item \"%s\" is not an integer", key); + } + val = iter->second.getInt(); + return Status::OK(); +} + +Status Configuration::fetchAsDouble(const char *key, double &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isDouble()) { + return Status::Error("Item \"%s\" is not a double", key); + } + val = iter->second.getDouble(); + return Status::OK(); +} + +Status Configuration::fetchAsBool(const char *key, bool &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isBool()) { + return Status::Error("Item \"%s\" is not a boolean", key); + } + val = iter->second.getBool(); + return Status::OK(); +} + +Status Configuration::fetchAsString(const char *key, std::string &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isString()) { + return Status::Error("Item \"%s\" is not a string", key); + } + val = iter->second.getString(); + return Status::OK(); +} + +Status Configuration::fetchAsSubConf(const char *key, Configuration &subconf) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isObject()) { + return Status::Error("Item \"%s\" is not an JSON object", key); + } + subconf.content_ = std::make_unique(iter->second); + return Status::OK(); +} + +Status Configuration::upsertStringField(const char *key, const std::string &val) { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end() || iter->second.isString()) { + (*content_)[key] = val; + return Status::OK(); + } + return Status::Error("Item \"%s\" not found or it is not an string", key); +} + +Status Configuration::fetchAsIntArray(const char *key, std::vector &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isArray()) { + return Status::Error("Item \"%s\" is not an array", key); + } + + for (auto &entry : iter->second) { + try { + val.emplace_back(entry.asInt()); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +Status Configuration::fetchAsDoubleArray(const char *key, std::vector &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isArray()) { + return Status::Error("Item \"%s\" is not an array", key); + } + + for (auto &entry : iter->second) { + try { + val.emplace_back(entry.asDouble()); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +Status Configuration::fetchAsBoolArray(const char *key, std::vector &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isArray()) { + return Status::Error("Item \"%s\" is not an array", key); + } + + for (auto &entry : iter->second) { + try { + val.emplace_back(entry.asBool()); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +Status Configuration::fetchAsStringArray(const char *key, std::vector &val) const { + DCHECK(content_ != nullptr); + auto iter = content_->find(key); + if (iter == content_->items().end()) { + return Status::Error("Item \"%s\" not found", key); + } + if (!iter->second.isArray()) { + return Status::Error("Item \"%s\" is not an array", key); + } + + for (auto &entry : iter->second) { + try { + val.emplace_back(entry.asString()); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +Status Configuration::forEachKey(std::function processor) const { + DCHECK(content_ != nullptr); + for (auto &key : content_->keys()) { + try { + processor(key.asString()); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +Status Configuration::forEachItem( + std::function processor) const { + DCHECK(content_ != nullptr); + for (auto &item : content_->items()) { + try { + processor(item.first.asString(), item.second); + } catch (const std::exception &ex) { + // Avoid format secure by literal + return Status::Error("%s", ex.what()); + } + } + return Status::OK(); +} + +} // namespace conf +} // namespace nebula diff --git a/src/common/conf/Configuration.h b/src/common/conf/Configuration.h new file mode 100644 index 0000000..ab09925 --- /dev/null +++ b/src/common/conf/Configuration.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#ifndef COMMON_CONF_CONFIGURATION_H_ +#define COMMON_CONF_CONFIGURATION_H_ + +#include "common/base/Base.h" +#include "common/base/Status.h" + +/** + * Configuration loads config infos from a json file or string buffer. + * For a json file, both the C++ and shell style comments are supported. + */ + +namespace nebula { +namespace conf { + +class Configuration final { + public: + Configuration(); + explicit Configuration(folly::dynamic content); + + ~Configuration() = default; + + /** + * Parse from a file + */ + Status NG_MUST_USE_RESULT parseFromFile(const std::string &filename); + /** + * Parse from a string buffer + */ + Status NG_MUST_USE_RESULT parseFromString(const std::string &content); + + std::string dumpToString() const; + + std::string dumpToPrettyString() const; + + /** + * Fetch the config item. + * @key item key + * @val to hold the item value. + */ + Status NG_MUST_USE_RESULT fetchAsInt(const char *key, int64_t &val) const; + Status NG_MUST_USE_RESULT fetchAsDouble(const char *key, double &val) const; + Status NG_MUST_USE_RESULT fetchAsBool(const char *key, bool &val) const; + Status NG_MUST_USE_RESULT fetchAsString(const char *key, std::string &val) const; + + Status NG_MUST_USE_RESULT fetchAsIntArray(const char *key, std::vector &val) const; + Status NG_MUST_USE_RESULT fetchAsDoubleArray(const char *key, std::vector &val) const; + Status NG_MUST_USE_RESULT fetchAsBoolArray(const char *key, std::vector &val) const; + Status NG_MUST_USE_RESULT fetchAsStringArray(const char *key, + std::vector &val) const; + + Status NG_MUST_USE_RESULT fetchAsSubConf(const char *key, Configuration &val) const; + + Status NG_MUST_USE_RESULT upsertStringField(const char *key, const std::string &val); + + // Iterate through every key in the configuration + Status forEachKey(std::function processor) const; + // Iterate through every key/value pair in the configuration + Status forEachItem( + std::function processor) const; + + private: + std::unique_ptr content_; +}; + +} // namespace conf +} // namespace nebula +#endif // COMMON_CONF_CONFIGURATION_H_ diff --git a/src/common/conf/test/CMakeLists.txt b/src/common/conf/test/CMakeLists.txt new file mode 100644 index 0000000..d7ccd43 --- /dev/null +++ b/src/common/conf/test/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_test( + NAME configuration_test + SOURCES ConfigurationTest.cpp + OBJECTS $ $ $ + LIBRARIES gtest gtest_main +) diff --git a/src/common/conf/test/ConfigurationTest.cpp b/src/common/conf/test/ConfigurationTest.cpp new file mode 100644 index 0000000..daa18d3 --- /dev/null +++ b/src/common/conf/test/ConfigurationTest.cpp @@ -0,0 +1,234 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include + +#include "common/base/Base.h" +#include "common/conf/Configuration.h" +#include "common/fs/TempFile.h" + +using nebula::fs::TempFile; + +namespace nebula { +namespace conf { + +TEST(Configuration, Basic) { + std::string json = + "{" + "\"int\": 123, " + "\"double\": 3.14, " + "\"bool\": true, " + "\"string\": \"string\"" + "}"; + Configuration conf; + auto status = conf.parseFromString(json); + ASSERT_TRUE(status.ok()) << status.toString(); + + { + int64_t val; + status = conf.fetchAsInt("int", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_EQ(123, val); + } + { + double val; + status = conf.fetchAsDouble("double", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_DOUBLE_EQ(3.14, val); + } + { + bool val; + status = conf.fetchAsBool("bool", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_TRUE(val); + } + { + std::string val; + status = conf.fetchAsString("string", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_EQ("string", val); + } + + { + double val; + status = conf.fetchAsDouble("int", val); + ASSERT_FALSE(status.ok()) << status.toString(); + } + { + int64_t val; + status = conf.fetchAsInt("string", val); + ASSERT_FALSE(status.ok()) << status.toString(); + } + { + bool val; + status = conf.fetchAsBool("nonexistent", val); + ASSERT_FALSE(status.ok()) << status.toString(); + } +} + +TEST(Configuration, IllegalFormat) { + Configuration conf; + { + std::string empty; + auto status = conf.parseFromString(empty); + ASSERT_FALSE(status.ok()); + } + { + std::string ill = "\"key\": \"value\""; + auto status = conf.parseFromString(ill); + ASSERT_FALSE(status.ok()); + } + { + std::string ill = "{\"key\" \"value\"}"; + auto status = conf.parseFromString(ill); + ASSERT_FALSE(status.ok()); + } + { + std::string ill = "{\"key\": \"value\""; + auto status = conf.parseFromString(ill); + ASSERT_FALSE(status.ok()); + } +} + +TEST(Configuration, SubConf) { + std::string json = + "{" + "\"notsub\": 123," + "\"subconf\": {" + "\"key\": \"value\"" + "}" + "}"; + Configuration conf; + auto status = conf.parseFromString(json); + ASSERT_TRUE(status.ok()) << status.toString(); + + Configuration subconf; + status = conf.fetchAsSubConf("subconf", subconf); + ASSERT_TRUE(status.ok()) << status.toString(); + + std::string val; + status = subconf.fetchAsString("key", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_EQ("value", val); + + status = conf.fetchAsSubConf("notsub", subconf); + ASSERT_FALSE(status.ok()) << status.toString(); +} + +TEST(Configuration, ParseFromFile) { + TempFile tmpfile("/tmp/configuration_test.json.XXXXXX"); + FILE* file = ::fopen(tmpfile.path(), "w"); + fprintf(file, "// This a C++ style comment line\n"); + fprintf(file, "# This a shell style comment line\n"); + fprintf(file, "{ \n"); + fprintf(file, "\t\"int\": 123, // this is an integer\n"); + fprintf(file, "\t\"double\": 3.14 # this is a double\n"); + fprintf(file, "} \n"); + ::fflush(file); + ::fclose(file); + + Configuration conf; + auto status = conf.parseFromFile(tmpfile.path()); + ASSERT_TRUE(status.ok()) << status.toString(); + { + int64_t val; + status = conf.fetchAsInt("int", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_EQ(123, val); + } + { + double val; + status = conf.fetchAsDouble("double", val); + ASSERT_TRUE(status.ok()) << status.toString(); + ASSERT_DOUBLE_EQ(3.14, val); + } +} + +TEST(Configuration, GetArray) { + std::string json = + "{" + " \"int_array\" : [1, \"2\", 3, \"4\", 5]," + " \"string_array\" : [\"Hello\", \"World\"]," + " \"bool_array\" : [0, 1]," + " \"double_array\" : [1.2, \"2.3\", 3.4]" + "}"; + + Configuration conf; + auto status = conf.parseFromString(json); + ASSERT_TRUE(status.ok()) << status.toString(); + + std::vector intArray; + ASSERT_TRUE(conf.fetchAsIntArray("int_array", intArray).ok()); + ASSERT_EQ(5L, intArray.size()); + for (int64_t i = 1; i <= 5; i++) { + EXPECT_EQ(i, intArray[i - 1]); + } + + std::vector strArray; + ASSERT_TRUE(conf.fetchAsStringArray("string_array", strArray).ok()); + ASSERT_EQ(2L, strArray.size()); + EXPECT_EQ("Hello", strArray[0]); + EXPECT_EQ("World", strArray[1]); + + std::vector boolArray; + ASSERT_TRUE(conf.fetchAsBoolArray("bool_array", boolArray).ok()); + ASSERT_EQ(2L, boolArray.size()); + EXPECT_FALSE(boolArray[0]); + EXPECT_TRUE(boolArray[1]); + + std::vector dblArray; + ASSERT_TRUE(conf.fetchAsDoubleArray("double_array", dblArray).ok()); + ASSERT_EQ(3L, dblArray.size()); + EXPECT_DOUBLE_EQ(1.2, dblArray[0]); + EXPECT_DOUBLE_EQ(2.3, dblArray[1]); + EXPECT_DOUBLE_EQ(3.4, dblArray[2]); +} + +TEST(Configuration, Iterator) { + std::string json = + "{" + " \"int_array\" : [1, \"2\", 3, \"4\", 5]," + " \"string_array\" : [\"Hello\", \"World\"]," + " \"int_val\" : 100," + " \"double_val\" : 2.3" + "}"; + + Configuration conf; + auto status = conf.parseFromString(json); + ASSERT_TRUE(status.ok()) << status.toString(); + + static std::unordered_set keys{"int_array", "string_array", "int_val", "double_val"}; + + int32_t idx = 0; + conf.forEachKey([&idx](const std::string& key) { + EXPECT_TRUE(keys.find(key) != keys.end()); + idx++; + }); + EXPECT_EQ(keys.size(), idx); + + idx = 0; + conf.forEachItem([&idx](const std::string& key, const folly::dynamic& val) { + if (key == "int_array") { + ASSERT_TRUE(val.isArray()); + EXPECT_EQ(5L, val.size()); + } else if (key == "string_array") { + ASSERT_TRUE(val.isArray()); + EXPECT_EQ(2L, val.size()); + } else if (key == "int_val") { + ASSERT_TRUE(val.isInt()); + EXPECT_EQ(100, val.asInt()); + } else if (key == "double_val") { + ASSERT_TRUE(val.isDouble()); + EXPECT_DOUBLE_EQ(2.3, val.asDouble()); + } else { + ASSERT_TRUE(false); + } + idx++; + }); + EXPECT_EQ(keys.size(), idx); +} + +} // namespace conf +} // namespace nebula diff --git a/src/common/context/ExpressionContext.h b/src/common/context/ExpressionContext.h new file mode 100644 index 0000000..f53f391 --- /dev/null +++ b/src/common/context/ExpressionContext.h @@ -0,0 +1,94 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_CONTEXT_EXPRESSIONCONTEXT_H_ +#define COMMON_CONTEXT_EXPRESSIONCONTEXT_H_ + +#include + +#include "common/base/Base.h" +#include "common/base/StatusOr.h" +#include "common/datatypes/DataSet.h" +#include "common/datatypes/Value.h" + +namespace nebula { + +/*************************************************************************** + * + * The base class for all ExpressionContext implementations + * + * The context is NOT thread-safe + * + **************************************************************************/ +class ExpressionContext { + public: + virtual ~ExpressionContext() = default; + + // Get the latest version value for the given variable name, such as $a, $b + virtual const Value& getVar(const std::string& var) const = 0; + + // Set the value of innerVar. The innerVar is a variable defined in an expression. + // e.g. ListComprehension + virtual void setInnerVar(const std::string& var, Value val) = 0; + + // Get the value of innerVar. + virtual const Value& getInnerVar(const std::string& var) const = 0; + + // Get the given version value for the given variable name, such as $a, $b + virtual const Value& getVersionedVar(const std::string& var, int64_t version) const = 0; + + // Get the specified property from a variable, such as $a.prop_name + virtual const Value& getVarProp(const std::string& var, const std::string& prop) const = 0; + + // Get index of variable property in tuple + virtual StatusOr getVarPropIndex(const std::string& var, + const std::string& prop) const = 0; + + // Get the specified property from the edge, such as edge_type.prop_name + virtual Value getEdgeProp(const std::string& edgeType, const std::string& prop) const = 0; + + // Get the specified property from the tag, such as tag.prop_name + virtual Value getTagProp(const std::string& tag, const std::string& prop) const = 0; + + // Get the specified property from the source vertex, such as + // $^.tag_name.prop_name + virtual Value getSrcProp(const std::string& tag, const std::string& prop) const = 0; + + // Get the specified property from the destination vertex, such as + // $$.tag_name.prop_name + virtual const Value& getDstProp(const std::string& tag, const std::string& prop) const = 0; + + // Get the specified property from the input, such as $-.prop_name + virtual const Value& getInputProp(const std::string& prop) const = 0; + + // Get index of input property in tuple + virtual StatusOr getInputPropIndex(const std::string& prop) const = 0; + + // Get Vertex + virtual Value getVertex(const std::string& name = "") const = 0; + + // Get Edge + virtual Value getEdge() const = 0; + + // Get Value by Column index + virtual const Value& getColumn(int32_t index) const = 0; + + // Get regex + const std::regex& getRegex(const std::string& pattern) { + auto iter = regex_.find(pattern); + if (iter == regex_.end()) { + iter = regex_.emplace(pattern, std::regex(pattern)).first; + } + return iter->second; + } + + virtual void setVar(const std::string& var, Value val) = 0; + + private: + std::unordered_map regex_; +}; + +} // namespace nebula +#endif // COMMON_CONTEXT_EXPRESSIONCONTEXT_H_ diff --git a/src/common/cpp/helpers.h b/src/common/cpp/helpers.h new file mode 100644 index 0000000..8ae3b05 --- /dev/null +++ b/src/common/cpp/helpers.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2018 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#ifndef COMMON_CPP_HELPERS_H_ +#define COMMON_CPP_HELPERS_H_ + +namespace nebula { +namespace cpp { + +class NonMovable { + public: + NonMovable() = default; + ~NonMovable() = default; + + NonMovable(const NonMovable&) = default; + NonMovable& operator=(const NonMovable&) = default; + + NonMovable(NonMovable&&) = delete; + NonMovable& operator=(NonMovable&&) = delete; +}; + +} // namespace cpp +} // namespace nebula +#endif // COMMON_CPP_HELPERS_H_ diff --git a/src/common/datatypes/CMakeLists.txt b/src/common/datatypes/CMakeLists.txt new file mode 100644 index 0000000..61827f7 --- /dev/null +++ b/src/common/datatypes/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + datatypes_obj OBJECT + Date.cpp + Path.cpp + Value.cpp + HostAddr.cpp + Edge.cpp + Vertex.cpp + Map.cpp + List.cpp + Set.cpp + Geography.cpp + Duration.cpp +) + +nebula_add_subdirectory(test) diff --git a/src/common/datatypes/CommonCpp2Ops.h b/src/common/datatypes/CommonCpp2Ops.h new file mode 100644 index 0000000..bc77347 --- /dev/null +++ b/src/common/datatypes/CommonCpp2Ops.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_COMMON_CPP2OPS_H_ +#define COMMON_DATATYPES_COMMON_CPP2OPS_H_ +#include "common/thrift/ThriftCpp2OpsHelper.h" + +namespace nebula { +struct Value; +struct Vertex; +struct Tag; +struct Edge; +struct Step; +struct Path; +struct HostAddr; +struct KeyValue; +struct Date; +struct DateTime; +struct Time; +struct Map; +struct Set; +struct List; +struct DataSet; +struct Coordinate; +struct Point; +struct LineString; +struct Polygon; +struct Geography; +struct Duration; +} // namespace nebula + +namespace apache::thrift { + +SPECIALIZE_CPP2OPS(nebula::Value); +SPECIALIZE_CPP2OPS(nebula::Vertex); +SPECIALIZE_CPP2OPS(nebula::Tag); +SPECIALIZE_CPP2OPS(nebula::Edge); +SPECIALIZE_CPP2OPS(nebula::Step); +SPECIALIZE_CPP2OPS(nebula::Path); +SPECIALIZE_CPP2OPS(nebula::HostAddr); +SPECIALIZE_CPP2OPS(nebula::KeyValue); +SPECIALIZE_CPP2OPS(nebula::Date); +SPECIALIZE_CPP2OPS(nebula::Time); +SPECIALIZE_CPP2OPS(nebula::DateTime); +SPECIALIZE_CPP2OPS(nebula::Map); +SPECIALIZE_CPP2OPS(nebula::Set); +SPECIALIZE_CPP2OPS(nebula::List); +SPECIALIZE_CPP2OPS(nebula::DataSet); +SPECIALIZE_CPP2OPS(nebula::Coordinate); +SPECIALIZE_CPP2OPS(nebula::Point); +SPECIALIZE_CPP2OPS(nebula::LineString); +SPECIALIZE_CPP2OPS(nebula::Polygon); +SPECIALIZE_CPP2OPS(nebula::Geography); +SPECIALIZE_CPP2OPS(nebula::Duration); + +} // namespace apache::thrift + +#endif // COMMON_DATATYPES_COMMON_CPP2OPS_H_ diff --git a/src/common/datatypes/DataSet.h b/src/common/datatypes/DataSet.h new file mode 100644 index 0000000..600ef23 --- /dev/null +++ b/src/common/datatypes/DataSet.h @@ -0,0 +1,226 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DATASET_H_ +#define COMMON_DATATYPES_DATASET_H_ + +#include + +#include +#include +#include +#include + +#include "common/datatypes/List.h" +#include "common/datatypes/Value.h" + +namespace nebula { + +using Row = List; + +struct DataSet { + std::vector colNames; + std::vector rows; + + DataSet() = default; + explicit DataSet(std::vector columns) : colNames(std::move(columns)) {} + DataSet(const DataSet& ds) { + colNames = ds.colNames; + rows = ds.rows; + } + DataSet(DataSet&& ds) noexcept { + colNames = std::move(ds.colNames); + rows = std::move(ds.rows); + } + DataSet& operator=(const DataSet& ds) { + if (&ds != this) { + colNames = ds.colNames; + rows = ds.rows; + } + return *this; + } + DataSet& operator=(DataSet&& ds) noexcept { + if (&ds != this) { + colNames = std::move(ds.colNames); + rows = std::move(ds.rows); + } + return *this; + } + + const std::vector& keys() const { + return colNames; + } + + const std::vector& rowValues(std::size_t index) const { + return rows[index].values; + } + + std::vector colValues(const std::string& colName) const { + std::vector col; + const auto find = std::find(colNames.begin(), colNames.end(), colName); + if (find == colNames.end()) { + return col; + } + std::size_t index = std::distance(colNames.begin(), find); + col.reserve(rows.size()); + for (const auto& row : rows) { + col.emplace_back(row.values[index]); + } + return col; + } + + using iterator = std::vector::iterator; + using const_iterator = std::vector::const_iterator; + + iterator begin() { + return rows.begin(); + } + + const_iterator begin() const { + return rows.begin(); + } + + iterator end() { + return rows.end(); + } + + const_iterator end() const { + return rows.end(); + } + + template ::value, T>::type> + bool emplace_back(T&& row) { + if (row.size() != colNames.size()) { + return false; + } + rows.emplace_back(std::forward(row)); + return true; + } + + // append the DataSet to one with same header + bool append(DataSet&& o) { + if (colNames.empty()) { + colNames = std::move(o.colNames); + } else { + if (colNames != o.colNames) { + return false; + } + } + rows.reserve(rowSize() + o.rowSize()); + rows.insert( + rows.end(), std::make_move_iterator(o.rows.begin()), std::make_move_iterator(o.rows.end())); + std::vector().swap(o.rows); + return true; + } + + // merge two DataSet Horizontally with same row count + bool merge(DataSet&& o) { + if (rowSize() != o.rowSize()) { + return false; + } + auto newColSize = colSize() + o.colSize(); + colNames.reserve(newColSize); + colNames.insert(colNames.end(), + std::make_move_iterator(o.colNames.begin()), + std::make_move_iterator(o.colNames.end())); + for (std::size_t i = 0; i < rowSize(); ++i) { + rows[i].values.reserve(newColSize); + rows[i].values.insert(rows[i].values.end(), + std::make_move_iterator(o.rows[i].values.begin()), + std::make_move_iterator(o.rows[i].values.end())); + } + return true; + } + + void clear() { + colNames.clear(); + rows.clear(); + } + + void __clear() { + clear(); + } + + std::size_t size() const { + return rowSize(); + } + + std::size_t rowSize() const { + return rows.size(); + } + + std::size_t colSize() const { + return colNames.size(); + } + + std::string toString() const { + std::stringstream os; + // header + for (const auto& h : colNames) { + os << h << "|"; + } + os << std::endl; + + // body + for (const auto& row : rows) { + for (const auto& col : row.values) { + os << col << "|"; + } + os << std::endl; + } + os << std::endl; + return os.str(); + } + + // format: + // [ + // { + // "row": [ row-data ], + // "meta": [ metadata ] + // }, + // ] + folly::dynamic toJson() const { + // parse rows to json + auto dataBody = folly::dynamic::array(); + for (auto& row : rows) { + dataBody.push_back(rowToJson(row)); + } + + return dataBody; + } + + // parse Nebula::Row to json + // format: + // { + // "row": [ row-data ], + // "meta": [ metadata ] + // } + folly::dynamic rowToJson(const Row& row) const { + folly::dynamic rowJsonObj = folly::dynamic::object(); + auto rowDataList = folly::dynamic::array(); + auto metaDataList = folly::dynamic::array(); + + for (const auto& ele : row.values) { + rowDataList.push_back(ele.toJson()); + metaDataList.push_back(ele.getMetaData()); + } + + rowJsonObj.insert("row", rowDataList); + rowJsonObj.insert("meta", metaDataList); + return rowJsonObj; + } + + bool operator==(const DataSet& rhs) const { + return colNames == rhs.colNames && rows == rhs.rows; + } +}; + +inline std::ostream& operator<<(std::ostream& os, const DataSet& d) { + return os << d.toString(); +} + +} // namespace nebula +#endif // COMMON_DATATYPES_DATASET_H_ diff --git a/src/common/datatypes/DataSetOps-inl.h b/src/common/datatypes/DataSetOps-inl.h new file mode 100644 index 0000000..89207de --- /dev/null +++ b/src/common/datatypes/DataSetOps-inl.h @@ -0,0 +1,198 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DATASETOPS_H_ +#define COMMON_DATATYPES_DATASETOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/DataSet.h" +#include "common/memory/MemoryTracker.h" + +namespace apache { +namespace thrift { + +/************************************** + * + * Ops for class DataSet + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "column_names") { + fid = 1; + _ftype = apache::thrift::protocol::T_LIST; + } else if (_fname == "rows") { + fid = 2; + _ftype = apache::thrift::protocol::T_LIST; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::DataSet const* obj) { + // we do not turn on memory tracker here, when the DataSet object is creating & inserting, it is + // in Processor::process(), where memory tracker is turned on. so we think that is enough. + uint32_t xfer = 0; + + xfer += proto->writeStructBegin("DataSet"); + + xfer += proto->writeFieldBegin("column_names", protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods, + std::vector>::write(*proto, obj->colNames); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("rows", apache::thrift::protocol::T_LIST, 2); + xfer += detail::pm::protocol_methods, + std::vector>::write(*proto, obj->rows); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::DataSet* obj) { + // memory usage during decode a StorageResponse should be mostly occupied + // by DataSet (see interface/storage.thrift), turn on memory check here. + // + // MemoryTrackerVerified: + // throw std::bad_alloc has verified, can be captured in + // StorageClientBase::getResponse's onError + nebula::memory::MemoryCheckGuard guard; + + apache::thrift::detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_LIST))) { + goto _loop; + } + +_readField_column_names : { + obj->colNames = std::vector(); + detail::pm::protocol_methods, + std::vector>::read(*proto, obj->colNames); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_LIST))) { + goto _loop; + } + +_readField_rows : { + obj->rows = std::vector(); + detail::pm::protocol_methods, + std::vector>::read(*proto, obj->rows); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_column_names; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_rows; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::DataSet const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("DataSet"); + + xfer += proto->serializedFieldSize("column_names", protocol::T_LIST, 1); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->colNames); + + xfer += proto->serializedFieldSize("rows", apache::thrift::protocol::T_LIST, 2); + xfer += detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->rows); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::DataSet const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("DataSet"); + + xfer += proto->serializedFieldSize("column_names", protocol::T_LIST, 1); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->colNames); + + xfer += proto->serializedFieldSize("rows", protocol::T_LIST, 2); + xfer += detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->rows); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache + +#endif // COMMON_DATATYPES_DATASETOPS_H_ diff --git a/src/common/datatypes/Date.cpp b/src/common/datatypes/Date.cpp new file mode 100644 index 0000000..7742442 --- /dev/null +++ b/src/common/datatypes/Date.cpp @@ -0,0 +1,363 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Date.h" + +#include +#include + +#include + +namespace nebula { + +const int64_t kDaysSoFar[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; +const int64_t kLeapDaysSoFar[] = {0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366}; + +int8_t dayOfMonth(int16_t year, int8_t month) { + return isLeapYear(year) ? kLeapDaysSoFar[month] - kLeapDaysSoFar[month - 1] + : kDaysSoFar[month] - kDaysSoFar[month - 1]; +} + +Date::Date(uint64_t days) { + fromInt(days); +} + +int64_t Date::toInt() const { + // Year + int64_t yearsPassed = year + 32768L; + int64_t days = yearsPassed * 365L; + // Add one day per leap year + if (yearsPassed > 0) { + days += (yearsPassed - 1) / 4 + 1; + } + + // Month + if (yearsPassed % 4 == 0) { + // Leap year + days += kLeapDaysSoFar[month - 1]; + } else { + days += kDaysSoFar[month - 1]; + } + + // Day + days += day; + + // Since we start from -32768/1/1, we need to reduce one day + return days - 1; +} + +void Date::fromInt(int64_t days) { + // year + int64_t yearsPassed = (days + 1) / 365; + year = yearsPassed - 32768; + int64_t daysInYear = (days + 1) % 365; + + // Deduce the number of days for leap years + if (yearsPassed > 0) { + daysInYear -= (yearsPassed - 1) / 4 + 1; + } + + // Adjust the year if necessary + while (daysInYear <= 0) { + year = year - 1; + if (year % 4 == 0) { + // Leap year + daysInYear += 366; + } else { + daysInYear += 365; + } + } + + // Month and day + month = 1; + while (true) { + if (year % 4 == 0) { + // Leap year + if (daysInYear <= kLeapDaysSoFar[month]) { + day = daysInYear - kLeapDaysSoFar[month - 1]; + break; + } + } else { + if (daysInYear <= kDaysSoFar[month]) { + day = daysInYear - kDaysSoFar[month - 1]; + break; + } + } + month++; + } +} + +Date Date::operator+(int64_t days) const { + int64_t daysSince = toInt(); + return Date(daysSince + days); +} + +Date Date::operator-(int64_t days) const { + int64_t daysSince = toInt(); + return Date(daysSince - days); +} + +void Date::addDuration(const Duration& duration) { + int64_t tmp{0}, carry{0}; + tmp = month + duration.months; + if (std::abs(tmp) > 12) { + carry = tmp / 12; + month = tmp % 12; + } else { + month = tmp; + } + if (month <= 0) { + carry--; + month += 12; + } + year += carry; + + tmp = day + duration.days(); + if (tmp > 0) { + int8_t dom = dayOfMonth(year, month); + while (tmp > dom) { + tmp -= dom; + month += 1; + if (month > 12) { + year += 1; + month = 1; + } + dom = dayOfMonth(year, month); + } + } else { + int8_t dom = (month == 1 ? dayOfMonth(year - 1, 12) : dayOfMonth(year, month - 1)); + while (tmp <= 0) { + tmp += dom; + month -= 1; + if (month <= 0) { + year--; + month += 12; + } + dom = (month == 1 ? dayOfMonth(year - 1, 12) : dayOfMonth(year, month - 1)); + } + } + day = tmp; +} + +void Date::subDuration(const Duration& duration) { + return addDuration(-duration); +} + +std::string Date::toString() const { + // It's in current timezone already + return folly::stringPrintf("%d-%02d-%02d", year, month, day); +} + +void Time::addDuration(const Duration& duration) { + int64_t tmp{0}, carry{0}; + tmp = microsec + duration.microsecondsInSecond(); + if (std::abs(tmp) >= 1000000) { + carry = tmp / 1000000; + microsec = tmp % 1000000; + } else { + microsec = tmp; + } + if (microsec < 0) { + carry--; + microsec += 1000000; + } + tmp = sec + duration.seconds + carry; + carry = 0; + if (std::abs(tmp) >= 60) { + carry = tmp / 60; + sec = tmp % 60; + } else { + sec = tmp; + } + if (sec < 0) { + carry--; + sec += 60; + } + tmp = minute + carry; + carry = 0; + if (std::abs(tmp) >= 60) { + carry = tmp / 60; + minute = tmp % 60; + } else { + minute = tmp; + } + if (minute < 0) { + carry--; + minute += 60; + } + tmp = hour + carry; + carry = 0; + if (std::abs(tmp) >= 24) { + carry = tmp / 24; + hour = tmp % 24; + } else { + hour = tmp; + } + if (hour < 0) { + carry--; + hour += 24; + } +} + +void Time::subDuration(const Duration& duration) { + addDuration(-duration); +} + +std::string Time::toString() const { + return folly::sformat("{:0>2}:{:0>2}:{:0>2}.{:0>6}000", + static_cast(hour), + static_cast(minute), + static_cast(sec), + static_cast(microsec)); +} + +void DateTime::addDuration(const Duration& duration) { + // The origin fields of DateTime is unsigned, but there will be some negative intermediate results + // so I define some variable(field, tYear, tMonth) for this. + int64_t tmp{0}, carry{0}, field{0}; + tmp = month + duration.months; + if (std::abs(tmp) > 12) { + carry = tmp / 12; + /*month*/ field = tmp % 12; + } else { + /*month*/ field = tmp; + } + if (/*month*/ field <= 0) { + carry--; + /*month*/ field += 12; + } + year += carry; + carry = 0; + month = field; + field = 0; + + tmp = microsec + duration.microsecondsInSecond(); + if (std::abs(tmp) >= 1000000) { + carry = tmp / 1000000; + /*microsec*/ field = tmp % 1000000; + } else { + /*microsec*/ field = tmp; + } + if (/*microsec*/ field < 0) { + carry--; + /*microsec*/ field += 1000000; + } + microsec = field; + field = 0; + tmp = sec + duration.seconds + carry; + carry = 0; + if (std::abs(tmp) >= 60) { + carry = tmp / 60; + /*sec*/ field = tmp % 60; + } else { + /*sec*/ field = tmp; + } + if (/*sec*/ field < 0) { + carry--; + /*sec*/ field += 60; + } + sec = field; + field = 0; + tmp = minute + carry; + carry = 0; + if (std::abs(tmp) >= 60) { + carry = tmp / 60; + /*minute*/ field = tmp % 60; + } else { + /*minute*/ field = tmp; + } + if (/*minute*/ field < 0) { + carry--; + /*minute*/ field += 60; + } + minute = field; + field = 0; + tmp = hour + carry; + carry = 0; + if (std::abs(tmp) >= 24) { + carry = tmp / 24; + /*hour*/ field = tmp % 24; + } else { + /*hour*/ field = tmp; + } + if (/*hour*/ field < 0) { + carry--; + /*hour*/ field += 24; + } + hour = field; + field = 0; + + tmp = day + carry; + carry = 0; + if (tmp > 0) { + int8_t dom = dayOfMonth(year, month); + while (tmp > dom) { + tmp -= dom; + month += 1; + if (month > 12) { + year += 1; + month = 1; + } + dom = dayOfMonth(year, month); + } + } else { + int8_t dom = (month == 1 ? dayOfMonth(year - 1, 12) : dayOfMonth(year, month - 1)); + int64_t tMonth = month; + int64_t tYear = year; + while (tmp <= 0) { + tmp += dom; + /*month*/ tMonth -= 1; + if (/*month*/ tMonth <= 0) { + /*year*/ tYear--; + /*month*/ tMonth += 12; + } + dom = (/*month*/ tMonth == 1 ? dayOfMonth(/*year*/ tYear - 1, 12) + : dayOfMonth(/*year*/ tYear, /*month*/ tMonth - 1)); + } + month = tMonth; + year = tYear; + } + day = tmp; +} + +void DateTime::subDuration(const Duration& duration) { + return addDuration(-duration); +} + +std::string DateTime::toString() const { + // It's in current timezone already + return folly::sformat("{}-{:0>2}-{:0>2}T{:0>2}:{:0>2}:{:0>2}.{:0>6}000", + static_cast(year), + static_cast(month), + static_cast(day), + static_cast(hour), + static_cast(minute), + static_cast(sec), + static_cast(microsec)); +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +std::size_t hash::operator()(const nebula::Date& h) const noexcept { + size_t hv = folly::hash::fnv64_buf(reinterpret_cast(&h.year), sizeof(h.year)); + hv = folly::hash::fnv64_buf(reinterpret_cast(&h.month), sizeof(h.month), hv); + return folly::hash::fnv64_buf(reinterpret_cast(&h.day), sizeof(h.day), hv); +} + +std::size_t hash::operator()(const nebula::Time& h) const noexcept { + std::size_t hv = folly::hash::fnv64_buf(reinterpret_cast(&h.hour), sizeof(h.hour)); + hv = folly::hash::fnv64_buf(reinterpret_cast(&h.minute), sizeof(h.minute), hv); + hv = folly::hash::fnv64_buf(reinterpret_cast(&h.sec), sizeof(h.sec), hv); + return folly::hash::fnv64_buf(reinterpret_cast(&h.microsec), sizeof(h.microsec), hv); +} + +std::size_t hash::operator()(const nebula::DateTime& h) const noexcept { + return h.qword; +} + +} // namespace std diff --git a/src/common/datatypes/Date.h b/src/common/datatypes/Date.h new file mode 100644 index 0000000..bdb5e2f --- /dev/null +++ b/src/common/datatypes/Date.h @@ -0,0 +1,331 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DATE_H_ +#define COMMON_DATATYPES_DATE_H_ + +#include + +#include + +#include "common/datatypes/Duration.h" + +namespace nebula { + +// In nebula only store UTC time, and the interpretation of time value based on +// the timezone configuration in current system. + +extern const int64_t kDaysSoFar[]; +extern const int64_t kLeapDaysSoFar[]; + +// https://en.wikipedia.org/wiki/Leap_year#Leap_day +static inline bool isLeapYear(int16_t year) { + if (year % 4 != 0) { + return false; + } else if (year % 100 != 0) { + return true; + } else if (year % 400 != 0) { + return false; + } else { + return true; + } +} + +int8_t dayOfMonth(int16_t year, int8_t month); + +struct Date { + int16_t year; // Any integer + int8_t month; // 1 - 12 + int8_t day; // 1 - 31 + + Date() : year{0}, month{1}, day{1} {} + Date(int16_t y, int8_t m, int8_t d) : year{y}, month{m}, day{d} {} + // Tak the number of days since -32768/1/1, and convert to the real date + explicit Date(uint64_t days); + + void clear() { + year = 0; + month = 1; + day = 1; + } + + void __clear() { + clear(); + } + + void reset(int16_t y, int8_t m, int8_t d) { + year = y; + month = m; + day = d; + } + + bool operator==(const Date& rhs) const { + return year == rhs.year && month == rhs.month && day == rhs.day; + } + + bool operator<(const Date& rhs) const { + if (!(year == rhs.year)) { + return year < rhs.year; + } + if (!(month == rhs.month)) { + return month < rhs.month; + } + if (!(day == rhs.day)) { + return day < rhs.day; + } + return false; + } + + Date operator+(int64_t days) const; + Date operator-(int64_t days) const; + + void addDuration(const Duration& duration); + void subDuration(const Duration& duration); + + std::string toString() const; + folly::dynamic toJson() const { + return toString(); + } + + // Return the number of days since -32768/1/1 + int64_t toInt() const; + // Convert the number of days since -32768/1/1 to the real date + void fromInt(int64_t days); +}; + +inline Date operator+(const Date& l, const Duration& r) { + Date d = l; + d.addDuration(r); + return d; +} + +inline Date operator-(const Date& l, const Duration& r) { + Date d = l; + d.subDuration(r); + return d; +} + +inline std::ostream& operator<<(std::ostream& os, const Date& d) { + os << d.toString(); + return os; +} + +struct Time { + int8_t hour; + int8_t minute; + int8_t sec; + int32_t microsec; + + Time() : hour{0}, minute{0}, sec{0}, microsec{0} {} + Time(int8_t h, int8_t min, int8_t s, int32_t us) : hour{h}, minute{min}, sec{s}, microsec{us} {} + + void clear() { + hour = 0; + minute = 0; + sec = 0; + microsec = 0; + } + + void __clear() { + clear(); + } + + bool operator==(const Time& rhs) const { + return hour == rhs.hour && minute == rhs.minute && sec == rhs.sec && microsec == rhs.microsec; + } + + bool operator<(const Time& rhs) const { + if (!(hour == rhs.hour)) { + return hour < rhs.hour; + } + if (!(minute == rhs.minute)) { + return minute < rhs.minute; + } + if (!(sec == rhs.sec)) { + return sec < rhs.sec; + } + if (!(microsec == rhs.microsec)) { + return microsec < rhs.microsec; + } + return false; + } + + void addDuration(const Duration& duration); + void subDuration(const Duration& duration); + + std::string toString() const; + // 'Z' representing UTC timezone + folly::dynamic toJson() const { + return toString() + "Z"; + } +}; + +inline std::ostream& operator<<(std::ostream& os, const Time& d) { + os << d.toString(); + return os; +} + +inline Time operator+(const Time& l, const Duration& r) { + Time t = l; + t.addDuration(r); + return t; +} + +inline Time operator-(const Time& l, const Duration& r) { + Time t = l; + t.subDuration(r); + return t; +} + +struct DateTime { +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#endif // defined(__GNUC__) + union { + struct { + int64_t year : 16; + uint64_t month : 4; + uint64_t day : 5; + uint64_t hour : 5; + uint64_t minute : 6; + uint64_t sec : 6; + uint64_t microsec : 22; + }; + uint64_t qword; + }; +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif // defined(__GNUC__) + + DateTime() : year{0}, month{1}, day{1}, hour{0}, minute{0}, sec{0}, microsec{0} {} + DateTime(int16_t y, int8_t m, int8_t d, int8_t h, int8_t min, int8_t s, int32_t us) { + year = y; + month = m; + day = d; + hour = h; + minute = min; + sec = s; + microsec = us; + } + explicit DateTime(const Date& date) { + year = date.year; + month = date.month; + day = date.day; + hour = 0; + minute = 0; + sec = 0; + microsec = 0; + } + DateTime(const Date& date, const Time& time) { + year = date.year; + month = date.month; + day = date.day; + hour = time.hour; + minute = time.minute; + sec = time.sec; + microsec = time.microsec; + } + + Date date() const { + return Date(year, month, day); + } + + Time time() const { + return Time(hour, minute, sec, microsec); + } + + void clear() { + year = 0; + month = 1; + day = 1; + hour = 0; + minute = 0; + sec = 0; + microsec = 0; + } + + void __clear() { + clear(); + } + + bool operator==(const DateTime& rhs) const { + return year == rhs.year && month == rhs.month && day == rhs.day && hour == rhs.hour && + minute == rhs.minute && sec == rhs.sec && microsec == rhs.microsec; + } + bool operator<(const DateTime& rhs) const { + if (!(year == rhs.year)) { + return year < rhs.year; + } + if (!(month == rhs.month)) { + return month < rhs.month; + } + if (!(day == rhs.day)) { + return day < rhs.day; + } + if (!(hour == rhs.hour)) { + return hour < rhs.hour; + } + if (!(minute == rhs.minute)) { + return minute < rhs.minute; + } + if (!(sec == rhs.sec)) { + return sec < rhs.sec; + } + if (!(microsec == rhs.microsec)) { + return microsec < rhs.microsec; + } + return false; + } + + void addDuration(const Duration& duration); + void subDuration(const Duration& duration); + + std::string toString() const; + // 'Z' representing UTC timezone + folly::dynamic toJson() const { + return toString() + "Z"; + } +}; + +inline std::ostream& operator<<(std::ostream& os, const DateTime& d) { + os << d.toString(); + return os; +} + +inline DateTime operator+(const DateTime& l, const Duration& r) { + DateTime dt = l; + dt.addDuration(r); + return dt; +} + +inline DateTime operator-(const DateTime& l, const Duration& r) { + DateTime dt = l; + dt.subDuration(r); + return dt; +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const nebula::Date& h) const noexcept; +}; + +template <> +struct hash { + std::size_t operator()(const nebula::Time& h) const noexcept; +}; + +template <> +struct hash { + std::size_t operator()(const nebula::DateTime& h) const noexcept; +}; + +} // namespace std +#endif // COMMON_DATATYPES_DATE_H_ diff --git a/src/common/datatypes/DateOps-inl.h b/src/common/datatypes/DateOps-inl.h new file mode 100644 index 0000000..e8f92ba --- /dev/null +++ b/src/common/datatypes/DateOps-inl.h @@ -0,0 +1,723 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DATEOPS_H_ +#define COMMON_DATATYPES_DATEOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Date.h" + +namespace apache { +namespace thrift { + +/************************************** + * + * Ops for class Date + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "year") { + fid = 1; + _ftype = apache::thrift::protocol::T_I16; + } else if (_fname == "month") { + fid = 2; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "day") { + fid = 3; + _ftype = apache::thrift::protocol::T_BYTE; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Date const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Date"); + + xfer += proto->writeFieldBegin("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::write(*proto, obj->year); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("month", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::write(*proto, obj->month); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::write(*proto, obj->day); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Date* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_I16))) { + goto _loop; + } + +_readField_year : { + detail::pm::protocol_methods::read(*proto, obj->year); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_BYTE))) { + goto _loop; + } + +_readField_month : { + detail::pm::protocol_methods::read(*proto, obj->month); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_BYTE))) { + goto _loop; + } + +_readField_day : { + detail::pm::protocol_methods::read(*proto, obj->day); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I16)) { + goto _readField_year; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_month; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_day; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Date const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Date"); + + xfer += proto->serializedFieldSize("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->year); + + xfer += proto->serializedFieldSize("month", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->month); + + xfer += proto->serializedFieldSize("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->day); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Date const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Date"); + + xfer += proto->serializedFieldSize("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->year); + + xfer += proto->serializedFieldSize("month", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->month); + + xfer += proto->serializedFieldSize("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->day); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for class Time + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "hour") { + fid = 1; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "minute") { + fid = 2; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "sec") { + fid = 3; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "microsec") { + fid = 4; + _ftype = apache::thrift::protocol::T_I32; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Time const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Time"); + + xfer += proto->writeFieldBegin("hour", apache::thrift::protocol::T_BYTE, 1); + xfer += detail::pm::protocol_methods::write(*proto, obj->hour); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("minute", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::write(*proto, obj->minute); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("sec", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::write(*proto, obj->sec); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("microsec", apache::thrift::protocol::T_I32, 4); + xfer += detail::pm::protocol_methods::write(*proto, obj->microsec); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Time* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_I16))) { + goto _loop; + } + +_readField_hour : { + detail::pm::protocol_methods::read(*proto, obj->hour); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_BYTE))) { + goto _loop; + } + +_readField_minute : { + detail::pm::protocol_methods::read(*proto, obj->minute); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_BYTE))) { + goto _loop; + } + +_readField_sec : { + detail::pm::protocol_methods::read(*proto, obj->sec); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 4, protocol::T_I32))) { + goto _loop; + } + +_readField_microsec : { + detail::pm::protocol_methods::read(*proto, obj->microsec); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 4, 5, protocol::T_I32))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_hour; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_minute; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_sec; + } else { + goto _skip; + } + } + case 4: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_microsec; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Time const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Time"); + + xfer += proto->serializedFieldSize("hour", apache::thrift::protocol::T_BYTE, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->hour); + + xfer += proto->serializedFieldSize("minute", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->minute); + + xfer += proto->serializedFieldSize("sec", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->sec); + + xfer += proto->serializedFieldSize("microsec", apache::thrift::protocol::T_I32, 4); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microsec); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Time const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Time"); + + xfer += proto->serializedFieldSize("hour", apache::thrift::protocol::T_BYTE, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->hour); + + xfer += proto->serializedFieldSize("minute", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->minute); + + xfer += proto->serializedFieldSize("sec", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->sec); + + xfer += proto->serializedFieldSize("microsec", apache::thrift::protocol::T_I32, 4); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microsec); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for class DateTime + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "year") { + fid = 1; + _ftype = apache::thrift::protocol::T_I16; + } else if (_fname == "month") { + fid = 2; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "day") { + fid = 3; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "hour") { + fid = 4; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "minute") { + fid = 5; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "sec") { + fid = 6; + _ftype = apache::thrift::protocol::T_BYTE; + } else if (_fname == "microsec") { + fid = 7; + _ftype = apache::thrift::protocol::T_I32; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::DateTime const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("DateTime"); + + xfer += proto->writeFieldBegin("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::write(*proto, obj->year); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("month", apache::thrift::protocol::T_BYTE, 2); + xfer += + detail::pm::protocol_methods::write(*proto, obj->month); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::write(*proto, obj->day); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("hour", apache::thrift::protocol::T_BYTE, 4); + xfer += detail::pm::protocol_methods::write(*proto, obj->hour); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("minute", apache::thrift::protocol::T_BYTE, 5); + xfer += detail::pm::protocol_methods::write(*proto, obj->minute); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("sec", apache::thrift::protocol::T_BYTE, 6); + xfer += detail::pm::protocol_methods::write(*proto, obj->sec); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("microsec", apache::thrift::protocol::T_I32, 7); + xfer += detail::pm::protocol_methods::write(*proto, obj->microsec); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::DateTime* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_I16))) { + goto _loop; + } + +_readField_year : { + int16_t year; + detail::pm::protocol_methods::read(*proto, year); + obj->year = year; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_BYTE))) { + goto _loop; + } + +_readField_month : { + int8_t month; + detail::pm::protocol_methods::read(*proto, month); + obj->month = month; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_BYTE))) { + goto _loop; + } + +_readField_day : { + int8_t day; + detail::pm::protocol_methods::read(*proto, day); + obj->day = day; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 4, protocol::T_BYTE))) { + goto _loop; + } + +_readField_hour : { + int8_t hour; + detail::pm::protocol_methods::read(*proto, hour); + obj->hour = hour; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 4, 5, protocol::T_BYTE))) { + goto _loop; + } + +_readField_minute : { + int8_t minute; + detail::pm::protocol_methods::read(*proto, minute); + obj->minute = minute; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 5, 6, protocol::T_BYTE))) { + goto _loop; + } + +_readField_sec : { + int8_t sec; + detail::pm::protocol_methods::read(*proto, sec); + obj->sec = sec; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 6, 7, protocol::T_I32))) { + goto _loop; + } + +_readField_microsec : { + int32_t microsec; + detail::pm::protocol_methods::read(*proto, microsec); + obj->microsec = microsec; +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 7, 8, protocol::T_I32))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I16)) { + goto _readField_year; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_month; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_day; + } else { + goto _skip; + } + } + case 4: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_hour; + } else { + goto _skip; + } + } + case 5: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_minute; + } else { + goto _skip; + } + } + case 6: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_BYTE)) { + goto _readField_sec; + } else { + goto _skip; + } + } + case 7: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_microsec; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::DateTime const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("DateTime"); + + xfer += proto->serializedFieldSize("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->year); + + xfer += proto->serializedFieldSize("month", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->month); + + xfer += proto->serializedFieldSize("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->day); + + xfer += proto->serializedFieldSize("hour", apache::thrift::protocol::T_BYTE, 4); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->hour); + + xfer += proto->serializedFieldSize("minute", apache::thrift::protocol::T_BYTE, 5); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->minute); + + xfer += proto->serializedFieldSize("sec", apache::thrift::protocol::T_BYTE, 6); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->sec); + + xfer += proto->serializedFieldSize("microsec", apache::thrift::protocol::T_I32, 7); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microsec); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::DateTime const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("DateTime"); + + xfer += proto->serializedFieldSize("year", apache::thrift::protocol::T_I16, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->year); + + xfer += proto->serializedFieldSize("month", apache::thrift::protocol::T_BYTE, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->month); + + xfer += proto->serializedFieldSize("day", apache::thrift::protocol::T_BYTE, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->day); + + xfer += proto->serializedFieldSize("hour", apache::thrift::protocol::T_BYTE, 4); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->hour); + + xfer += proto->serializedFieldSize("minute", apache::thrift::protocol::T_BYTE, 5); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->minute); + + xfer += proto->serializedFieldSize("sec", apache::thrift::protocol::T_BYTE, 6); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->sec); + + xfer += proto->serializedFieldSize("microsec", apache::thrift::protocol::T_I32, 7); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microsec); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_DATEOPS_H_ diff --git a/src/common/datatypes/Duration.cpp b/src/common/datatypes/Duration.cpp new file mode 100644 index 0000000..1369ce8 --- /dev/null +++ b/src/common/datatypes/Duration.cpp @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Duration.h" + +#include +#include + +#include + +namespace nebula {} // namespace nebula + +namespace std {} // namespace std diff --git a/src/common/datatypes/Duration.h b/src/common/datatypes/Duration.h new file mode 100644 index 0000000..4ff7fd0 --- /dev/null +++ b/src/common/datatypes/Duration.h @@ -0,0 +1,154 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DURATION_H +#define COMMON_DATATYPES_DURATION_H + +#include + +#include + +#include "common/time/Constants.h" + +namespace nebula { + +// Duration equals to months + seconds + microseconds +// The base between months and days is not fixed, so we store years and months +// separately. +struct Duration { + // day + hours + minutes + seconds + microseconds + int64_t seconds; + int32_t microseconds; + // years + months + int32_t months; + + Duration() : seconds(0), microseconds(0), months(0) {} + Duration(int32_t m, int64_t s, int32_t us) : seconds(s), microseconds(us), months(m) {} + + int64_t years() const { + return months / 12; + } + + int64_t monthsInYear() const { + return months % 12; + } + + int64_t days() const { + return seconds / time::kSecondsOfDay; + } + + int64_t hours() const { + return seconds % time::kSecondsOfDay / time::kSecondsOfHour; + } + + int64_t minutes() const { + return seconds % time::kSecondsOfHour / time::kSecondsOfMinute; + } + + int64_t secondsInMinute() const { + return seconds % time::kSecondsOfMinute; + } + + int64_t microsecondsInSecond() const { + return microseconds; + } + + Duration operator-() const { + return Duration(-months, -seconds, -microseconds); + } + + Duration operator+(const Duration& rhs) const { + return Duration(months + rhs.months, seconds + rhs.seconds, microseconds + rhs.microseconds); + } + + Duration operator-(const Duration& rhs) const { + return Duration(months - rhs.months, seconds - rhs.seconds, microseconds - rhs.microseconds); + } + + Duration& addYears(int32_t y) { + months += y * 12; + return *this; + } + + Duration& addQuarters(int32_t q) { + months += q * 3; + return *this; + } + + Duration& addMonths(int32_t m) { + months += m; + return *this; + } + + Duration& addWeeks(int32_t w) { + seconds += (w * 7 * time::kSecondsOfDay); + return *this; + } + + Duration& addDays(int64_t d) { + seconds += d * time::kSecondsOfDay; + return *this; + } + + Duration& addHours(int64_t h) { + seconds += h * time::kSecondsOfHour; + return *this; + } + + Duration& addMinutes(int64_t minutes) { + seconds += minutes * time::kSecondsOfMinute; + return *this; + } + + Duration& addSeconds(int64_t s) { + seconds += s; + return *this; + } + + Duration& addMilliseconds(int64_t ms) { + seconds += ms / 1000; + microseconds += ((ms % 1000) * 1000); + return *this; + } + + Duration& addMicroseconds(int32_t us) { + microseconds += us; + return *this; + } + + // can't compare + bool operator<(const Duration& rhs) const = delete; + + bool operator==(const Duration& rhs) const { + return months == rhs.months && seconds == rhs.seconds && microseconds == rhs.microseconds; + } + + std::string toString() const { + return folly::sformat( + "P{}MT{}.{:0>6}000S", months, seconds + microseconds / 1000000, microseconds % 1000000); + } + + folly::dynamic toJson() const { + return toString(); + } +}; + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const nebula::Duration& d) const noexcept { + size_t hv = folly::hash::fnv64_buf(reinterpret_cast(&d.months), sizeof(d.months)); + hv = folly::hash::fnv64_buf(reinterpret_cast(d.seconds), sizeof(d.seconds), hv); + return folly::hash::fnv64_buf( + reinterpret_cast(d.microseconds), sizeof(d.microseconds), hv); + } +}; + +} // namespace std +#endif diff --git a/src/common/datatypes/DurationOps-inl.h b/src/common/datatypes/DurationOps-inl.h new file mode 100644 index 0000000..f525173 --- /dev/null +++ b/src/common/datatypes/DurationOps-inl.h @@ -0,0 +1,203 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_DURATIONOPS_INL_H +#define COMMON_DATATYPES_DURATIONOPS_INL_H + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Duration.h" + +namespace apache { +namespace thrift { + +/************************************** + * + * Ops for class Duration + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "seconds") { + fid = 1; + _ftype = apache::thrift::protocol::T_I64; + } else if (_fname == "microseconds") { + fid = 2; + _ftype = apache::thrift::protocol::T_I32; + } else if (_fname == "months") { + fid = 3; + _ftype = apache::thrift::protocol::T_I32; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Duration const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Duration"); + + xfer += proto->writeFieldBegin("seconds", apache::thrift::protocol::T_I64, 1); + xfer += detail::pm::protocol_methods::write(*proto, obj->seconds); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("microseconds", apache::thrift::protocol::T_I32, 2); + xfer += + detail::pm::protocol_methods::write(*proto, obj->microseconds); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("months", apache::thrift::protocol::T_I32, 3); + xfer += detail::pm::protocol_methods::write(*proto, obj->months); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Duration* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_I64))) { + goto _loop; + } + +_readField_seconds : { + detail::pm::protocol_methods::read(*proto, obj->seconds); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_I32))) { + goto _loop; + } + +_readField_microseconds : { + detail::pm::protocol_methods::read(*proto, obj->microseconds); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_I32))) { + goto _loop; + } + +_readField_months : { + detail::pm::protocol_methods::read(*proto, obj->months); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I64)) { + goto _readField_seconds; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_microseconds; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_months; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::Duration const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Duration"); + + xfer += proto->serializedFieldSize("seconds", apache::thrift::protocol::T_I64, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->seconds); + + xfer += proto->serializedFieldSize("microseconds", apache::thrift::protocol::T_I32, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microseconds); + + xfer += proto->serializedFieldSize("months", apache::thrift::protocol::T_I32, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->months); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::Duration const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Duration"); + + xfer += proto->serializedFieldSize("seconds", apache::thrift::protocol::T_I64, 1); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->seconds); + + xfer += proto->serializedFieldSize("microseconds", apache::thrift::protocol::T_I32, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->microseconds); + + xfer += proto->serializedFieldSize("months", apache::thrift::protocol::T_I32, 3); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->months); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp new file mode 100644 index 0000000..fe1b80e --- /dev/null +++ b/src/common/datatypes/Edge.cpp @@ -0,0 +1,179 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include +#include +#include +#include + +#include + +namespace nebula { + +std::string Edge::toString() const { + std::stringstream os; + os << "(" << src << ")" + << "-" + << "[" << name << "(" << type << ")]" + << "->" + << "(" << dst << ")" + << "@" << ranking; + if (!props.empty()) { + std::vector value(props.size()); + std::transform(props.begin(), props.end(), value.begin(), [](const auto& iter) -> std::string { + std::stringstream out; + out << iter.first << ":" << iter.second; + return out.str(); + }); + os << " " << folly::join(",", value); + } + return os.str(); +} + +// format: +// { +// "prop1": val1, +// "prop2": val2, +// } +folly::dynamic Edge::toJson() const { + folly::dynamic propObj = folly::dynamic::object(); + + for (const auto& iter : props) { + propObj.insert(iter.first, iter.second.toJson()); + } + + return propObj; +} + +// Used in Json form query result +// format: +// { +// "id": { +// "name": _name, +// "src": srcVID, +// "dst": dstVID, +// "type": _type, +// "ranking": _rankding +// } +// "type": "edge" +// } +folly::dynamic Edge::getMetaData() const { + folly::dynamic edgeMetadataObj = folly::dynamic::object(); + + folly::dynamic edgeIdObj = folly::dynamic::object("name", name)("src", src.toJson())( + "dst", dst.toJson())("type", type)("ranking", ranking); + + edgeMetadataObj.insert("id", edgeIdObj); + edgeMetadataObj.insert("type", "edge"); + + return edgeMetadataObj; +} + +bool Edge::contains(const Value& key) const { + if (!key.isStr()) { + return false; + } + return props.find(key.getStr()) != props.end(); +} + +const Value& Edge::value(const std::string& key) const { + auto find = props.find(key); + if (find != props.end()) { + return find->second; + } + return Value::kNullValue; +} + +bool Edge::operator<(const Edge& rhs) const { + if (src != rhs.src) { + return src < rhs.src; + } + if (dst != rhs.dst) { + return dst < rhs.dst; + } + if (type != rhs.type) { + return type < rhs.type; + } + if (ranking != rhs.ranking) { + return ranking < rhs.ranking; + } + if (props.size() != rhs.props.size()) { + return props.size() < rhs.props.size(); + } + return false; +} + +bool Edge::operator==(const Edge& rhs) const { + return keyEqual(rhs); +} + +void Edge::reverse() { + type = -type; + auto tmp = std::move(src); + src = std::move(dst); + dst = std::move(tmp); +} + +void Edge::clear() { + src.clear(); + dst.clear(); + type = 0; + name.clear(); + ranking = 0; + props.clear(); +} + +bool Edge::keyEqual(const Edge& rhs) const { + if (type != rhs.type && type != -rhs.type) { + return false; + } + if (type == rhs.type) { + return src == rhs.src && dst == rhs.dst && ranking == rhs.ranking; + } + return src == rhs.dst && dst == rhs.src && ranking == rhs.ranking; +} + +std::string Edge::id() const { + std::string s; + // FIXME(czp): How to distinguish between `11->2@90` and `1->12@90`? + if (src.type() == Value::Type::INT) { + EdgeType t = type > 0 ? type : -type; + const int64_t& srcId = type > 0 ? src.getInt() : dst.getInt(); + const int64_t& dstId = type > 0 ? dst.getInt() : src.getInt(); + s.reserve(sizeof(srcId) + sizeof(dstId) + sizeof(type) + sizeof(ranking)); + s.append(reinterpret_cast(&srcId), sizeof(srcId)); + s.append(reinterpret_cast(&dstId), sizeof(dstId)); + s.append(reinterpret_cast(&t), sizeof(t)); + s.append(reinterpret_cast(&ranking), sizeof(ranking)); + } else { + DCHECK(src.type() == Value::Type::STRING); + EdgeType t = type > 0 ? type : -type; + const std::string& srcId = type > 0 ? src.getStr() : dst.getStr(); + const std::string& dstId = type > 0 ? dst.getStr() : src.getStr(); + s.reserve(srcId.size() + dstId.size() + sizeof(t) + sizeof(ranking)); + s.append(srcId.data(), srcId.size()); + s.append(dstId.data(), dstId.size()); + s.append(reinterpret_cast(&t), sizeof(t)); + s.append(reinterpret_cast(&ranking), sizeof(ranking)); + } + return s; +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +std::size_t hash::operator()(const nebula::Edge& h) const { + const auto& src = h.type > 0 ? h.src.toString() : h.dst.toString(); + const auto& dst = h.type > 0 ? h.dst.toString() : h.src.toString(); + auto type = h.type > 0 ? h.type : -h.type; + size_t hv = folly::hash::fnv64(src); + hv = folly::hash::fnv64(dst, hv); + hv = folly::hash::fnv64_buf(reinterpret_cast(&type), sizeof(type), hv); + return folly::hash::fnv64_buf(reinterpret_cast(&h.ranking), sizeof(h.ranking), hv); +} + +} // namespace std diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h new file mode 100644 index 0000000..dcdef8a --- /dev/null +++ b/src/common/datatypes/Edge.h @@ -0,0 +1,107 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_EDGE_H_ +#define COMMON_DATATYPES_EDGE_H_ + +#include + +#include "common/datatypes/Value.h" +#include "common/thrift/ThriftTypes.h" + +namespace nebula { + +struct Edge { + Value src; + Value dst; + EdgeType type; + std::string name; + EdgeRanking ranking; + std::unordered_map props; + std::atomic refcnt{1}; + + Edge() {} + Edge(Edge&& v) noexcept + : src(std::move(v.src)), + dst(std::move(v.dst)), + type(std::move(v.type)), + name(std::move(v.name)), + ranking(std::move(v.ranking)), + props(std::move(v.props)) {} + Edge(const Edge& v) + : src(v.src), dst(v.dst), type(v.type), name(v.name), ranking(v.ranking), props(v.props) {} + Edge(Value s, + Value d, + EdgeType t, + std::string n, + EdgeRanking r, + std::unordered_map&& p) + : src(std::move(s)), + dst(std::move(d)), + type(std::move(t)), + name(std::move(n)), + ranking(std::move(r)), + props(std::move(p)) {} + + size_t ref() { + return ++refcnt; + } + size_t unref() { + return --refcnt; + } + + void clear(); + + void __clear() { + clear(); + } + + std::string toString() const; + folly::dynamic toJson() const; + // Used in Json form query result + folly::dynamic getMetaData() const; + + bool operator==(const Edge& rhs) const; + + bool operator!=(const Edge& rhs) const { + return !(*this == rhs); + } + + void format() { + if (type < 0) { + reverse(); + } + } + + void reverse(); + + bool operator<(const Edge& rhs) const; + + bool contains(const Value& key) const; + + const Value& value(const std::string& key) const; + + bool keyEqual(const Edge& rhs) const; + + // Return this edge's id encoded in string + std::string id() const; +}; + +inline std::ostream& operator<<(std::ostream& os, const Edge& v) { + return os << v.toString(); +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const nebula::Edge& h) const; +}; + +} // namespace std +#endif // COMMON_DATATYPES_EDGE_H_ diff --git a/src/common/datatypes/EdgeOps-inl.h b/src/common/datatypes/EdgeOps-inl.h new file mode 100644 index 0000000..f86ec5f --- /dev/null +++ b/src/common/datatypes/EdgeOps-inl.h @@ -0,0 +1,294 @@ +/* + * + * Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + * + */ + +#ifndef COMMON_DATATYPES_EDGEOPS_H_ +#define COMMON_DATATYPES_EDGEOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Edge.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "src") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "dst") { + fid = 2; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "type") { + fid = 3; + _ftype = apache::thrift::protocol::T_I32; + } else if (_fname == "name") { + fid = 4; + _ftype = apache::thrift::protocol::T_STRING; + } else if (_fname == "ranking") { + fid = 5; + _ftype = apache::thrift::protocol::T_I64; + } else if (_fname == "props") { + fid = 6; + _ftype = apache::thrift::protocol::T_MAP; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Edge const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Edge"); + + xfer += proto->writeFieldBegin("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += ::apache::thrift::Cpp2Ops::write(proto, &obj->src); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("dst", apache::thrift::protocol::T_STRUCT, 2); + xfer += ::apache::thrift::Cpp2Ops::write(proto, &obj->dst); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("type", apache::thrift::protocol::T_I32, 3); + // NOTICE: The original id will be transformed to +1/-1, to indicate the edge + // direction. + auto type = obj->type; + if (type != 0) { + type > 0 ? type = 1 : type = -1; + } + xfer += detail::pm::protocol_methods::write(*proto, type); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("name", apache::thrift::protocol::T_STRING, 4); + xfer += proto->writeBinary(obj->name); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("ranking", apache::thrift::protocol::T_I64, 5); + xfer += detail::pm::protocol_methods::write( + *proto, obj->ranking); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("props", apache::thrift::protocol::T_MAP, 6); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::write(*proto, obj->props); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Edge* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_STRUCT))) { + goto _loop; + } + +_readField_src : { ::apache::thrift::Cpp2Ops::read(proto, &obj->src); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_STRUCT))) { + goto _loop; + } + +_readField_dst : { ::apache::thrift::Cpp2Ops::read(proto, &obj->dst); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_I32))) { + goto _loop; + } + +_readField_type : { + detail::pm::protocol_methods::read(*proto, obj->type); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 4, protocol::T_STRING))) { + goto _loop; + } + +_readField_name : { proto->readBinary(obj->name); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 4, 5, protocol::T_I64))) { + goto _loop; + } + +_readField_ranking : { + detail::pm::protocol_methods::read(*proto, + obj->ranking); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 5, 6, protocol::T_MAP))) { + goto _loop; + } + +_readField_props : { + obj->props = std::unordered_map(); + detail::pm::protocol_methods, + std::unordered_map>::read(*proto, + obj->props); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 6, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRUCT)) { + goto _readField_src; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRUCT)) { + goto _readField_dst; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_type; + } else { + goto _skip; + } + } + case 4: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRING)) { + goto _readField_name; + } else { + goto _skip; + } + } + case 5: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I64)) { + goto _readField_ranking; + } else { + goto _skip; + } + } + case 6: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_MAP)) { + goto _readField_props; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Edge const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Edge"); + + xfer += proto->serializedFieldSize("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += ::apache::thrift::Cpp2Ops::serializedSize(proto, &obj->src); + + xfer += proto->serializedFieldSize("dst", apache::thrift::protocol::T_STRUCT, 2); + xfer += ::apache::thrift::Cpp2Ops::serializedSize(proto, &obj->dst); + + xfer += proto->serializedFieldSize("type", apache::thrift::protocol::T_I32, 3); + xfer += + detail::pm::protocol_methods::serializedSize( + *proto, obj->type); + + xfer += proto->serializedFieldSize("name", apache::thrift::protocol::T_STRING, 4); + xfer += proto->serializedSizeBinary(obj->name); + + xfer += proto->serializedFieldSize("ranking", apache::thrift::protocol::T_I64, 5); + xfer += detail::pm::protocol_methods::serializedSize(*proto, + obj->ranking); + + xfer += proto->serializedFieldSize("props", apache::thrift::protocol::T_MAP, 6); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->props); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Edge const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Edge"); + + xfer += proto->serializedFieldSize("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += ::apache::thrift::Cpp2Ops::serializedSizeZC(proto, &obj->src); + + xfer += proto->serializedFieldSize("dst", apache::thrift::protocol::T_STRUCT, 2); + xfer += ::apache::thrift::Cpp2Ops::serializedSizeZC(proto, &obj->dst); + + xfer += proto->serializedFieldSize("type", apache::thrift::protocol::T_I32, 3); + xfer += + detail::pm::protocol_methods::serializedSize( + *proto, obj->type); + + xfer += proto->serializedFieldSize("name", apache::thrift::protocol::T_STRING, 4); + xfer += proto->serializedSizeZCBinary(obj->name); + + xfer += proto->serializedFieldSize("ranking", apache::thrift::protocol::T_I64, 5); + xfer += detail::pm::protocol_methods::serializedSize(*proto, + obj->ranking); + + xfer += proto->serializedFieldSize("props", apache::thrift::protocol::T_MAP, 6); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->props); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_EDGEOPS_H_ diff --git a/src/common/datatypes/Geography.cpp b/src/common/datatypes/Geography.cpp new file mode 100644 index 0000000..01eb56e --- /dev/null +++ b/src/common/datatypes/Geography.cpp @@ -0,0 +1,395 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Geography.h" + +#include +#include + +#include + +#include "common/geo/GeoUtils.h" +#include "common/geo/io/wkb/WKBReader.h" +#include "common/geo/io/wkb/WKBWriter.h" +#include "common/geo/io/wkt/WKTReader.h" +#include "common/geo/io/wkt/WKTWriter.h" + +namespace nebula { + +std::ostream& operator<<(std::ostream& os, const GeoShape& shape) { + switch (shape) { + case GeoShape::POINT: { + os << "POINT"; + break; + } + case GeoShape::LINESTRING: { + os << "LINESTRING"; + break; + } + case GeoShape::POLYGON: { + os << "POLYGON"; + break; + } + case GeoShape::UNKNOWN: + default: { + os << "__UNKNOWN__"; + break; + } + } + + return os; +} + +constexpr double kMaxLongitude = 180.0; +constexpr double kMaxLatitude = 90.0; + +void Coordinate::normalize() { + // Reduce the x(longitude) to the range [-180, 180] degrees + x = std::remainder(x, 360.0); + + // Reduce the y(latitude) to the range [-90, 90] degrees + double tmp = remainder(y, 360.0); + if (tmp > kMaxLatitude) { + y = 180.0 - tmp; + } else if (tmp < -kMaxLatitude) { + y = -180.0 - tmp; + } +} + +Status Coordinate::isValid() const { + if (std::abs(x) > kMaxLongitude) { + return Status::Error("Invalid longitude: %f", x); + } + if (std::abs(y) > kMaxLatitude) { + return Status::Error("Invalid latitude: %f", y); + } + return Status::OK(); +} + +std::string Coordinate::toString() const { + return "(" + std::to_string(x) + " " + std::to_string(y) + ")"; +} + +void Point::normalize() {} + +Status Point::isValid() const { + return coord.isValid(); +} + +void LineString::normalize() { + geo::GeoUtils::removeAdjacentDuplicateCoordinates(coordList); +} + +Status LineString::isValid() const { + if (coordList.size() < 2) { + return Status::Error("LineString contains %zu coordinates, requires at least 2 coordinates", + coordList.size()); + } + for (size_t i = 0; i < coordList.size(); ++i) { + auto& coord = coordList[i]; + auto status = coord.isValid(); + if (!status.ok()) { + return Status::Error("The %zu-th coordinate is invalid: %s", i, status.message().c_str()); + } + } + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + + S2Error error; + if (static_cast(s2Region.get())->FindValidationError(&error)) { + std::stringstream ss; + ss << "FindValidationError, S2 Errorcode: " << error.code() << ", message: " << error.text(); + return Status::Error(ss.str()); + } + return Status::OK(); +} + +void Polygon::normalize() { + for (auto& coordList : coordListList) { + geo::GeoUtils::removeAdjacentDuplicateCoordinates(coordList); + } +} + +Status Polygon::isValid() const { + for (size_t i = 0; i < coordListList.size(); ++i) { + auto& coordList = coordListList[i]; + if (coordList.size() < 4) { + return Status::Error( + "Polygon's %zu-th LinearRing contains %zu coordinates, requires at least 4", + i, + coordList.size()); + } + if (coordList.front() != coordList.back()) { + return Status::Error("Polygon's %zu-th LinearRing is not closed", i); + } + for (size_t j = 0; j < coordList.size(); ++j) { + auto& coord = coordList[j]; + auto status = coord.isValid(); + if (!status.ok()) { + return Status::Error("The %zu-th coordinate of %zu-th LinearRing is invalid: %s", + j, + i, + status.message().c_str()); + } + } + } + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + + S2Error error; + if (static_cast(s2Region.get())->FindValidationError(&error)) { + std::stringstream ss; + ss << "FindValidationError, S2 Errorcode: " << error.code() << ", message: " << error.text(); + return Status::Error(ss.str()); + } + return Status::OK(); +} + +StatusOr Geography::fromWKT(const std::string& wkt, + bool needNormalize, + bool verifyValidity) { + auto geogRet = geo::WKTReader().read(wkt); + if (!geogRet.ok()) { + return geogRet; + } + auto geog = std::move(geogRet).value(); + if (needNormalize) { + geog.normalize(); + } + if (verifyValidity) { + auto status = geog.isValid(); + if (!status.ok()) { + std::stringstream ss; + ss << "The geography " << geog.toString() << " built from wkt " << wkt + << " is invalid, error: " << status.message(); + return Status::Error(ss.str()); + } + } + + return geog; +} + +StatusOr Geography::fromWKB(const std::string& wkb, + bool needNormalize, + bool verifyValidity) { + auto geogRet = geo::WKBReader().read(wkb); + if (!geogRet.ok()) { + return geogRet; + } + auto geog = std::move(geogRet).value(); + if (needNormalize) { + geog.normalize(); + } + if (verifyValidity) { + auto status = geog.isValid(); + if (!status.ok()) { + std::stringstream ss; + ss << "The geography " << geog.toString() << " built from wkb " << wkb + << " is invalid, error: " << status.message(); + return Status::Error(ss.str()); + } + } + + return geog; +} + +GeoShape Geography::shape() const { + switch (geo_.index()) { + case 0: + return GeoShape::POINT; + case 1: + return GeoShape::LINESTRING; + case 2: + return GeoShape::POLYGON; + default: // May never reaches here, because the default constructor of the variant geo_ will + // hold the value-initialized value of the first alternative(Point). + return GeoShape::UNKNOWN; + } +} + +const Point& Geography::point() const { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +const LineString& Geography::lineString() const { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +const Polygon& Geography::polygon() const { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +Point& Geography::mutablePoint() { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +LineString& Geography::mutableLineString() { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +Polygon& Geography::mutablePolygon() { + CHECK(std::holds_alternative(geo_)); + return std::get(geo_); +} + +void Geography::normalize() { + switch (shape()) { + case GeoShape::POINT: { + auto& point = mutablePoint(); + point.normalize(); + return; + } + case GeoShape::LINESTRING: { + auto& line = mutableLineString(); + line.normalize(); + return; + } + case GeoShape::POLYGON: { + auto& polygon = mutablePolygon(); + polygon.normalize(); + return; + } + case GeoShape::UNKNOWN: + default: { + LOG(ERROR) + << "Geography shapes other than Point/LineString/Polygon are not currently supported"; + return; + } + } +} + +Status Geography::isValid() const { + switch (shape()) { + case GeoShape::POINT: { + const auto& point = this->point(); + return point.isValid(); + } + case GeoShape::LINESTRING: { + const auto& line = this->lineString(); + return line.isValid(); + } + case GeoShape::POLYGON: { + const auto& polygon = this->polygon(); + return polygon.isValid(); + } + case GeoShape::UNKNOWN: + default: { + return Status::Error( + "Geography shapes other than Point/LineString/Polygon are not currently supported"); + } + } +} + +Point Geography::centroid() const { + switch (shape()) { + case GeoShape::POINT: { + return this->point(); + } + case GeoShape::LINESTRING: { + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + S2Point s2Point = static_cast(s2Region.get())->GetCentroid(); + return Point(geo::GeoUtils::coordinateFromS2Point(s2Point)); + } + case GeoShape::POLYGON: { + auto s2Region = geo::GeoUtils::s2RegionFromGeography(*this); + CHECK_NOTNULL(s2Region); + S2Point s2Point = static_cast(s2Region.get())->GetCentroid(); + return Point(geo::GeoUtils::coordinateFromS2Point(s2Point)); + } + case GeoShape::UNKNOWN: + default: { + LOG(ERROR) + << "Geography shapes other than Point/LineString/Polygon are not currently supported"; + return {}; + } + } +} + +std::string Geography::asWKT() const { + return geo::WKTWriter().write(*this); +} + +std::string Geography::asWKB() const { + return geo::WKBWriter().write(*this); +} + +std::string Geography::asWKBHex() const { + return folly::hexlify(geo::WKBWriter().write(*this)); +} + +std::unique_ptr Geography::asS2() const { + return geo::GeoUtils::s2RegionFromGeography(*this); +} + +bool Geography::operator==(const Geography& rhs) const { + auto lhsShape = shape(); + auto rhsShape = rhs.shape(); + if (lhsShape != rhsShape) { + return false; + } + + switch (lhsShape) { + case GeoShape::POINT: { + return point() == rhs.point(); + } + case GeoShape::LINESTRING: { + return lineString() == rhs.lineString(); + } + case GeoShape::POLYGON: { + return polygon() == rhs.polygon(); + } + case GeoShape::UNKNOWN: + default: { + LOG(ERROR) << "Geography shapes other than Point/LineString/Polygon are not currently " + "supported"; + return false; + } + } +} + +bool Geography::operator<(const Geography& rhs) const { + auto lhsShape = shape(); + auto rhsShape = rhs.shape(); + if (lhsShape != rhsShape) { + return lhsShape < rhsShape; + } + + switch (lhsShape) { + case GeoShape::POINT: { + return point() < rhs.point(); + } + case GeoShape::LINESTRING: { + return lineString() < rhs.lineString(); + } + case GeoShape::POLYGON: { + return polygon() < rhs.polygon(); + } + case GeoShape::UNKNOWN: + default: { + LOG(ERROR) << "Geography shapes other than Point/LineString/Polygon are not currently " + "supported"; + return false; + } + } + return false; +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +std::size_t hash::operator()(const nebula::Geography& v) const { + std::string wkb = v.asWKB(); + return hash{}(wkb); +} + +} // namespace std diff --git a/src/common/datatypes/Geography.h b/src/common/datatypes/Geography.h new file mode 100644 index 0000000..2f93c15 --- /dev/null +++ b/src/common/datatypes/Geography.h @@ -0,0 +1,245 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_GEOGRAPHY_H +#define COMMON_DATATYPES_GEOGRAPHY_H + +#include +#include +#include +#include + +#include + +#include "common/base/Base.h" +#include "common/base/StatusOr.h" +#include "common/datatypes/Value.h" + +// Do not include here, it will indirectly includes a header file which defines a +// enum `BEGIN`(not enum class). While Geography.h is indirectly included by parser.yy, which has a +// macro named `BEGIN`. So they will be conflicted. + +class S2Polygon; + +namespace nebula { + +enum class GeoShape : uint32_t { + UNKNOWN = 0, // illegal + POINT = 1, + LINESTRING = 2, + POLYGON = 3, +}; + +std::ostream& operator<<(std::ostream& os, const GeoShape& shape); + +// clang-format off +/* +static const std::unordered_map kShapeTypeToS2Region = { + // S2PointRegion is a wrapper of S2Point, and it inherits from the S2Region class while S2Point doesn't. + {GeoShape::POINT, S2PointRegion}, + {GeoShape::LINESTRING, S2Polyline}, + {GeoShape::POLYGON, S2Polygon}, +}; +*/ +// clang-format on + +struct Coordinate { + double x, y; + + Coordinate() = default; + Coordinate(double lng, double lat) : x(lng), y(lat) {} + + void normalize(); + Status isValid() const; + + std::string toString() const; + + void clear() { + x = 0.0; + y = 0.0; + } + void __clear() { + clear(); + } + + bool operator==(const Coordinate& rhs) const { + return std::abs(x - rhs.x) < kEpsilon && std::abs(y - rhs.y) < kEpsilon; + } + bool operator!=(const Coordinate& rhs) const { + return !(*this == rhs); + } + bool operator<(const Coordinate& rhs) const { + if (x != rhs.x) { + return x < rhs.x; + } + if (y != rhs.y) { + return y < rhs.y; + } + return false; + } +}; + +struct Point { + Coordinate coord; + + Point() = default; + explicit Point(const Coordinate& v) : coord(v) {} + explicit Point(Coordinate&& v) : coord(std::move(v)) {} + + void normalize(); + Status isValid() const; + + void clear() { + coord.clear(); + } + void __clear() { + clear(); + } + + bool operator==(const Point& rhs) const { + return coord == rhs.coord; + } + bool operator<(const Point& rhs) const { + return coord < rhs.coord; + } +}; + +struct LineString { + std::vector coordList; + + LineString() = default; + explicit LineString(const std::vector& v) : coordList(v) {} + explicit LineString(std::vector&& v) : coordList(std::move(v)) {} + + uint32_t numCoord() const { + return coordList.size(); + } + + void normalize(); + Status isValid() const; + + void clear() { + coordList.clear(); + } + void __clear() { + clear(); + } + + bool operator==(const LineString& rhs) const { + return coordList == rhs.coordList; + } + bool operator<(const LineString& rhs) const { + return coordList < rhs.coordList; + } +}; + +struct Polygon { + std::vector> coordListList; + + Polygon() = default; + explicit Polygon(const std::vector>& v) : coordListList(v) {} + explicit Polygon(std::vector>&& v) : coordListList(std::move(v)) {} + + uint32_t numCoordList() const { + return coordListList.size(); + } + + void normalize(); + Status isValid() const; + + void clear() { + coordListList.clear(); + } + void __clear() { + clear(); + } + + bool operator==(const Polygon& rhs) const { + return coordListList == rhs.coordListList; + } + bool operator<(const Polygon& rhs) const { + return coordListList < rhs.coordListList; + } +}; + +struct Geography { + std::variant geo_; + + // Factory method + static StatusOr fromWKT(const std::string& wkt, + bool needNormalize = false, + bool verifyValidity = false); + + static StatusOr fromWKB(const std::string& wkb, + bool needNormalize = false, + bool verifyValidity = false); + + Geography() = default; + Geography(const Point& v) : geo_(v) {} // NOLINT + Geography(Point&& v) : geo_(std::move(v)) {} // NOLINT + Geography(const LineString& v) : geo_(v) {} // NOLINT + Geography(LineString&& v) : geo_(std::move(v)) {} // NOLINT + Geography(const Polygon& v) : geo_(v) {} // NOLINT + Geography(Polygon&& v) : geo_(std::move(v)) {} // NOLINT + + GeoShape shape() const; + + const Point& point() const; + const LineString& lineString() const; + const Polygon& polygon() const; + + Point& mutablePoint(); + LineString& mutableLineString(); + Polygon& mutablePolygon(); + + void normalize(); + Status isValid() const; + + Point centroid() const; + + std::string asWKT() const; + + std::string asWKB() const; + + std::string asWKBHex() const; + + std::unique_ptr asS2() const; + + std::string toString() const { + return asWKT(); + } + + folly::dynamic toJson() const { + return toString(); + } + + void clear() { + geo_.~variant(); + } + + void __clear() { + clear(); + } + + bool operator==(const Geography& rhs) const; + bool operator<(const Geography& rhs) const; +}; + +inline std::ostream& operator<<(std::ostream& os, const Geography& g) { + return os << g.toString(); +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const nebula::Geography& h) const; +}; + +} // namespace std +#endif diff --git a/src/common/datatypes/GeographyOps-inl.h b/src/common/datatypes/GeographyOps-inl.h new file mode 100644 index 0000000..75abe0d --- /dev/null +++ b/src/common/datatypes/GeographyOps-inl.h @@ -0,0 +1,684 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_GEOGRAPHYOPS_H_ +#define COMMON_DATATYPES_GEOGRAPHYOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Geography.h" + +namespace apache { +namespace thrift { + +/************************************** + * + * Ops for struct Coordinate + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "x") { + fid = 1; + _ftype = apache::thrift::protocol::T_DOUBLE; + } else if (_fname == "y") { + fid = 2; + _ftype = apache::thrift::protocol::T_DOUBLE; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Coordinate const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Coordinate"); + xfer += proto->writeFieldBegin("x", protocol::T_DOUBLE, 1); + xfer += proto->writeDouble(obj->x); + xfer += proto->writeFieldEnd(); + xfer += proto->writeFieldBegin("y", protocol::T_DOUBLE, 2); + xfer += proto->writeDouble(obj->y); + xfer += proto->writeFieldEnd(); + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Coordinate* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_DOUBLE))) { + goto _loop; + } +_readField_x : { proto->readDouble(obj->x); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_DOUBLE))) { + goto _loop; + } +_readField_y : { proto->readDouble(obj->y); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 0, apache::thrift::protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_DOUBLE)) { + goto _readField_x; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_DOUBLE)) { + goto _readField_y; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::Coordinate const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Coordinate"); + xfer += proto->serializedFieldSize("x", apache::thrift::protocol::T_DOUBLE, 1); + xfer += proto->serializedSizeDouble(obj->x); + xfer += proto->serializedFieldSize("y", apache::thrift::protocol::T_DOUBLE, 2); + xfer += proto->serializedSizeDouble(obj->y); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::Coordinate const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Coordinate"); + xfer += proto->serializedFieldSize("x", apache::thrift::protocol::T_DOUBLE, 1); + xfer += proto->serializedSizeDouble(obj->x); + xfer += proto->serializedFieldSize("y", apache::thrift::protocol::T_DOUBLE, 2); + xfer += proto->serializedSizeDouble(obj->y); + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for struct Point + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "coord") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Point const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Point"); + xfer += proto->writeFieldBegin("coord", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::write(proto, &obj->coord); + xfer += proto->writeFieldEnd(); + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Point* obj) { + apache::thrift::detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, apache::thrift::protocol::T_STRUCT))) { + goto _loop; + } +_readField_coord : { Cpp2Ops::read(proto, &obj->coord); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, apache::thrift::protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRUCT)) { + goto _readField_coord; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Point const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Point"); + xfer += proto->serializedFieldSize("coord", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->coord); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Point const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Point"); + xfer += proto->serializedFieldSize("coord", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->coord); + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for struct LineString + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "coordList") { + fid = 1; + _ftype = apache::thrift::protocol::T_LIST; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_LIST; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::LineString const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("LineString"); + xfer += proto->writeFieldBegin("coordList", apache::thrift::protocol::T_LIST, 1); + xfer += + detail::pm::protocol_methods, + std::vector>::write(*proto, obj->coordList); + xfer += proto->writeFieldEnd(); + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::LineString* obj) { + apache::thrift::detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, apache::thrift::protocol::T_LIST))) { + goto _loop; + } +_readField_coordList : { + obj->coordList = std::vector(); + detail::pm::protocol_methods, + std::vector>::read(*proto, obj->coordList); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, apache::thrift::protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_coordList; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::LineString const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("LineString"); + xfer += proto->serializedFieldSize("coordList", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods< + type_class::list, + std::vector>::serializedSize(*proto, obj->coordList); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::LineString const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("LineString"); + xfer += proto->serializedFieldSize("coordList", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods< + type_class::list, + std::vector>::serializedSize(*proto, obj->coordList); + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for struct Polygon + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "coordListList") { + fid = 1; + _ftype = apache::thrift::protocol::T_LIST; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_LIST; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Polygon const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Polygon"); + xfer += proto->writeFieldBegin("coordListList", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods< + type_class::list, + std::vector>>::write(*proto, obj->coordListList); + xfer += proto->writeFieldEnd(); + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Polygon* obj) { + apache::thrift::detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, apache::thrift::protocol::T_LIST))) { + goto _loop; + } +_readField_coordListList : { + obj->coordListList = std::vector>(); + detail::pm::protocol_methods< + type_class::list, + std::vector>>::read(*proto, obj->coordListList); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, apache::thrift::protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_coordListList; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::Polygon const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Polygon"); + xfer += proto->serializedFieldSize("coordListList", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods< + type_class::list, + std::vector>>::serializedSize(*proto, + obj->coordListList); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::Polygon const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Polygon"); + xfer += proto->serializedFieldSize("coordListList", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods< + type_class::list, + std::vector>>::serializedSize(*proto, + obj->coordListList); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for class Geography + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "ptVal") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "lsVal") { + fid = 2; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "pgVal") { + fid = 3; + _ftype = apache::thrift::protocol::T_STRUCT; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Geography const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Geography"); + switch (obj->shape()) { + case nebula::GeoShape::POINT: { + xfer += proto->writeFieldBegin("ptVal", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::write(proto, &obj->point()); + xfer += proto->writeFieldEnd(); + break; + } + case nebula::GeoShape::LINESTRING: { + xfer += proto->writeFieldBegin("lsVal", apache::thrift::protocol::T_STRUCT, 2); + xfer += Cpp2Ops::write(proto, &obj->lineString()); + xfer += proto->writeFieldEnd(); + break; + } + case nebula::GeoShape::POLYGON: { + xfer += proto->writeFieldBegin("ptVal", apache::thrift::protocol::T_STRUCT, 3); + xfer += Cpp2Ops::write(proto, &obj->polygon()); + xfer += proto->writeFieldEnd(); + break; + } + case nebula::GeoShape::UNKNOWN: { + break; + } + } + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Geography* obj) { + apache::thrift::detail::ProtocolReaderStructReadState readState; + readState.fieldId = 0; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + readState.readFieldBegin(proto); + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + obj->clear(); + } else { + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + switch (readState.fieldId) { + case 1: { + if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { + obj->geo_ = nebula::Point(); + Cpp2Ops::read(proto, &obj->mutablePoint()); + } else { + proto->skip(readState.fieldType); + } + break; + } + case 2: { + if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { + obj->geo_ = nebula::LineString(); + Cpp2Ops::read(proto, &obj->mutableLineString()); + } else { + proto->skip(readState.fieldType); + } + break; + } + case 3: { + if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { + obj->geo_ = nebula::Polygon(); + Cpp2Ops::read(proto, &obj->mutablePolygon()); + } else { + proto->skip(readState.fieldType); + } + break; + } + default: { + proto->skip(readState.fieldType); + break; + } + } + readState.readFieldEnd(proto); + readState.readFieldBegin(proto); + if (UNLIKELY(readState.fieldType != apache::thrift::protocol::T_STOP)) { + using apache::thrift::protocol::TProtocolException; + TProtocolException::throwUnionMissingStop(); + } + } + readState.readStructEnd(proto); +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::Geography const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Geography"); + switch (obj->shape()) { + case nebula::GeoShape::POINT: { + xfer += proto->serializedFieldSize("ptVal", protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->point()); + break; + } + case nebula::GeoShape::LINESTRING: { + xfer += proto->serializedFieldSize("lsVal", protocol::T_STRUCT, 2); + xfer += Cpp2Ops::serializedSize(proto, &obj->lineString()); + break; + } + case nebula::GeoShape::POLYGON: { + xfer += proto->serializedFieldSize("pgVal", protocol::T_STRUCT, 3); + xfer += Cpp2Ops::serializedSize(proto, &obj->polygon()); + break; + } + case nebula::GeoShape::UNKNOWN: { + break; + } + } + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::Geography const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Geography"); + switch (obj->shape()) { + case nebula::GeoShape::POINT: { + xfer += proto->serializedFieldSize("ptVal", protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->point()); + break; + } + case nebula::GeoShape::LINESTRING: { + xfer += proto->serializedFieldSize("lsVal", protocol::T_STRUCT, 2); + xfer += Cpp2Ops::serializedSize(proto, &obj->lineString()); + break; + } + case nebula::GeoShape::POLYGON: { + xfer += proto->serializedFieldSize("pgVal", protocol::T_STRUCT, 3); + xfer += Cpp2Ops::serializedSize(proto, &obj->polygon()); + break; + } + case nebula::GeoShape::UNKNOWN: { + break; + } + } + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_GEOGRAPHYOPS_H_ diff --git a/src/common/datatypes/HostAddr.cpp b/src/common/datatypes/HostAddr.cpp new file mode 100644 index 0000000..5d32d73 --- /dev/null +++ b/src/common/datatypes/HostAddr.cpp @@ -0,0 +1,47 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/HostAddr.h" + +#include + +#include "common/base/Base.h" + +namespace nebula { + +bool HostAddr::operator==(const HostAddr& rhs) const { + return host == rhs.host && port == rhs.port; +} + +bool HostAddr::operator!=(const HostAddr& rhs) const { + return !(*this == rhs); +} + +HostAddr& HostAddr::operator=(const HostAddr& rhs) { + host = rhs.host; + port = rhs.port; + return *this; +} + +bool HostAddr::operator<(const HostAddr& rhs) const { + if (host == rhs.host) { + return port < rhs.port; + } + return host < rhs.host; +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +std::size_t hash::operator()(const nebula::HostAddr& h) const noexcept { + int64_t code = folly::hash::fnv32_buf(h.host.data(), h.host.size()); + code <<= 32; + code |= folly::hash::fnv32_buf(&(h.port), sizeof(nebula::Port)); + return code; +} + +} // namespace std diff --git a/src/common/datatypes/HostAddr.h b/src/common/datatypes/HostAddr.h new file mode 100644 index 0000000..b326d21 --- /dev/null +++ b/src/common/datatypes/HostAddr.h @@ -0,0 +1,94 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_HOSTADDR_H_ +#define COMMON_DATATYPES_HOSTADDR_H_ + +#include + +#include "common/base/Logging.h" +#include "common/thrift/ThriftTypes.h" + +namespace nebula { + +// Host address type and utility functions +struct HostAddr { + std::string host; + Port port; + + HostAddr() : host(), port(0) {} + /* + * some one may ctor HostAddr this way : HostAddr host(0, 0) + * C++ will compile this successfully even we don't support an (int, int) ctor + * so, add an explicit delete ctor + * */ + HostAddr(int h, int p) = delete; + HostAddr(std::string h, Port p) : host(std::move(h)), port(p) {} + HostAddr(const HostAddr& other) : host(other.host), port(other.port) {} + HostAddr(HostAddr&& other) : host(std::move(other.host)), port(std::move(other.port)) {} + + void clear() { + host.clear(); + port = 0; + } + + void __clear() { + clear(); + } + + std::string toString() const { + std::stringstream os; + os << "\"" << host << "\":" << port; + return os.str(); + } + + std::string toRawString() const { + std::stringstream os; + os << host << ":" << port; + return os.str(); + } + + HostAddr& operator=(const HostAddr& rhs); + + bool operator==(const HostAddr& rhs) const; + + bool operator!=(const HostAddr& rhs) const; + + bool operator<(const HostAddr& rhs) const; + + static HostAddr fromString(const std::string& str) { + HostAddr ha; + auto pos = str.find(":"); + if (pos == std::string::npos) { + LOG(ERROR) << "HostAddr: parse string error"; + return ha; + } + ha.host = str.substr(1, pos - 2); + ha.port = std::stoi(str.substr(pos + 1)); + return ha; + } + + static HostAddr nullAddr() { + return HostAddr("", 0); + } +}; + +inline std::ostream& operator<<(std::ostream& os, const HostAddr& addr) { + return os << addr.toString(); +} + +} // namespace nebula + +namespace std { + +// Inject a customized hash function +template <> +struct hash { + std::size_t operator()(const nebula::HostAddr& h) const noexcept; +}; + +} // namespace std + +#endif // COMMON_DATATYPES_HOSTADDR_H_ diff --git a/src/common/datatypes/HostAddrOps-inl.h b/src/common/datatypes/HostAddrOps-inl.h new file mode 100644 index 0000000..47f8554 --- /dev/null +++ b/src/common/datatypes/HostAddrOps-inl.h @@ -0,0 +1,167 @@ +/* + * + * Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + * + */ + +#ifndef COMMON_DATATYPES_HOSTADDROPS_H_ +#define COMMON_DATATYPES_HOSTADDROPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/HostAddr.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "host") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRING; + } else if (_fname == "port") { + fid = 2; + _ftype = apache::thrift::protocol::T_I32; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::HostAddr const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("HostAddr"); + + xfer += proto->writeFieldBegin("host", apache::thrift::protocol::T_STRING, 1); + xfer += proto->writeString(obj->host); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("port", apache::thrift::protocol::T_I32, 2); + xfer += + detail::pm::protocol_methods::write(*proto, obj->port); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::HostAddr* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_STRING))) { + goto _loop; + } + +_readField_host : { proto->readString(obj->host); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_I32))) { + goto _loop; + } + +_readField_port : { + detail::pm::protocol_methods::read(*proto, obj->port); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRING)) { + goto _readField_host; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_port; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::HostAddr const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("HostAddr"); + + xfer += proto->serializedFieldSize("host", apache::thrift::protocol::T_STRING, 1); + xfer += proto->serializedSizeString(obj->host); + + xfer += proto->serializedFieldSize("port", apache::thrift::protocol::T_I32, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->port); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::HostAddr const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("HostAddr"); + + xfer += proto->serializedFieldSize("host", apache::thrift::protocol::T_STRING, 1); + xfer += proto->serializedSizeString(obj->host); + + xfer += proto->serializedFieldSize("port", apache::thrift::protocol::T_I32, 2); + xfer += detail::pm::protocol_methods::serializedSize( + *proto, obj->port); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_HOSTADDROPS_H_ diff --git a/src/common/datatypes/KeyValue.h b/src/common/datatypes/KeyValue.h new file mode 100644 index 0000000..a9cd39e --- /dev/null +++ b/src/common/datatypes/KeyValue.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_KEYVALUE_H_ +#define COMMON_DATATYPES_KEYVALUE_H_ + +namespace nebula { + +struct KeyValue { + std::string key; + std::string value; + + KeyValue() {} + KeyValue(KeyValue&& rhs) noexcept : key(std::move(rhs.key)), value(std::move(rhs.value)) {} + KeyValue(const KeyValue& rhs) : key(rhs.key), value(rhs.value) {} + explicit KeyValue(std::pair kv) + : key(std::move(kv.first)), value(std::move(kv.second)) {} + + void clear() { + key.clear(); + value.clear(); + } + + void __clear() { + clear(); + } + + auto& operator=(const KeyValue& rhs) { + this->key = rhs.key; + this->value = rhs.value; + return *this; + } + + auto& operator=(KeyValue&& rhs) { + this->key = std::move(rhs.key); + this->value = std::move(rhs.value); + return *this; + } + + bool operator==(const KeyValue& rhs) const { + if (key != rhs.key) { + return false; + } + return value == rhs.value; + } + + bool operator<(const KeyValue& rhs) const { + if (key != rhs.key) { + return key < rhs.key; + } + return value < rhs.value; + } +}; + +} // namespace nebula +#endif // COMMON_DATATYPES_KEYVALUE_H_ diff --git a/src/common/datatypes/KeyValueOps-inl.h b/src/common/datatypes/KeyValueOps-inl.h new file mode 100644 index 0000000..9c4dcc3 --- /dev/null +++ b/src/common/datatypes/KeyValueOps-inl.h @@ -0,0 +1,159 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_KEYVALUEOPS_H_ +#define COMMON_DATATYPES_KEYVALUEOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/KeyValue.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "key") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRING; + } else if (_fname == "value") { + fid = 2; + _ftype = apache::thrift::protocol::T_STRING; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::KeyValue const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("KeyValue"); + + xfer += proto->writeFieldBegin("key", apache::thrift::protocol::T_STRING, 1); + xfer += proto->writeBinary(obj->key); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("value", apache::thrift::protocol::T_STRING, 2); + xfer += proto->writeBinary(obj->value); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::KeyValue* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_STRING))) { + goto _loop; + } + +_readField_key : { proto->readBinary(obj->key); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_STRING))) { + goto _loop; + } + +_readField_value : { proto->readBinary(obj->value); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRING)) { + goto _readField_key; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRING)) { + goto _readField_value; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, + nebula::KeyValue const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("KeyValue"); + + xfer += proto->serializedFieldSize("key", apache::thrift::protocol::T_STRING, 1); + xfer += proto->serializedSizeBinary(obj->key); + + xfer += proto->serializedFieldSize("value", apache::thrift::protocol::T_STRING, 2); + xfer += proto->serializedSizeBinary(obj->value); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, + nebula::KeyValue const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("KeyValue"); + + xfer += proto->serializedFieldSize("key", apache::thrift::protocol::T_STRING, 1); + xfer += proto->serializedSizeZCBinary(obj->key); + + xfer += proto->serializedFieldSize("value", apache::thrift::protocol::T_STRING, 2); + xfer += proto->serializedSizeZCBinary(obj->value); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_KEYVALUEOPS_H_ diff --git a/src/common/datatypes/List.cpp b/src/common/datatypes/List.cpp new file mode 100644 index 0000000..fa1c35c --- /dev/null +++ b/src/common/datatypes/List.cpp @@ -0,0 +1,44 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/List.h" + +#include + +#include + +namespace nebula { + +std::string List::toString() const { + std::vector value(values.size()); + std::transform(values.begin(), values.end(), value.begin(), [](const auto& v) -> std::string { + return v.toString(); + }); + std::stringstream os; + os << "[" << folly::join(",", value) << "]"; + return os.str(); +} + +folly::dynamic List::toJson() const { + auto listJsonObj = folly::dynamic::array(); + + for (const auto& val : values) { + listJsonObj.push_back(val.toJson()); + } + + return listJsonObj; +} + +folly::dynamic List::getMetaData() const { + auto listMetadataObj = folly::dynamic::array(); + + for (const auto& val : values) { + listMetadataObj.push_back(val.getMetaData()); + } + + return listMetadataObj; +} + +} // namespace nebula diff --git a/src/common/datatypes/List.h b/src/common/datatypes/List.h new file mode 100644 index 0000000..a2694c5 --- /dev/null +++ b/src/common/datatypes/List.h @@ -0,0 +1,151 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_LIST_H_ +#define COMMON_DATATYPES_LIST_H_ + +#include +#include + +#include "common/datatypes/Value.h" + +namespace nebula { + +struct List { + std::vector values; + + List() = default; + List(const List&) = default; + List(List&&) noexcept = default; + explicit List(std::vector&& vals) { + values = std::move(vals); + } + explicit List(const std::vector& l) : values(l) {} + + bool empty() const { + return values.empty(); + } + + void reserve(std::size_t n) { + values.reserve(n); + } + + template ::value>::type> + void emplace_back(T&& v) { + values.emplace_back(std::forward(v)); + } + + void append(List&& other) { + values.reserve(size() + other.size()); + values.insert(values.end(), + std::make_move_iterator(other.values.begin()), + std::make_move_iterator(other.values.end())); + } + + void clear() { + values.clear(); + } + + void __clear() { + clear(); + } + + List& operator=(const List& rhs) { + if (this == &rhs) { + return *this; + } + values = rhs.values; + return *this; + } + List& operator=(List&& rhs) noexcept { + if (this == &rhs) { + return *this; + } + values = std::move(rhs.values); + return *this; + } + + bool operator==(const List& rhs) const { + return values == rhs.values; + } + + bool operator<(const List& rhs) const { + return values < rhs.values; + } + + const Value& operator[](size_t i) const { + return values[i]; + } + + Value& operator[](size_t i) { + return values[i]; + } + + bool contains(const Value& value) const { + return std::find(values.begin(), values.end(), value) != values.end(); + } + + size_t size() const { + return values.size(); + } + + std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of each element + folly::dynamic getMetaData() const; +}; + +inline std::ostream& operator<<(std::ostream& os, const List& l) { + return os << l.toString(); +} + +} // namespace nebula + +namespace std { +template <> +struct hash { + std::size_t operator()(const nebula::List& h) const { + if (h.values.size() == 1) { + return std::hash()(h.values[0]); + } + size_t seed = 0; + for (auto& v : h.values) { + seed ^= hash()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; + } +}; + +template <> +struct equal_to { + bool operator()(const nebula::List* lhs, const nebula::List* rhs) const { + return lhs == rhs ? true : (lhs != nullptr) && (rhs != nullptr) && (*lhs == *rhs); + } +}; + +template <> +struct equal_to { + bool operator()(const nebula::List* lhs, const nebula::List* rhs) const { + return lhs == rhs ? true : (lhs != nullptr) && (rhs != nullptr) && (*lhs == *rhs); + } +}; + +template <> +struct hash { + size_t operator()(const nebula::List* row) const { + return !row ? 0 : hash()(*row); + } +}; + +template <> +struct hash { + size_t operator()(const nebula::List* row) const { + return !row ? 0 : hash()(*row); + } +}; + +} // namespace std +#endif // COMMON_DATATYPES_LIST_H_ diff --git a/src/common/datatypes/ListOps-inl.h b/src/common/datatypes/ListOps-inl.h new file mode 100644 index 0000000..047ec61 --- /dev/null +++ b/src/common/datatypes/ListOps-inl.h @@ -0,0 +1,140 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_LISTOPS_H_ +#define COMMON_DATATYPES_LISTOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/List.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "values") { + fid = 1; + _ftype = apache::thrift::protocol::T_LIST; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::List const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("NList"); + + xfer += proto->writeFieldBegin("values", apache::thrift::protocol::T_LIST, 1); + xfer += detail::pm::protocol_methods, + std::vector>::write(*proto, obj->values); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::List* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_LIST))) { + goto _loop; + } + +_readField_values : { + obj->values = std::vector(); + detail::pm::protocol_methods, + std::vector>::read(*proto, obj->values); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_values; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::List const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NList"); + + xfer += proto->serializedFieldSize("values", apache::thrift::protocol::T_LIST, 1); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->values); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::List const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NList"); + + xfer += proto->serializedFieldSize("values", apache::thrift::protocol::T_LIST, 1); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->values); + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_LISTOPS_H_ diff --git a/src/common/datatypes/Map.cpp b/src/common/datatypes/Map.cpp new file mode 100644 index 0000000..7cb1de2 --- /dev/null +++ b/src/common/datatypes/Map.cpp @@ -0,0 +1,103 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Map.h" + +#include +#include + +#include + +namespace nebula { + +std::string Map::toString() const { + std::vector value(kvs.size()); + std::transform(kvs.begin(), kvs.end(), value.begin(), [](const auto& iter) -> std::string { + std::stringstream out; + out << iter.first << ":" << iter.second; + return out.str(); + }); + + std::stringstream os; + os << "{" << folly::join(",", value) << "}"; + return os.str(); +} + +folly::dynamic Map::toJson() const { + folly::dynamic mapJsonObj = folly::dynamic::object(); + + for (const auto& iter : kvs) { + mapJsonObj.insert(iter.first, iter.second.toJson()); + } + + return mapJsonObj; +} + +folly::dynamic Map::getMetaData() const { + auto mapMetadataObj = folly::dynamic::array(); + + for (const auto& kv : kvs) { + mapMetadataObj.push_back(kv.second.getMetaData()); + } + + return mapMetadataObj; +} + +// Map constructor to covert from folly::dynamic object +// Called by function: json_extract() + +// TODO(wey-gu) support Datetime, deeper nested Map/Datatypes +Map::Map(const folly::dynamic& obj) { + DCHECK(obj.isObject()); + for (auto& kv : obj.items()) { + if (kv.second.isString()) { + kvs.emplace(kv.first.asString(), Value(kv.second.asString())); + } else if (kv.second.isInt()) { + kvs.emplace(kv.first.asString(), Value(kv.second.asInt())); + } else if (kv.second.isDouble()) { + kvs.emplace(kv.first.asString(), Value(kv.second.asDouble())); + } else if (kv.second.isBool()) { + kvs.emplace(kv.first.asString(), Value(kv.second.asBool())); + } else if (kv.second.isNull()) { + kvs.emplace(kv.first.asString(), Value()); + } else if (kv.second.isObject()) { + std::unordered_map values; + for (auto& nkv : kv.second.items()) { + if (nkv.second.isString()) { + values.emplace(nkv.first.asString(), Value(nkv.second.asString())); + } else if (nkv.second.isInt()) { + values.emplace(nkv.first.asString(), Value(nkv.second.asInt())); + } else if (nkv.second.isDouble()) { + values.emplace(nkv.first.asString(), Value(nkv.second.asDouble())); + } else if (nkv.second.isBool()) { + values.emplace(nkv.first.asString(), Value(nkv.second.asBool())); + } else { + LOG(WARNING) << "JSON_EXTRACT nested layer 1: Map can be populated only by " + "Bool, Double, Int, String value and null, now trying to parse from: " + << nkv.second.typeName(); + } + } + kvs.emplace(kv.first.asString(), Value(Map(std::move(values)))); + } else { + LOG(WARNING) << "JSON_EXTRACT Only Bool, Double, Int, String value, null and Map(depth==1) " + "are supported, now trying to parse from: " + << kv.second.typeName(); + } + } +} + +} // namespace nebula + +namespace std { +std::size_t hash::operator()(const nebula::Map& m) const { + size_t seed = 0; + for (auto& v : m.kvs) { + seed ^= hash()(v.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + seed ^= hash()(v.second) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; +} + +} // namespace std diff --git a/src/common/datatypes/Map.h b/src/common/datatypes/Map.h new file mode 100644 index 0000000..15a0dd9 --- /dev/null +++ b/src/common/datatypes/Map.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_MAP_H_ +#define COMMON_DATATYPES_MAP_H_ + +#include + +#include "common/base/Logging.h" +#include "common/datatypes/Value.h" + +namespace nebula { + +struct Map { + std::unordered_map kvs; + + Map() = default; + Map(const Map&) = default; + Map(Map&&) noexcept = default; + explicit Map(std::unordered_map values) { + kvs = std::move(values); + } + + explicit Map(const folly::dynamic& obj); + + Map& operator=(const Map& rhs) { + if (this == &rhs) { + return *this; + } + kvs = rhs.kvs; + return *this; + } + Map& operator=(Map&& rhs) noexcept { + if (this == &rhs) { + return *this; + } + kvs = std::move(rhs.kvs); + return *this; + } + + void clear() { + kvs.clear(); + } + + void __clear() { + clear(); + } + + // the configs of rocksdb will use the interface, so the value need modify to + // string + std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of the value of each kv pair + folly::dynamic getMetaData() const; + + bool operator==(const Map& rhs) const { + return kvs == rhs.kvs; + } + + bool contains(const Value& value) const { + if (!value.isStr()) { + return false; + } + return kvs.count(value.getStr()) != 0; + } + + const Value& at(const std::string& key) const { + auto iter = kvs.find(key); + if (iter == kvs.end()) { + return Value::kNullUnknownProp; + } + return iter->second; + } + + size_t size() const { + return kvs.size(); + } +}; + +inline std::ostream& operator<<(std::ostream& os, const Map& m) { + return os << m.toString(); +} + +} // namespace nebula + +namespace std { +template <> +struct hash { + std::size_t operator()(const nebula::Map& m) const; +}; + +} // namespace std +#endif // COMMON_DATATYPES_MAP_H_ diff --git a/src/common/datatypes/MapOps-inl.h b/src/common/datatypes/MapOps-inl.h new file mode 100644 index 0000000..517510d --- /dev/null +++ b/src/common/datatypes/MapOps-inl.h @@ -0,0 +1,141 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_MAPOPS_H_ +#define COMMON_DATATYPES_MAPOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Map.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "kvs") { + fid = 1; + _ftype = apache::thrift::protocol::T_MAP; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Map const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("NMap"); + + xfer += proto->writeFieldBegin("kvs", apache::thrift::protocol::T_MAP, 1); + xfer += + detail::pm::protocol_methods, + std::unordered_map>::write(*proto, + obj->kvs); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Map* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_MAP))) { + goto _loop; + } + +_readField_kvs : { + obj->kvs = std::unordered_map(); + detail::pm::protocol_methods, + std::unordered_map>::read(*proto, + obj->kvs); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_MAP)) { + goto _readField_kvs; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Map const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NMap"); + + xfer += proto->serializedFieldSize("kvs", apache::thrift::protocol::T_MAP, 1); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->kvs); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Map const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NMap"); + + xfer += proto->serializedFieldSize("kvs", apache::thrift::protocol::T_MAP, 1); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->kvs); + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_MAPOPS_H_ diff --git a/src/common/datatypes/Path.cpp b/src/common/datatypes/Path.cpp new file mode 100644 index 0000000..9ebb33b --- /dev/null +++ b/src/common/datatypes/Path.cpp @@ -0,0 +1,100 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Path.h" + +#include +#include + +#include +#include +#include + +namespace nebula { +void Path::reverse() { + if (steps.empty()) { + return; + } + std::reverse(steps.begin(), steps.end()); + swap(src, steps.front().dst); + for (size_t i = 0; i < steps.size() - 1; ++i) { + swap(steps[i].dst, steps[i + 1].dst); + steps[i].type = -steps[i].type; + } + steps.back().type = -steps.back().type; +} + +bool Path::append(Path path) { + if (src != path.src && steps.back().dst != path.src) { + return false; + } + steps.reserve(steps.size() + path.steps.size()); + steps.insert(steps.end(), + std::make_move_iterator(path.steps.begin()), + std::make_move_iterator(path.steps.end())); + return true; +} + +bool Path::hasDuplicateVertices() const { + if (steps.empty()) { + return false; + } + std::unordered_set uniqueVid; + uniqueVid.reserve(steps.size() + 1); + uniqueVid.emplace(src.vid); + for (const auto& step : steps) { + auto ret = uniqueVid.emplace(step.dst.vid); + if (!ret.second) { + return true; + } + } + return false; +} + +bool Path::hasDuplicateEdges() const { + if (steps.size() < 2) { + return false; + } + using Key = std::tuple; + std::unordered_set uniqueSet; + uniqueSet.reserve(steps.size()); + auto srcVid = src.vid; + for (const auto& step : steps) { + const auto& dstVid = step.dst.vid; + bool ret = true; + if (step.type > 0) { + ret = uniqueSet.emplace(srcVid, dstVid, step.type, step.ranking).second; + } else { + ret = uniqueSet.emplace(dstVid, srcVid, -step.type, step.ranking).second; + } + if (!ret) { + return true; + } + srcVid = dstVid; + } + return false; +} + +} // namespace nebula + +namespace std { + +std::size_t hash::operator()(const nebula::Step& h) const noexcept { + size_t hv = hash()(h.dst); + hv = folly::hash::fnv64_buf(reinterpret_cast(&h.type), sizeof(h.type), hv); + return folly::hash::fnv64_buf(reinterpret_cast(&h.ranking), sizeof(h.ranking), hv); +} + +std::size_t hash::operator()(const nebula::Path& h) const { + size_t hv = hash()(h.src); + for (auto& s : h.steps) { + hv += (hv << 1) + (hv << 4) + (hv << 5) + (hv << 7) + (hv << 8) + (hv << 40); + hv ^= hash()(s); + } + + return hv; +} + +} // namespace std diff --git a/src/common/datatypes/Path.h b/src/common/datatypes/Path.h new file mode 100644 index 0000000..aa9dc7c --- /dev/null +++ b/src/common/datatypes/Path.h @@ -0,0 +1,250 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_PATH_H_ +#define COMMON_DATATYPES_PATH_H_ + +#include "common/datatypes/Value.h" +#include "common/datatypes/Vertex.h" +#include "common/thrift/ThriftTypes.h" + +namespace nebula { + +struct Step { + Vertex dst; + EdgeType type; + std::string name; + EdgeRanking ranking; + std::unordered_map props; + + Step() = default; + Step(const Step& s) + : dst(s.dst), type(s.type), name(s.name), ranking(s.ranking), props(s.props) {} + Step(Step&& s) noexcept + : dst(std::move(s.dst)), + type(std::move(s.type)), + name(std::move(s.name)), + ranking(std::move(s.ranking)), + props(std::move(s.props)) {} + Step(Vertex d, EdgeType t, std::string n, EdgeRanking r, std::unordered_map p) + : dst(std::move(d)), type(t), name(std::move(n)), ranking(r), props(std::move(p)) {} + + void clear() { + dst.clear(); + type = 0; + name.clear(); + ranking = 0; + props.clear(); + } + + void __clear() { + clear(); + } + + std::string toString() const { + std::stringstream os; + os << "-[" << name << "(" << type << ")]->" + << "(" << dst << ")" + << "@" << ranking << " "; + for (const auto& prop : props) { + os << prop.first << ":" << prop.second << ","; + } + auto path = os.str(); + path.pop_back(); + return path; + } + + Step& operator=(Step&& rhs) noexcept { + if (&rhs != this) { + dst = std::move(rhs.dst); + type = std::move(rhs.type); + name = std::move(rhs.name); + ranking = std::move(rhs.ranking); + props = std::move(rhs.props); + } + return *this; + } + + Step& operator=(const Step& rhs) { + if (&rhs != this) { + dst = rhs.dst; + type = rhs.type; + name = rhs.name; + ranking = rhs.ranking; + props = rhs.props; + } + return *this; + } + + bool operator==(const Step& rhs) const { + return dst == rhs.dst && (type == rhs.type || type == -rhs.type) && name == rhs.name && + ranking == rhs.ranking && props == rhs.props; + } + + bool operator<(const Step& rhs) const { + if (dst != rhs.dst) { + return dst < rhs.dst; + } + if (type != rhs.type && type != -rhs.type) { + return abs(type) < abs(rhs.type); + } + if (ranking != rhs.ranking) { + return ranking < rhs.ranking; + } + if (props.size() != rhs.props.size()) { + return props.size() < rhs.props.size(); + } + return false; + } +}; + +struct Path { + Vertex src; + std::vector steps; + + Path() = default; + Path(const Path& p) = default; + Path(Path&& p) noexcept : src(std::move(p.src)), steps(std::move(p.steps)) {} + Path(Vertex v, std::vector s) : src(std::move(v)), steps(std::move(s)) {} + + void clear() { + src.clear(); + steps.clear(); + } + + void __clear() { + clear(); + } + + std::string toString() const { + std::stringstream os; + os << "(" << src << ")"; + os << " "; + for (const auto& s : steps) { + os << s.toString(); + os << " "; + } + return os.str(); + } + + folly::dynamic toJson() const { + folly::dynamic pathJsonObj = folly::dynamic::array(); + auto srcVertex = src; + pathJsonObj.push_back(srcVertex.toJson()); + + for (const auto& s : steps) { + folly::dynamic edgeJsonObj = folly::dynamic::object(); + // parse edge props map as json + for (const auto& iter : s.props) { + edgeJsonObj.insert(iter.first, iter.second.toJson()); + } + // add edge json obj to path + pathJsonObj.push_back(edgeJsonObj); + + // reset src vertex and add vertex json obj to path + srcVertex = s.dst; + pathJsonObj.push_back(srcVertex.toJson()); + } + + return pathJsonObj; + } + + // Used in Json form query result + // format: + // [vertex1_metadata, edge1_metadata, vertex2_metadata, edge2_metadata,....] + folly::dynamic getMetaData() const { + auto dynamicObj = folly::dynamic::array(); + auto srcVertex = src; + dynamicObj.push_back(srcVertex.getMetaData()); + + // Construct edge metadata + for (const auto& s : steps) { + folly::dynamic edgeIdObj = folly::dynamic::object(); + edgeIdObj.insert("src", srcVertex.vid.toJson()); + edgeIdObj.insert("dst", s.dst.vid.toJson()); + edgeIdObj.insert("type", s.type); + edgeIdObj.insert("name", s.name); + edgeIdObj.insert("ranking", s.ranking); + + folly::dynamic edgeMetadataObj = folly::dynamic::object("id", edgeIdObj)("type", "edge"); + dynamicObj.push_back(edgeMetadataObj); + dynamicObj.push_back(s.dst.getMetaData()); + // reset src vertex + srcVertex = s.dst; + } + + return dynamicObj; + } + + Path& operator=(Path&& rhs) noexcept { + if (&rhs != this) { + src = std::move(rhs.src); + steps = std::move(rhs.steps); + } + return *this; + } + + Path& operator=(const Path& rhs) { + if (&rhs != this) { + src = rhs.src; + steps = rhs.steps; + } + return *this; + } + + bool operator==(const Path& rhs) const { + return src == rhs.src && steps == rhs.steps; + } + + void addStep(Step step) { + steps.emplace_back(std::move(step)); + } + + void reverse(); + + // Append a path to another one. + // 5->4>3 appended by 3->2->1 => 5->4->3->2->1 + bool append(Path path); + + bool operator<(const Path& rhs) const { + if (src != rhs.src) { + return src < rhs.src; + } + if (steps != rhs.steps) { + return steps < rhs.steps; + } + return false; + } + + bool hasDuplicateEdges() const; + bool hasDuplicateVertices() const; +}; + +inline void swap(Step& a, Step& b) { + auto tmp = std::move(a); + a = std::move(b); + b = std::move(tmp); +} + +inline std::ostream& operator<<(std::ostream& os, const Path& p) { + return os << p.toString(); +} + +} // namespace nebula + +namespace std { + +template <> +struct hash { + std::size_t operator()(const nebula::Step& h) const noexcept; +}; + +template <> +struct hash { + std::size_t operator()(const nebula::Path& h) const; +}; + +} // namespace std +#endif // COMMON_DATATYPES_PATH_H_ diff --git a/src/common/datatypes/PathOps-inl.h b/src/common/datatypes/PathOps-inl.h new file mode 100644 index 0000000..a22074b --- /dev/null +++ b/src/common/datatypes/PathOps-inl.h @@ -0,0 +1,425 @@ +/* + * + * Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + * + */ + +#ifndef COMMON_DATATYPES_PATHOPS_H_ +#define COMMON_DATATYPES_PATHOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Path.h" + +namespace apache { +namespace thrift { + +/************************************** + * + * Ops for class Step + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "dst") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "type") { + fid = 2; + _ftype = apache::thrift::protocol::T_I32; + } else if (_fname == "name") { + fid = 3; + _ftype = apache::thrift::protocol::T_STRING; + } else if (_fname == "ranking") { + fid = 4; + _ftype = apache::thrift::protocol::T_I64; + } else if (_fname == "props") { + fid = 5; + _ftype = apache::thrift::protocol::T_MAP; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Step const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Step"); + + xfer += proto->writeFieldBegin("dst", apache::thrift::protocol::T_STRUCT, 1); + xfer += ::apache::thrift::Cpp2Ops::write(proto, &obj->dst); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("type", apache::thrift::protocol::T_I32, 2); + // NOTICE: The original id will be transformed to +1/-1, to indicate the edge + // direction. + auto type = obj->type; + if (type != 0) { + type > 0 ? type = 1 : type = -1; + } + xfer += detail::pm::protocol_methods::write(*proto, type); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("name", apache::thrift::protocol::T_STRING, 3); + xfer += proto->writeBinary(obj->name); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("ranking", apache::thrift::protocol::T_I64, 4); + xfer += detail::pm::protocol_methods::write( + *proto, obj->ranking); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("props", apache::thrift::protocol::T_MAP, 5); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::write(*proto, obj->props); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Step* obj) { + apache::thrift::detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_STRUCT))) { + goto _loop; + } + +_readField_dst : { Cpp2Ops::read(proto, &obj->dst); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_I32))) { + goto _loop; + } + +_readField_type : { + detail::pm::protocol_methods::read(*proto, obj->type); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 3, protocol::T_STRING))) { + goto _loop; + } + +_readField_name : { proto->readBinary(obj->name); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 3, 4, protocol::T_I64))) { + goto _loop; + } + +_readField_ranking : { + detail::pm::protocol_methods::read(*proto, + obj->ranking); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 4, 5, protocol::T_MAP))) { + goto _loop; + } + +_readField_props : { + obj->props = std::unordered_map(); + detail::pm::protocol_methods, + std::unordered_map>::read(*proto, + obj->props); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 5, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRUCT)) { + goto _readField_dst; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I32)) { + goto _readField_type; + } else { + goto _skip; + } + } + case 3: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRING)) { + goto _readField_name; + } else { + goto _skip; + } + } + case 4: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_I64)) { + goto _readField_ranking; + } else { + goto _skip; + } + } + case 5: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_MAP)) { + goto _readField_props; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Step const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Step"); + + xfer += proto->serializedFieldSize("dst", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->dst); + + xfer += proto->serializedFieldSize("type", apache::thrift::protocol::T_I32, 2); + xfer += + detail::pm::protocol_methods::serializedSize( + *proto, obj->type); + + xfer += proto->serializedFieldSize("name", apache::thrift::protocol::T_STRING, 3); + xfer += proto->serializedSizeBinary(obj->name); + + xfer += proto->serializedFieldSize("ranking", apache::thrift::protocol::T_I64, 4); + xfer += detail::pm::protocol_methods::serializedSize(*proto, + obj->ranking); + + xfer += proto->serializedFieldSize("props", apache::thrift::protocol::T_MAP, 5); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->props); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Step const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Step"); + + xfer += proto->serializedFieldSize("dst", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSizeZC(proto, &obj->dst); + + xfer += proto->serializedFieldSize("type", apache::thrift::protocol::T_I32, 2); + xfer += + detail::pm::protocol_methods::serializedSize( + *proto, obj->type); + + xfer += proto->serializedFieldSize("name", apache::thrift::protocol::T_STRING, 3); + xfer += proto->serializedSizeZCBinary(obj->name); + + xfer += proto->serializedFieldSize("ranking", apache::thrift::protocol::T_I64, 4); + xfer += detail::pm::protocol_methods::serializedSize(*proto, + obj->ranking); + + xfer += proto->serializedFieldSize("props", apache::thrift::protocol::T_MAP, 5); + xfer += detail::pm::protocol_methods< + type_class::map, + std::unordered_map>::serializedSize(*proto, obj->props); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +/************************************** + * + * Ops for class Path + * + *************************************/ +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "src") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } else if (_fname == "steps") { + fid = 2; + _ftype = apache::thrift::protocol::T_LIST; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Path const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("Path"); + + xfer += proto->writeFieldBegin("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::write(proto, &obj->src); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldBegin("steps", apache::thrift::protocol::T_LIST, 2); + xfer += detail::pm::protocol_methods, + std::vector>::write(*proto, obj->steps); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Path* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_STRUCT))) { + goto _loop; + } + +_readField_src : { Cpp2Ops::read(proto, &obj->src); } + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 2, protocol::T_LIST))) { + goto _loop; + } + +_readField_steps : { + obj->steps = std::vector(); + detail::pm::protocol_methods, + std::vector>::read(*proto, obj->steps); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 2, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_STRUCT)) { + goto _readField_src; + } else { + goto _skip; + } + } + case 2: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_LIST)) { + goto _readField_steps; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Path const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Path"); + + xfer += proto->serializedFieldSize("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSize(proto, &obj->src); + + xfer += proto->serializedFieldSize("steps", apache::thrift::protocol::T_LIST, 2); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->steps); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Path const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("Path"); + + xfer += proto->serializedFieldSize("src", apache::thrift::protocol::T_STRUCT, 1); + xfer += Cpp2Ops::serializedSizeZC(proto, &obj->src); + + xfer += proto->serializedFieldSize("steps", apache::thrift::protocol::T_LIST, 2); + xfer += + detail::pm::protocol_methods, + std::vector>::serializedSize(*proto, + obj->steps); + + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_PATHOPS_H_ diff --git a/src/common/datatypes/Set.cpp b/src/common/datatypes/Set.cpp new file mode 100644 index 0000000..b100994 --- /dev/null +++ b/src/common/datatypes/Set.cpp @@ -0,0 +1,69 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "common/datatypes/Set.h" + +#include + +#include + +namespace nebula { + +Set Set::set_intersection(const Set& lhs, const Set& rhs) { + if (lhs.size() <= rhs.size()) { + Set iset; + for (const Value& key : lhs.values) { + if (rhs.values.count(key) > 0) { + iset.values.insert(key); + } + } + return iset; + } else { + return set_intersection(rhs, lhs); + } +} + +std::string Set::toString() const { + std::vector value(values.size()); + std::transform(values.begin(), values.end(), value.begin(), [](const auto& v) -> std::string { + return v.toString(); + }); + std::stringstream os; + os << "{" << folly::join(",", value) << "}"; + return os.str(); +} + +folly::dynamic Set::toJson() const { + auto setJsonObj = folly::dynamic::array(); + + for (const auto& val : values) { + setJsonObj.push_back(val.toJson()); + } + + return setJsonObj; +} + +folly::dynamic Set::getMetaData() const { + auto setMetadataObj = folly::dynamic::array(); + + for (const auto& val : values) { + setMetadataObj.push_back(val.getMetaData()); + } + + return setMetadataObj; +} + +} // namespace nebula + +namespace std { +std::size_t hash::operator()(const nebula::Set& s) const { + size_t seed = 0; + for (auto& v : s.values) { + seed ^= hash()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; +} + +} // namespace std diff --git a/src/common/datatypes/Set.h b/src/common/datatypes/Set.h new file mode 100644 index 0000000..b7194a1 --- /dev/null +++ b/src/common/datatypes/Set.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_SET_H_ +#define COMMON_DATATYPES_SET_H_ + +#include + +#include "common/datatypes/Value.h" + +namespace nebula { + +struct Set { + std::unordered_set values; + + Set() = default; + Set(const Set&) = default; + Set(Set&&) noexcept = default; + explicit Set(std::unordered_set value) { + values = std::move(value); + } + + void clear() { + values.clear(); + } + + void __clear() { + clear(); + } + + std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of each element + folly::dynamic getMetaData() const; + static Set set_intersection(const Set& lhs, const Set& rhs); + + Set& operator=(const Set& rhs) { + if (this == &rhs) { + return *this; + } + values = rhs.values; + return *this; + } + + Set& operator=(Set&& rhs) noexcept { + if (this == &rhs) { + return *this; + } + values = std::move(rhs.values); + return *this; + } + + bool operator==(const Set& rhs) const { + return values == rhs.values; + } + + bool contains(const Value& value) const { + return values.count(value) != 0; + } + + size_t size() const { + return values.size(); + } +}; + +inline std::ostream& operator<<(std::ostream& os, const Set& s) { + return os << s.toString(); +} +} // namespace nebula + +namespace std { +template <> +struct hash { + std::size_t operator()(const nebula::Set& s) const; +}; + +} // namespace std +#endif // COMMON_DATATYPES_SET_H_ diff --git a/src/common/datatypes/SetOps-inl.h b/src/common/datatypes/SetOps-inl.h new file mode 100644 index 0000000..3f4d62d --- /dev/null +++ b/src/common/datatypes/SetOps-inl.h @@ -0,0 +1,137 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef COMMON_DATATYPES_SETOPS_H_ +#define COMMON_DATATYPES_SETOPS_H_ + +#include +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/Set.h" + +namespace apache { +namespace thrift { + +namespace detail { + +template <> +struct TccStructTraits { + static void translateFieldName(MAYBE_UNUSED folly::StringPiece _fname, + MAYBE_UNUSED int16_t& fid, + MAYBE_UNUSED apache::thrift::protocol::TType& _ftype) { + if (_fname == "values") { + fid = 1; + _ftype = apache::thrift::protocol::T_STRUCT; + } + } +}; + +} // namespace detail + +inline constexpr protocol::TType Cpp2Ops::thriftType() { + return apache::thrift::protocol::T_STRUCT; +} + +template +uint32_t Cpp2Ops::write(Protocol* proto, nebula::Set const* obj) { + uint32_t xfer = 0; + xfer += proto->writeStructBegin("NSet"); + + xfer += proto->writeFieldBegin("values", apache::thrift::protocol::T_SET, 1); + xfer += + detail::pm::protocol_methods, + std::unordered_set>::write(*proto, obj->values); + xfer += proto->writeFieldEnd(); + + xfer += proto->writeFieldStop(); + xfer += proto->writeStructEnd(); + return xfer; +} + +template +void Cpp2Ops::read(Protocol* proto, nebula::Set* obj) { + detail::ProtocolReaderStructReadState readState; + + readState.readStructBegin(proto); + + using apache::thrift::protocol::TProtocolException; + + if (UNLIKELY(!readState.advanceToNextField(proto, 0, 1, protocol::T_SET))) { + goto _loop; + } + +_readField_values : { + obj->values = std::unordered_set(); + detail::pm::protocol_methods, + std::unordered_set>::read(*proto, obj->values); +} + + if (UNLIKELY(!readState.advanceToNextField(proto, 1, 0, protocol::T_STOP))) { + goto _loop; + } + +_end: + readState.readStructEnd(proto); + + return; + +_loop: + if (readState.fieldType == apache::thrift::protocol::T_STOP) { + goto _end; + } + + if (proto->kUsesFieldNames()) { + detail::TccStructTraits::translateFieldName( + readState.fieldName(), readState.fieldId, readState.fieldType); + } + + switch (readState.fieldId) { + case 1: { + if (LIKELY(readState.fieldType == apache::thrift::protocol::T_SET)) { + goto _readField_values; + } else { + goto _skip; + } + } + default: { +_skip: + proto->skip(readState.fieldType); + readState.readFieldEnd(proto); + readState.readFieldBeginNoInline(proto); + goto _loop; + } + } +} + +template +uint32_t Cpp2Ops::serializedSize(Protocol const* proto, nebula::Set const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NSet"); + xfer += proto->serializedFieldSize("values", apache::thrift::protocol::T_SET, 1); + xfer += detail::pm::protocol_methods< + type_class::set, + std::unordered_set>::serializedSize(*proto, obj->values); + xfer += proto->serializedSizeStop(); + return xfer; +} + +template +uint32_t Cpp2Ops::serializedSizeZC(Protocol const* proto, nebula::Set const* obj) { + uint32_t xfer = 0; + xfer += proto->serializedStructSize("NSet"); + xfer += proto->serializedFieldSize("values", apache::thrift::protocol::T_SET, 1); + xfer += detail::pm::protocol_methods< + type_class::set, + std::unordered_set>::serializedSize(*proto, obj->values); + xfer += proto->serializedSizeStop(); + return xfer; +} + +} // namespace thrift +} // namespace apache +#endif // COMMON_DATATYPES_SETOPS_H_ diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp new file mode 100644 index 0000000..7a6b5e4 --- /dev/null +++ b/src/common/datatypes/Value.cpp @@ -0,0 +1,3042 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include "common/datatypes/Value.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/datatypes/DataSet.h" +#include "common/datatypes/Edge.h" +#include "common/datatypes/Geography.h" +#include "common/datatypes/List.h" +#include "common/datatypes/Map.h" +#include "common/datatypes/Path.h" +#include "common/datatypes/Set.h" +#include "common/datatypes/Vertex.h" + +namespace nebula { + +const Value Value::kEmpty; +const Value Value::kNullValue(NullType::__NULL__); +const Value Value::kNullNaN(NullType::NaN); +const Value Value::kNullBadData(NullType::BAD_DATA); +const Value Value::kNullBadType(NullType::BAD_TYPE); +const Value Value::kNullOverflow(NullType::ERR_OVERFLOW); +const Value Value::kNullUnknownProp(NullType::UNKNOWN_PROP); +const Value Value::kNullDivByZero(NullType::DIV_BY_ZERO); +const Value Value::kNullOutOfRange(NullType::OUT_OF_RANGE); + +const uint64_t Value::kEmptyNullType = Value::Type::__EMPTY__ | Value::Type::NULLVALUE; +const uint64_t Value::kNumericType = Value::Type::INT | Value::Type::FLOAT; + +Value::Value(Value&& rhs) noexcept : type_(Value::Type::__EMPTY__) { + memcpy(reinterpret_cast(this), reinterpret_cast(&rhs), sizeof(*this)); + rhs.type_ = Type::__EMPTY__; +} + +Value::Value(const Value& rhs) : type_(Value::Type::__EMPTY__) { + if (this == &rhs) { + return; + } + if (rhs.type_ == Type::__EMPTY__) { + return; + } + switch (rhs.type_) { + case Type::NULLVALUE: { + setN(rhs.value_.nVal); + break; + } + case Type::BOOL: { + setB(rhs.value_.bVal); + break; + } + case Type::INT: { + setI(rhs.value_.iVal); + break; + } + case Type::FLOAT: { + setF(rhs.value_.fVal); + break; + } + case Type::STRING: { + setS(*rhs.value_.sVal); + break; + } + case Type::DATE: { + setD(rhs.value_.dVal); + break; + } + case Type::TIME: { + setT(rhs.value_.tVal); + break; + } + case Type::DATETIME: { + setDT(rhs.value_.dtVal); + break; + } + case Type::VERTEX: { + setV(rhs.value_.vVal); + break; + } + case Type::EDGE: { + setE(rhs.value_.eVal); + break; + } + case Type::PATH: { + setP(rhs.value_.pVal); + break; + } + case Type::LIST: { + setL(rhs.value_.lVal); + break; + } + case Type::MAP: { + setM(rhs.value_.mVal); + break; + } + case Type::SET: { + setU(rhs.value_.uVal); + break; + } + case Type::DATASET: { + setG(rhs.value_.gVal); + break; + } + case Type::GEOGRAPHY: { + setGG(rhs.value_.ggVal); + break; + } + case Type::DURATION: { + setDU(rhs.value_.duVal); + break; + } + default: { + assert(false); + break; + } + } +} + +Value::Value(const NullType& v) { + setN(v); +} + +Value::Value(NullType&& v) { + setN(std::move(v)); +} + +Value::Value(const bool& v) { + setB(v); +} + +Value::Value(bool&& v) { + setB(std::move(v)); +} + +Value::Value(const int8_t& v) { + setI(v); +} + +Value::Value(int8_t&& v) { + setI(std::move(v)); +} + +Value::Value(const int16_t& v) { + setI(v); +} + +Value::Value(int16_t&& v) { + setI(std::move(v)); +} + +Value::Value(const int32_t& v) { + setI(v); +} + +Value::Value(int32_t&& v) { + setI(std::move(v)); +} + +Value::Value(const int64_t& v) { + setI(v); +} + +Value::Value(int64_t&& v) { + setI(std::move(v)); +} + +Value::Value(const double& v) { + setF(v); +} + +Value::Value(double&& v) { + setF(std::move(v)); +} + +Value::Value(const std::string& v) { + setS(v); +} + +Value::Value(std::string&& v) { + setS(std::move(v)); +} + +Value::Value(const char* v) { + setS(v); +} + +Value::Value(const Date& v) { + setD(v); +} + +Value::Value(Date&& v) { + setD(std::move(v)); +} + +Value::Value(const Time& v) { + setT(v); +} + +Value::Value(Time&& v) { + setT(std::move(v)); +} + +Value::Value(const DateTime& v) { + setDT(v); +} + +Value::Value(DateTime&& v) { + setDT(std::move(v)); +} + +Value::Value(const Vertex& v) { + setV(v); +} + +Value::Value(Vertex&& v) { + setV(std::move(v)); +} + +Value::Value(const Edge& v) { + setE(v); +} + +Value::Value(Edge&& v) { + setE(std::move(v)); +} + +Value::Value(const Path& v) { + setP(v); +} + +Value::Value(Path&& v) { + setP(std::move(v)); +} + +Value::Value(const List& v) { + auto c = std::make_unique(v); + setL(std::move(c)); +} + +Value::Value(List&& v) { + setL(std::make_unique(std::move(v))); +} + +Value::Value(const Map& v) { + auto c = std::make_unique(v); + setM(std::move(c)); +} + +Value::Value(Map&& v) { + setM(std::make_unique(std::move(v))); +} + +Value::Value(const Set& v) { + auto c = std::make_unique(v); + setU(std::move(c)); +} + +Value::Value(Set&& v) { + setU(std::make_unique(std::move(v))); +} + +Value::Value(const DataSet& v) { + auto c = std::make_unique(v); + setG(std::move(c)); +} + +Value::Value(DataSet&& v) { + setG(std::make_unique(std::move(v))); +} + +Value::Value(const Geography& v) { + setGG(std::make_unique(v)); +} + +Value::Value(Geography&& v) { + setGG(std::make_unique(std::move(v))); +} + +Value::Value(const Duration& v) { + setDU(std::make_unique(v)); +} + +Value::Value(Duration&& v) { + setDU(std::make_unique(std::move(v))); +} + +Value::Value(const std::unordered_map& map) { + setM(std::make_unique(map)); +} + +Value::Value(std::unordered_map&& map) { + setM(std::make_unique(std::move(map))); +} + +const std::string& Value::typeName() const { + static const std::unordered_map typeNames = { + {Type::__EMPTY__, "__EMPTY__"}, + {Type::NULLVALUE, "__NULL__"}, + {Type::BOOL, "bool"}, + {Type::INT, "int"}, + {Type::FLOAT, "float"}, + {Type::STRING, "string"}, + {Type::DATE, "date"}, + {Type::TIME, "time"}, + {Type::DATETIME, "datetime"}, + {Type::VERTEX, "vertex"}, + {Type::EDGE, "edge"}, + {Type::PATH, "path"}, + {Type::LIST, "list"}, + {Type::MAP, "map"}, + {Type::SET, "set"}, + {Type::DATASET, "dataset"}, + {Type::GEOGRAPHY, "geography"}, + {Type::DURATION, "duration"}, + }; + + static const std::unordered_map nullTypes = { + {NullType::__NULL__, "__NULL__"}, + {NullType::NaN, "NaN"}, + {NullType::BAD_DATA, "BAD_DATA"}, + {NullType::BAD_TYPE, "BAD_TYPE"}, + {NullType::ERR_OVERFLOW, "ERR_OVERFLOW"}, + {NullType::UNKNOWN_PROP, "UNKNOWN_PROP"}, + {NullType::DIV_BY_ZERO, "DIV_BY_ZERO"}, + }; + + static const std::string unknownType = "__UNKNOWN__"; + + auto find = typeNames.find(type_); + if (find == typeNames.end()) { + return unknownType; + } + if (find->first == Type::NULLVALUE) { + auto nullFind = nullTypes.find(value_.nVal); + if (nullFind == nullTypes.end()) { + return unknownType; + } + return nullFind->second; + } + return find->second; +} + +void Value::setNull(const NullType& v) { + clear(); + setN(v); +} + +void Value::setNull(NullType&& v) { + clear(); + setN(std::move(v)); +} + +void Value::setBool(const bool& v) { + clear(); + setB(v); +} + +void Value::setBool(bool&& v) { + clear(); + setB(std::move(v)); +} + +void Value::setInt(const int8_t& v) { + clear(); + setI(v); +} + +void Value::setInt(int8_t&& v) { + clear(); + setI(std::move(v)); +} + +void Value::setInt(const int16_t& v) { + clear(); + setI(v); +} + +void Value::setInt(int16_t&& v) { + clear(); + setI(std::move(v)); +} + +void Value::setInt(const int32_t& v) { + clear(); + setI(v); +} + +void Value::setInt(int32_t&& v) { + clear(); + setI(std::move(v)); +} + +void Value::setInt(const int64_t& v) { + clear(); + setI(v); +} + +void Value::setInt(int64_t&& v) { + clear(); + setI(std::move(v)); +} + +void Value::setFloat(const double& v) { + clear(); + setF(v); +} + +void Value::setFloat(double&& v) { + clear(); + setF(std::move(v)); +} + +void Value::setStr(const std::string& v) { + clear(); + setS(v); +} + +void Value::setStr(std::string&& v) { + clear(); + setS(std::move(v)); +} + +void Value::setStr(const char* v) { + clear(); + setS(v); +} + +void Value::setDate(const Date& v) { + clear(); + setD(v); +} + +void Value::setDate(Date&& v) { + clear(); + setD(std::move(v)); +} + +void Value::setTime(const Time& v) { + clear(); + setT(v); +} + +void Value::setTime(Time&& v) { + clear(); + setT(std::move(v)); +} + +void Value::setDateTime(const DateTime& v) { + clear(); + setDT(v); +} + +void Value::setDateTime(DateTime&& v) { + clear(); + setDT(std::move(v)); +} + +void Value::setVertex(const Vertex& v) { + clear(); + setV(v); +} + +void Value::setVertex(Vertex&& v) { + clear(); + setV(std::move(v)); +} + +void Value::setVertex(Vertex* v) { + clear(); + setV(v); +} + +void Value::setEdge(const Edge& v) { + clear(); + setE(v); +} + +void Value::setEdge(Edge&& v) { + clear(); + setE(std::move(v)); +} + +void Value::setEdge(Edge* v) { + clear(); + setE(v); +} + +void Value::setPath(const Path& v) { + clear(); + setP(v); +} + +void Value::setPath(Path&& v) { + clear(); + setP(std::move(v)); +} + +void Value::setPath(std::unique_ptr&& v) { + clear(); + setP(std::move(v)); +} + +void Value::setList(const List& v) { + clear(); + setL(v); +} + +void Value::setList(List&& v) { + clear(); + setL(std::move(v)); +} + +void Value::setList(std::unique_ptr&& v) { + clear(); + setL(std::move(v)); +} + +void Value::setMap(const Map& v) { + clear(); + setM(v); +} + +void Value::setMap(Map&& v) { + clear(); + setM(std::move(v)); +} + +void Value::setMap(std::unique_ptr&& v) { + clear(); + setM(std::move(v)); +} + +void Value::setSet(const Set& v) { + clear(); + setU(v); +} + +void Value::setSet(Set&& v) { + clear(); + setU(std::move(v)); +} + +void Value::setSet(std::unique_ptr&& v) { + clear(); + setU(std::move(v)); +} + +void Value::setDataSet(const DataSet& v) { + clear(); + setG(v); +} + +void Value::setDataSet(DataSet&& v) { + clear(); + setG(std::move(v)); +} + +void Value::setDataSet(std::unique_ptr&& v) { + clear(); + setG(std::move(v)); +} + +void Value::setGeography(const Geography& v) { + clear(); + setGG(v); +} + +void Value::setGeography(Geography&& v) { + clear(); + setGG(std::move(v)); +} + +void Value::setGeography(std::unique_ptr&& v) { + clear(); + setGG(std::move(v)); +} + +void Value::setDuration(const Duration& v) { + clear(); + setDU(v); +} + +void Value::setDuration(Duration&& v) { + clear(); + setDU(std::move(v)); +} + +void Value::setDuration(std::unique_ptr&& v) { + clear(); + setDU(std::move(v)); +} + +const double& Value::getFloat() const { + CHECK_EQ(type_, Type::FLOAT); + return value_.fVal; +} + +const std::string& Value::getStr() const { + CHECK_EQ(type_, Type::STRING); + return *value_.sVal; +} + +const Date& Value::getDate() const { + CHECK_EQ(type_, Type::DATE); + return value_.dVal; +} + +const Time& Value::getTime() const { + CHECK_EQ(type_, Type::TIME); + return value_.tVal; +} + +const DateTime& Value::getDateTime() const { + CHECK_EQ(type_, Type::DATETIME); + return value_.dtVal; +} + +const Vertex& Value::getVertex() const { + CHECK_EQ(type_, Type::VERTEX); + return *(value_.vVal); +} + +const Vertex* Value::getVertexPtr() const { + CHECK_EQ(type_, Type::VERTEX); + return value_.vVal; +} + +const Edge& Value::getEdge() const { + CHECK_EQ(type_, Type::EDGE); + return *(value_.eVal); +} + +const Edge* Value::getEdgePtr() const { + CHECK_EQ(type_, Type::EDGE); + return value_.eVal; +} + +const Path& Value::getPath() const { + CHECK_EQ(type_, Type::PATH); + return *(value_.pVal); +} + +const Path* Value::getPathPtr() const { + CHECK_EQ(type_, Type::PATH); + return value_.pVal.get(); +} + +const List& Value::getList() const { + CHECK_EQ(type_, Type::LIST); + return *(value_.lVal); +} + +const List* Value::getListPtr() const { + CHECK_EQ(type_, Type::LIST); + return value_.lVal.get(); +} + +const Map& Value::getMap() const { + CHECK_EQ(type_, Type::MAP); + return *(value_.mVal); +} + +const Map* Value::getMapPtr() const { + CHECK_EQ(type_, Type::MAP); + return value_.mVal.get(); +} + +const Set& Value::getSet() const { + CHECK_EQ(type_, Type::SET); + return *(value_.uVal); +} + +const Set* Value::getSetPtr() const { + CHECK_EQ(type_, Type::SET); + return value_.uVal.get(); +} + +const DataSet& Value::getDataSet() const { + CHECK_EQ(type_, Type::DATASET); + return *(value_.gVal); +} + +const DataSet* Value::getDataSetPtr() const { + CHECK_EQ(type_, Type::DATASET); + return value_.gVal.get(); +} + +const Geography& Value::getGeography() const { + CHECK_EQ(type_, Type::GEOGRAPHY); + return *(value_.ggVal); +} + +const Geography* Value::getGeographyPtr() const { + CHECK_EQ(type_, Type::GEOGRAPHY); + return value_.ggVal.get(); +} + +const Duration& Value::getDuration() const { + CHECK_EQ(type_, Type::DURATION); + return *value_.duVal; +} + +const Duration* Value::getDurationPtr() const { + CHECK_EQ(type_, Type::DURATION); + return value_.duVal.get(); +} + +NullType& Value::mutableNull() { + CHECK_EQ(type_, Type::NULLVALUE); + return value_.nVal; +} + +bool& Value::mutableBool() { + CHECK_EQ(type_, Type::BOOL); + return value_.bVal; +} + +int64_t& Value::mutableInt() { + CHECK_EQ(type_, Type::INT); + return value_.iVal; +} + +double& Value::mutableFloat() { + CHECK_EQ(type_, Type::FLOAT); + return value_.fVal; +} + +std::string& Value::mutableStr() { + CHECK_EQ(type_, Type::STRING); + return *value_.sVal; +} + +Date& Value::mutableDate() { + CHECK_EQ(type_, Type::DATE); + return value_.dVal; +} + +Time& Value::mutableTime() { + CHECK_EQ(type_, Type::TIME); + return value_.tVal; +} + +DateTime& Value::mutableDateTime() { + CHECK_EQ(type_, Type::DATETIME); + return value_.dtVal; +} + +Vertex& Value::mutableVertex() { + CHECK_EQ(type_, Type::VERTEX); + return *(value_.vVal); +} + +Edge& Value::mutableEdge() { + CHECK_EQ(type_, Type::EDGE); + return *(value_.eVal); +} + +Path& Value::mutablePath() { + CHECK_EQ(type_, Type::PATH); + return *(value_.pVal); +} + +List& Value::mutableList() { + CHECK_EQ(type_, Type::LIST); + return *(value_.lVal); +} + +Map& Value::mutableMap() { + CHECK_EQ(type_, Type::MAP); + return *(value_.mVal); +} + +Set& Value::mutableSet() { + CHECK_EQ(type_, Type::SET); + return *(value_.uVal); +} + +DataSet& Value::mutableDataSet() { + CHECK_EQ(type_, Type::DATASET); + return *(value_.gVal); +} + +Geography& Value::mutableGeography() { + CHECK_EQ(type_, Type::GEOGRAPHY); + return *(value_.ggVal); +} + +Duration& Value::mutableDuration() { + CHECK_EQ(type_, Type::DURATION); + return *value_.duVal; +} + +NullType Value::moveNull() { + CHECK_EQ(type_, Type::NULLVALUE); + NullType v = std::move(value_.nVal); + clear(); + return std::move(v); +} + +bool Value::moveBool() { + CHECK_EQ(type_, Type::BOOL); + bool v = std::move(value_.bVal); + clear(); + return std::move(v); +} + +int64_t Value::moveInt() { + CHECK_EQ(type_, Type::INT); + int64_t v = std::move(value_.iVal); + clear(); + return std::move(v); +} + +double Value::moveFloat() { + CHECK_EQ(type_, Type::FLOAT); + double v = std::move(value_.fVal); + clear(); + return std::move(v); +} + +std::string Value::moveStr() { + CHECK_EQ(type_, Type::STRING); + std::string v = std::move(*value_.sVal); + clear(); + return v; +} + +Date Value::moveDate() { + CHECK_EQ(type_, Type::DATE); + Date v = std::move(value_.dVal); + clear(); + return v; +} + +Time Value::moveTime() { + CHECK_EQ(type_, Type::TIME); + Time v = std::move(value_.tVal); + clear(); + return v; +} + +DateTime Value::moveDateTime() { + CHECK_EQ(type_, Type::DATETIME); + DateTime v = std::move(value_.dtVal); + clear(); + return v; +} + +Vertex Value::moveVertex() { + CHECK_EQ(type_, Type::VERTEX); + Vertex v = std::move(*(value_.vVal)); + clear(); + return v; +} + +Edge Value::moveEdge() { + CHECK_EQ(type_, Type::EDGE); + Edge v = std::move(*(value_.eVal)); + clear(); + return v; +} + +Path Value::movePath() { + CHECK_EQ(type_, Type::PATH); + Path v = std::move(*(value_.pVal)); + clear(); + return v; +} + +List Value::moveList() { + CHECK_EQ(type_, Type::LIST); + List list = std::move(*(value_.lVal)); + clear(); + return list; +} + +Map Value::moveMap() { + CHECK_EQ(type_, Type::MAP); + Map map = std::move(*(value_.mVal)); + clear(); + return map; +} + +Set Value::moveSet() { + CHECK_EQ(type_, Type::SET); + Set set = std::move(*(value_.uVal)); + clear(); + return set; +} + +DataSet Value::moveDataSet() { + CHECK_EQ(type_, Type::DATASET); + DataSet ds = std::move(*(value_.gVal)); + clear(); + return ds; +} + +Geography Value::moveGeography() { + CHECK_EQ(type_, Type::GEOGRAPHY); + Geography v = std::move(*(value_.ggVal)); + clear(); + return v; +} + +Duration Value::moveDuration() { + CHECK_EQ(type_, Type::DURATION); + Duration v = std::move(*value_.duVal); + clear(); + return v; +} + +void Value::clearSlow() { + switch (type_) { + case Type::__EMPTY__: { + return; + } + case Type::NULLVALUE: { + destruct(value_.nVal); + break; + } + case Type::BOOL: { + destruct(value_.bVal); + break; + } + case Type::INT: { + destruct(value_.iVal); + break; + } + case Type::FLOAT: { + destruct(value_.fVal); + break; + } + case Type::STRING: { + destruct(value_.sVal); + break; + } + case Type::DATE: { + destruct(value_.dVal); + break; + } + case Type::TIME: { + destruct(value_.tVal); + break; + } + case Type::DATETIME: { + destruct(value_.dtVal); + break; + } + case Type::VERTEX: { + if (value_.vVal) { + if (value_.vVal->unref() == 0) { + delete value_.vVal; + } + value_.vVal = nullptr; + type_ = Type::__EMPTY__; + } + break; + } + case Type::EDGE: { + if (value_.eVal) { + if (value_.eVal->unref() == 0) { + delete value_.eVal; + } + value_.eVal = nullptr; + type_ = Type::__EMPTY__; + } + break; + } + case Type::PATH: { + destruct(value_.pVal); + break; + } + case Type::LIST: { + destruct(value_.lVal); + break; + } + case Type::MAP: { + destruct(value_.mVal); + break; + } + case Type::SET: { + destruct(value_.uVal); + break; + } + case Type::DATASET: { + destruct(value_.gVal); + break; + } + case Type::GEOGRAPHY: { + destruct(value_.ggVal); + break; + } + case Type::DURATION: { + destruct(value_.duVal); + break; + } + } + type_ = Type::__EMPTY__; +} + +Value& Value::operator=(Value&& rhs) noexcept { + if (this == &rhs) { + return *this; + } + clear(); + if (rhs.type_ == Type::__EMPTY__) { + return *this; + } + switch (rhs.type_) { + case Type::NULLVALUE: { + setN(std::move(rhs.value_.nVal)); + break; + } + case Type::BOOL: { + setB(std::move(rhs.value_.bVal)); + break; + } + case Type::INT: { + setI(std::move(rhs.value_.iVal)); + break; + } + case Type::FLOAT: { + setF(std::move(rhs.value_.fVal)); + break; + } + case Type::STRING: { + setS(std::move(rhs.value_.sVal)); + break; + } + case Type::DATE: { + setD(std::move(rhs.value_.dVal)); + break; + } + case Type::TIME: { + setT(std::move(rhs.value_.tVal)); + break; + } + case Type::DATETIME: { + setDT(std::move(rhs.value_.dtVal)); + break; + } + case Type::VERTEX: { + value_.vVal = rhs.value_.vVal; + type_ = Type::VERTEX; + rhs.value_.vVal = nullptr; + rhs.type_ = Type::__EMPTY__; + return *this; + } + case Type::EDGE: { + value_.eVal = rhs.value_.eVal; + type_ = Type::EDGE; + rhs.value_.eVal = nullptr; + rhs.type_ = Type::__EMPTY__; + return *this; + } + case Type::PATH: { + setP(std::move(rhs.value_.pVal)); + break; + } + case Type::LIST: { + setL(std::move(rhs.value_.lVal)); + break; + } + case Type::MAP: { + setM(std::move(rhs.value_.mVal)); + break; + } + case Type::SET: { + setU(std::move(rhs.value_.uVal)); + break; + } + case Type::DATASET: { + setG(std::move(rhs.value_.gVal)); + break; + } + case Type::GEOGRAPHY: { + setGG(std::move(rhs.value_.ggVal)); + break; + } + case Type::DURATION: { + setDU(std::move(rhs.value_.duVal)); + break; + } + default: { + assert(false); + break; + } + } + rhs.clear(); + return *this; +} + +Value& Value::operator=(const Value& rhs) { + if (this == &rhs) { + return *this; + } + clear(); + if (rhs.type_ == Type::__EMPTY__) { + return *this; + } + switch (rhs.type_) { + case Type::NULLVALUE: { + setN(rhs.value_.nVal); + break; + } + case Type::BOOL: { + setB(rhs.value_.bVal); + break; + } + case Type::INT: { + setI(rhs.value_.iVal); + break; + } + case Type::FLOAT: { + setF(rhs.value_.fVal); + break; + } + case Type::STRING: { + setS(*rhs.value_.sVal); + break; + } + case Type::DATE: { + setD(rhs.value_.dVal); + break; + } + case Type::TIME: { + setT(rhs.value_.tVal); + break; + } + case Type::DATETIME: { + setDT(rhs.value_.dtVal); + break; + } + case Type::VERTEX: { + setV(rhs.value_.vVal); + break; + } + case Type::EDGE: { + setE(rhs.value_.eVal); + break; + } + case Type::PATH: { + setP(rhs.value_.pVal); + break; + } + case Type::LIST: { + setL(rhs.value_.lVal); + break; + } + case Type::MAP: { + setM(rhs.value_.mVal); + break; + } + case Type::SET: { + setU(rhs.value_.uVal); + break; + } + case Type::DATASET: { + setG(rhs.value_.gVal); + break; + } + case Type::GEOGRAPHY: { + setGG(rhs.value_.ggVal); + break; + } + case Type::DURATION: { + setDU(rhs.value_.duVal); + break; + } + default: { + assert(false); + break; + } + } + type_ = rhs.type_; + return *this; +} + +void Value::setN(const NullType& v) { + new (std::addressof(value_.nVal)) NullType(v); + type_ = Type::NULLVALUE; +} + +void Value::setN(NullType&& v) { + new (std::addressof(value_.nVal)) NullType(std::move(v)); + type_ = Type::NULLVALUE; +} + +void Value::setB(const bool& v) { + new (std::addressof(value_.bVal)) bool(v); // NOLINT + type_ = Type::BOOL; +} + +void Value::setB(bool&& v) { + new (std::addressof(value_.bVal)) bool(std::move(v)); // NOLINT + type_ = Type::BOOL; +} + +void Value::setI(const int64_t& v) { + new (std::addressof(value_.iVal)) int64_t(v); // NOLINT + type_ = Type::INT; +} + +void Value::setI(int64_t&& v) { + new (std::addressof(value_.iVal)) int64_t(std::move(v)); // NOLINT + type_ = Type::INT; +} + +void Value::setF(const double& v) { + new (std::addressof(value_.fVal)) double(v); // NOLINT + type_ = Type::FLOAT; +} + +void Value::setF(double&& v) { + new (std::addressof(value_.fVal)) double(std::move(v)); // NOLINT + type_ = Type::FLOAT; +} + +void Value::setS(std::unique_ptr v) { + new (std::addressof(value_.sVal)) std::unique_ptr(std::move(v)); + type_ = Type::STRING; +} + +void Value::setS(const std::string& v) { + new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; +} + +void Value::setS(std::string&& v) { + new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(std::move(v))); + type_ = Type::STRING; +} + +void Value::setS(const char* v) { + new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; +} + +void Value::setD(const Date& v) { + new (std::addressof(value_.dVal)) Date(v); + type_ = Type::DATE; +} + +void Value::setD(Date&& v) { + new (std::addressof(value_.dVal)) Date(std::move(v)); + type_ = Type::DATE; +} + +void Value::setT(const Time& v) { + new (std::addressof(value_.tVal)) Time(v); + type_ = Type::TIME; +} + +void Value::setT(Time&& v) { + new (std::addressof(value_.tVal)) Time(std::move(v)); + type_ = Type::TIME; +} + +void Value::setDT(const DateTime& v) { + new (std::addressof(value_.dtVal)) DateTime(v); + type_ = Type::DATETIME; +} + +void Value::setDT(DateTime&& v) { + new (std::addressof(value_.dtVal)) DateTime(std::move(v)); + type_ = Type::DATETIME; +} + +void Value::setV(Vertex* v) { + value_.vVal = v; + value_.vVal->ref(); + type_ = Type::VERTEX; +} + +void Value::setV(const Vertex& v) { + new (std::addressof(value_.vVal)) Vertex*(new Vertex(v)); + type_ = Type::VERTEX; +} + +void Value::setV(Vertex&& v) { + new (std::addressof(value_.vVal)) Vertex*(new Vertex(std::move(v))); + type_ = Type::VERTEX; +} + +void Value::setE(Edge* v) { + value_.eVal = v; + value_.eVal->ref(); + type_ = Type::EDGE; +} + +void Value::setE(const Edge& v) { + new (std::addressof(value_.eVal)) Edge*(new Edge(v)); + type_ = Type::EDGE; +} + +void Value::setE(Edge&& v) { + new (std::addressof(value_.eVal)) Edge*(new Edge(std::move(v))); + type_ = Type::EDGE; +} + +void Value::setP(const std::unique_ptr& v) { + new (std::addressof(value_.pVal)) std::unique_ptr(new Path(*v)); + type_ = Type::PATH; +} + +void Value::setP(std::unique_ptr&& v) { + new (std::addressof(value_.pVal)) std::unique_ptr(std::move(v)); + type_ = Type::PATH; +} + +void Value::setP(const Path& v) { + new (std::addressof(value_.pVal)) std::unique_ptr(new Path(v)); + type_ = Type::PATH; +} + +void Value::setP(Path&& v) { + new (std::addressof(value_.pVal)) std::unique_ptr(new Path(std::move(v))); + type_ = Type::PATH; +} + +void Value::setL(const std::unique_ptr& v) { + new (std::addressof(value_.lVal)) std::unique_ptr(new List(*v)); + type_ = Type::LIST; +} + +void Value::setL(std::unique_ptr&& v) { + new (std::addressof(value_.lVal)) std::unique_ptr(std::move(v)); + type_ = Type::LIST; +} + +void Value::setL(const List& v) { + new (std::addressof(value_.lVal)) std::unique_ptr(new List(v)); + type_ = Type::LIST; +} + +void Value::setL(List&& v) { + new (std::addressof(value_.lVal)) std::unique_ptr(new List(std::move(v))); + type_ = Type::LIST; +} + +void Value::setM(const std::unique_ptr& v) { + new (std::addressof(value_.mVal)) std::unique_ptr(new Map(*v)); + type_ = Type::MAP; +} + +void Value::setM(std::unique_ptr&& v) { + new (std::addressof(value_.mVal)) std::unique_ptr(std::move(v)); + type_ = Type::MAP; +} + +void Value::setM(const Map& v) { + new (std::addressof(value_.mVal)) std::unique_ptr(new Map(v)); + type_ = Type::MAP; +} + +void Value::setM(Map&& v) { + new (std::addressof(value_.mVal)) std::unique_ptr(new Map(std::move(v))); + type_ = Type::MAP; +} + +void Value::setU(const std::unique_ptr& v) { + new (std::addressof(value_.uVal)) std::unique_ptr(new Set(*v)); + type_ = Type::SET; +} + +void Value::setU(std::unique_ptr&& v) { + new (std::addressof(value_.uVal)) std::unique_ptr(std::move(v)); + type_ = Type::SET; +} + +void Value::setU(const Set& v) { + new (std::addressof(value_.uVal)) std::unique_ptr(new Set(v)); + type_ = Type::SET; +} + +void Value::setU(Set&& v) { + new (std::addressof(value_.vVal)) std::unique_ptr(new Set(std::move(v))); + type_ = Type::SET; +} + +void Value::setG(const std::unique_ptr& v) { + new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(*v)); + type_ = Type::DATASET; +} + +void Value::setG(std::unique_ptr&& v) { + new (std::addressof(value_.gVal)) std::unique_ptr(std::move(v)); + type_ = Type::DATASET; +} + +void Value::setG(const DataSet& v) { + new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(v)); + type_ = Type::DATASET; +} + +void Value::setG(DataSet&& v) { + new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(std::move(v))); + type_ = Type::DATASET; +} + +void Value::setGG(const std::unique_ptr& v) { + new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(*v)); + type_ = Type::GEOGRAPHY; +} + +void Value::setGG(std::unique_ptr&& v) { + new (std::addressof(value_.ggVal)) std::unique_ptr(std::move(v)); + type_ = Type::GEOGRAPHY; +} + +void Value::setGG(const Geography& v) { + new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(v)); + type_ = Type::GEOGRAPHY; +} + +void Value::setGG(Geography&& v) { + new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(std::move(v))); + type_ = Type::GEOGRAPHY; +} + +void Value::setDU(const std::unique_ptr& v) { + new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(*v)); + type_ = Type::DURATION; +} + +void Value::setDU(std::unique_ptr&& v) { + new (std::addressof(value_.duVal)) std::unique_ptr(std::move(v)); + type_ = Type::DURATION; +} + +void Value::setDU(const Duration& v) { + new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(v)); + type_ = Type::DURATION; +} + +void Value::setDU(Duration&& v) { + new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(std::move(v))); + type_ = Type::DURATION; +} + +// Convert Nebula::Value to a value compatible with Json standard +// DATE, TIME, DATETIME will be converted to strings in UTC +// VERTEX, EDGES, PATH will be converted to objects +folly::dynamic Value::toJson() const { + switch (type_) { + case Value::Type::__EMPTY__: { + return "__EMPTY__"; + } + // Json null + case Value::Type::NULLVALUE: { + return folly::dynamic(nullptr); + } + // Json bool + case Value::Type::BOOL: { + return folly::dynamic(getBool()); + } + // Json int + case Value::Type::INT: { + return folly::dynamic(getInt()); + } + // json double + case Value::Type::FLOAT: { + return folly::dynamic(getFloat()); + } + // json string + case Value::Type::STRING: { + return folly::dynamic(getStr()); + } + // Json array + case Value::Type::LIST: { + return getList().toJson(); + } + case Value::Type::SET: { + return getSet().toJson(); + } + // Json object + case Value::Type::MAP: { + return getMap().toJson(); + } + case Value::Type::DATE: { + return getDate().toJson(); + } + case Value::Type::TIME: { + return getTime().toJson(); + } + case Value::Type::DATETIME: { + return getDateTime().toJson(); + } + case Value::Type::EDGE: { + return getEdge().toJson(); + } + case Value::Type::VERTEX: { + return getVertex().toJson(); + } + case Value::Type::PATH: { + return getPath().toJson(); + } + case Value::Type::DATASET: { + return getDataSet().toJson(); + } + case Value::Type::GEOGRAPHY: { + return getGeography().toJson(); + } + case Value::Type::DURATION: { + return getDuration().toJson(); + } + // no default so the compiler will warning when lack + } + + DLOG(FATAL) << "Unknown value type " << static_cast(type_); + return folly::dynamic(nullptr); +} + +folly::dynamic Value::getMetaData() const { + auto dynamicObj = folly::dynamic(); + switch (type_) { + // Privative datatypes has no meta data + case Value::Type::__EMPTY__: + case Value::Type::BOOL: + case Value::Type::INT: + case Value::Type::FLOAT: + case Value::Type::STRING: + case Value::Type::DATASET: + case Value::Type::NULLVALUE: { + return folly::dynamic(nullptr); + } + // Extract the meta info of each element as the metadata of the container + case Value::Type::LIST: { + return getList().getMetaData(); + } + case Value::Type::SET: { + return getSet().getMetaData(); + } + case Value::Type::MAP: { + return getMap().getMetaData(); + } + case Value::Type::DURATION: + case Value::Type::DATE: + case Value::Type::TIME: + case Value::Type::DATETIME: { + return folly::dynamic::object("type", typeName()); + } + case Value::Type::VERTEX: { + return getVertex().getMetaData(); + } + case Value::Type::EDGE: { + return getEdge().getMetaData(); + } + case Value::Type::PATH: { + return getPath().getMetaData(); + } + default: + break; + } + + DLOG(FATAL) << "Unknown value type " << static_cast(type_); + return folly::dynamic(nullptr); +} + +std::string Value::toString() const { + switch (type_) { + case Value::Type::__EMPTY__: { + return "__EMPTY__"; + } + case Value::Type::NULLVALUE: { + switch (getNull()) { + case NullType::__NULL__: + return "NULL"; + case NullType::BAD_DATA: + return "__NULL_BAD_DATA__"; + case NullType::BAD_TYPE: + return "__NULL_BAD_TYPE__"; + case NullType::DIV_BY_ZERO: + return "__NULL_DIV_BY_ZERO__"; + case NullType::ERR_OVERFLOW: + return "__NULL_OVERFLOW__"; + case NullType::NaN: + return "__NULL_NaN__"; + case NullType::UNKNOWN_PROP: + return "__NULL_UNKNOWN_PROP__"; + case NullType::OUT_OF_RANGE: + return "__NULL_OUT_OF_RANGE__"; + } + DLOG(FATAL) << "Unknown Null type " << static_cast(getNull()); + return "__NULL_BAD_TYPE__"; + } + case Value::Type::BOOL: { + return getBool() ? "true" : "false"; + } + case Value::Type::INT: { + return folly::to(getInt()); + } + case Value::Type::FLOAT: { + return folly::to(getFloat()); + } + case Value::Type::STRING: { + return "\"" + getStr() + "\""; + } + case Value::Type::DATE: { + return getDate().toString(); + } + case Value::Type::TIME: { + return getTime().toString(); + } + case Value::Type::DATETIME: { + return getDateTime().toString(); + } + case Value::Type::EDGE: { + return getEdge().toString(); + } + case Value::Type::VERTEX: { + return getVertex().toString(); + } + case Value::Type::PATH: { + return getPath().toString(); + } + case Value::Type::LIST: { + return getList().toString(); + } + case Value::Type::SET: { + return getSet().toString(); + } + case Value::Type::MAP: { + return getMap().toString(); + } + case Value::Type::DATASET: { + return getDataSet().toString(); + } + case Value::Type::GEOGRAPHY: { + return getGeography().toString(); + } + case Value::Type::DURATION: { + return getDuration().toString(); + } + // no default so the compiler will warning when lack + } + + DLOG(FATAL) << "Unknown value type " << static_cast(type_); + return "__NULL_BAD_TYPE__"; +} + +Value Value::toBool() const { + switch (type_) { + case Value::Type::__EMPTY__: + case Value::Type::NULLVALUE: { + return Value::kNullValue; + } + case Value::Type::BOOL: { + return *this; + } + case Value::Type::STRING: { + auto str = getStr(); + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + if (str.compare("true") == 0) { + return true; + } + if (str.compare("false") == 0) { + return false; + } + return Value::kNullValue; + } + default: { + return Value::kNullBadType; + } + } +} + +Value Value::toFloat() const { + switch (type_) { + case Value::Type::__EMPTY__: + case Value::Type::NULLVALUE: { + return Value::kNullValue; + } + case Value::Type::INT: { + return static_cast(getInt()); + } + case Value::Type::FLOAT: { + return *this; + } + case Value::Type::STRING: { + const auto& str = getStr(); + char* pEnd; + double val = strtod(str.c_str(), &pEnd); + if (*pEnd != '\0') { + return Value::kNullValue; + } + return val; + } + default: { + return Value::kNullBadType; + } + } +} + +Value Value::toInt() const { + switch (type_) { + case Value::Type::__EMPTY__: + case Value::Type::NULLVALUE: { + return Value::kNullValue; + } + case Value::Type::INT: { + return *this; + } + case Value::Type::FLOAT: { + // Check if float value is in the range of int_64 + // Return min/max int_64 value and false to accommodate Cypher + if (getFloat() <= std::numeric_limits::min()) { + return std::numeric_limits::min(); + } else if (getFloat() >= static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return static_cast(getFloat()); + } + case Value::Type::STRING: { + const auto& str = getStr(); + errno = 0; + char* pEnd; + auto it = std::find(str.begin(), str.end(), '.'); + + auto checkSciNotation = [](std::string s) { + auto it1 = std::find(s.begin(), s.end(), 'e'); + auto it2 = std::find(s.begin(), s.end(), 'E'); + + return (it1 != s.end() || it2 != s.end()); + }; + + auto isSciNotation = checkSciNotation(str); + int64_t val = (it != str.end() || + isSciNotation) // if the string contains '.' or 'e' or 'E', parse as a double + ? int64_t(strtod(str.c_str(), &pEnd)) // string representing double + : int64_t(strtoll(str.c_str(), &pEnd, 10)); // string representing int + if (*pEnd != '\0') { + return Value::kNullValue; + } + + // Check overflow + if ((val == std::numeric_limits::max() || + val == std::numeric_limits::min()) && + errno == ERANGE) { + return kNullOverflow; + } + return val; + } + default: { + return Value::kNullBadType; + } + } +} + +Value Value::toSet() const { + switch (type_) { + case Value::Type::__EMPTY__: + case Value::Type::NULLVALUE: { + return Value::kNullValue; + } + case Value::Type::SET: { + return *this; + } + case Value::Type::LIST: { + Set set; + for (auto& item : getList().values) { + set.values.emplace(item); + } + return set; + } + default: { + return Value::kNullBadType; + } + } +} +Value Value::lessThan(const Value& v) const { + if (UNLIKELY(empty() || v.empty())) { + return Value::kNullValue; + } + auto vType = v.type(); + auto hasNull = (type_ | vType) & Value::Type::NULLVALUE; + auto notSameType = type_ != vType; + auto notBothNumeric = ((type_ | vType) & Value::kNumericType) != Value::kNumericType; + if (hasNull || (notSameType && notBothNumeric)) { + return Value::kNullValue; + } + + switch (type_) { + case Value::Type::BOOL: { + return getBool() < v.getBool(); + } + case Value::Type::INT: { + switch (vType) { + case Value::Type::INT: { + return getInt() < v.getInt(); + } + case Value::Type::FLOAT: { + return (std::abs(getInt() - v.getFloat()) >= kEpsilon && getInt() < v.getFloat()); + } + default: { + return kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (vType) { + case Value::Type::INT: { + return (std::abs(getFloat() - v.getInt()) >= kEpsilon && getFloat() < v.getInt()); + } + case Value::Type::FLOAT: { + return (std::abs(getFloat() - v.getFloat()) >= kEpsilon && getFloat() < v.getFloat()); + } + default: { + return kNullBadType; + } + } + } + case Value::Type::STRING: { + return getStr() < v.getStr(); + } + case Value::Type::DATE: { + return getDate() < v.getDate(); + } + case Value::Type::TIME: { + // TODO(shylock) convert to UTC then compare + return getTime() < v.getTime(); + } + case Value::Type::DATETIME: { + // TODO(shylock) convert to UTC then compare + return getDateTime() < v.getDateTime(); + } + case Value::Type::VERTEX: { + return getVertex() < v.getVertex(); + } + case Value::Type::EDGE: { + return getEdge() < v.getEdge(); + } + case Value::Type::PATH: { + return getPath() < v.getPath(); + } + case Value::Type::LIST: { + return getList() < v.getList(); + } + case Value::Type::MAP: { + return getMap() < v.getMap(); + } + case Value::Type::SET: { + return getSet() < v.getSet(); + } + case Value::Type::DATASET: { + return getDataSet() < v.getDataSet(); + } + case Value::Type::GEOGRAPHY: { + return getGeography() < v.getGeography(); + } + case Value::Type::DURATION: { + // Duration can't compare, + // e.g. What is the result of `duration('P1M') < duration('P30D')`? + return kNullBadType; + } + case Value::Type::NULLVALUE: + case Value::Type::__EMPTY__: { + return kNullBadType; + } + } + DLOG(FATAL) << "Unknown type " << static_cast(v.type()); + return Value::kNullBadType; +} + +Value Value::equal(const Value& v) const { + if (UNLIKELY(empty() || v.empty())) { + return !empty() || !v.empty() ? false : Value::kNullValue; + } + auto vType = v.type(); + auto hasNull = (type_ | vType) & Value::Type::NULLVALUE; + auto notSameType = type_ != vType; + auto notBothNumeric = ((type_ | vType) & Value::kNumericType) != Value::kNumericType; + if (hasNull) return Value::kNullValue; + if (notSameType && notBothNumeric) { + return false; + } + + switch (type_) { + case Value::Type::BOOL: { + return getBool() == v.getBool(); + } + case Value::Type::INT: { + switch (vType) { + case Value::Type::INT: { + return getInt() == v.getInt(); + } + case Value::Type::FLOAT: { + return std::abs(getInt() - v.getFloat()) < kEpsilon; + } + default: { + return kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (vType) { + case Value::Type::INT: { + return std::abs(getFloat() - v.getInt()) < kEpsilon; + } + case Value::Type::FLOAT: { + return std::abs(getFloat() - v.getFloat()) < kEpsilon; + } + default: { + return kNullBadType; + } + } + } + case Value::Type::STRING: { + return getStr() == v.getStr(); + } + case Value::Type::DATE: { + return getDate() == v.getDate(); + } + case Value::Type::TIME: { + // TODO(shylock) convert to UTC then compare + return getTime() == v.getTime(); + } + case Value::Type::DATETIME: { + // TODO(shylock) convert to UTC then compare + return getDateTime() == v.getDateTime(); + } + case Value::Type::VERTEX: { + if (value_.vVal == v.value_.vVal) { + return true; + } + return getVertex() == v.getVertex(); + } + case Value::Type::EDGE: { + if (value_.eVal == v.value_.eVal) { + return true; + } + return getEdge() == v.getEdge(); + } + case Value::Type::PATH: { + return getPath() == v.getPath(); + } + case Value::Type::LIST: { + return getList() == v.getList(); + } + case Value::Type::MAP: { + return getMap() == v.getMap(); + } + case Value::Type::SET: { + return getSet() == v.getSet(); + } + case Value::Type::DATASET: { + return getDataSet() == v.getDataSet(); + } + case Value::Type::GEOGRAPHY: { + return getGeography() == v.getGeography(); + } + case Value::Type::DURATION: { + return getDuration() == v.getDuration(); + } + case Value::Type::NULLVALUE: + case Value::Type::__EMPTY__: { + return false; + } + } + DLOG(FATAL) << "Unknown type " << static_cast(v.type()); + return Value::kNullBadType; +} + +bool Value::isImplicitBool() const { + switch (type_) { + case Type::BOOL: + case Type::LIST: + return true; + default: + return false; + } +} + +bool Value::implicitBool() const { + DCHECK(isImplicitBool()); + switch (type_) { + case Type::BOOL: + return getBool(); + case Type::LIST: + return !getList().empty(); + default: + DLOG(FATAL) << "Impossible to reach here!"; + return false; + } +} + +void swap(Value& a, Value& b) { + Value temp(std::move(a)); + a = std::move(b); + b = std::move(temp); +} + +/*static*/ const std::string Value::toString(Type type) { + switch (type) { + case Value::Type::__EMPTY__: { + return "__EMPTY__"; + } + case Value::Type::NULLVALUE: { + return "NULL"; + } + case Value::Type::BOOL: { + return "BOOL"; + } + case Value::Type::INT: { + return "INT"; + } + case Value::Type::FLOAT: { + return "FLOAT"; + } + case Value::Type::STRING: { + return "STRING"; + } + case Value::Type::DATE: { + return "DATE"; + } + case Value::Type::TIME: { + return "TIME"; + } + case Value::Type::DATETIME: { + return "DATETIME"; + } + case Value::Type::VERTEX: { + return "VERTEX"; + } + case Value::Type::EDGE: { + return "EDGE"; + } + case Value::Type::PATH: { + return "PATH"; + } + case Value::Type::LIST: { + return "LIST"; + } + case Value::Type::MAP: { + return "MAP"; + } + case Value::Type::SET: { + return "SET"; + } + case Value::Type::DATASET: { + return "DATASET"; + } + case Value::Type::GEOGRAPHY: { + return "GEOGRAPHY"; + } + case Value::Type::DURATION: { + return "DURATION"; + } + default: { + return "__UNKNOWN__"; + } + } +} + +std::ostream& operator<<(std::ostream& os, const Value::Type& type) { + os << Value::toString(type); + return os; +} + +Value operator+(const Value& lhs, const Value& rhs) { + if (lhs.isNull() || (lhs.empty() && !rhs.isNull())) { + return lhs; + } + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (lhs.type()) { + case Value::Type::BOOL: { + switch (rhs.type()) { + case Value::Type::STRING: { + return folly::stringPrintf( + "%s%s", lhs.getBool() ? "true" : "false", rhs.getStr().c_str()); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::INT: { + switch (rhs.type()) { + case Value::Type::INT: { + int64_t lVal = lhs.getInt(); + int64_t rVal = rhs.getInt(); + int64_t sum; + if (__builtin_add_overflow(lVal, rVal, &sum)) { + return Value::kNullOverflow; + } + return sum; + } + case Value::Type::FLOAT: { + return lhs.getInt() + rhs.getFloat(); + } + case Value::Type::STRING: { + return folly::stringPrintf("%ld%s", lhs.getInt(), rhs.getStr().c_str()); + } + case Value::Type::DATE: { + return rhs.getDate() + lhs.getInt(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getFloat() + rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getFloat() + rhs.getFloat(); + } + case Value::Type::STRING: { + return folly::to(lhs.getFloat()) + rhs.getStr(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::STRING: { + switch (rhs.type()) { + case Value::Type::BOOL: { + return folly::stringPrintf( + "%s%s", lhs.getStr().c_str(), rhs.getBool() ? "true" : "false"); + } + case Value::Type::INT: { + return folly::stringPrintf("%s%ld", lhs.getStr().c_str(), rhs.getInt()); + } + case Value::Type::FLOAT: { + return lhs.getStr() + folly::to(rhs.getFloat()); + } + case Value::Type::STRING: { + return lhs.getStr() + rhs.getStr(); + } + case Value::Type::DATE: { + return lhs.getStr() + rhs.getDate().toString(); + } + case Value::Type::TIME: { + return lhs.getStr() + rhs.getTime().toString(); + } + case Value::Type::DATETIME: { + return lhs.getStr() + rhs.getDateTime().toString(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DATE: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getDate() + rhs.getInt(); + } + case Value::Type::STRING: { + return lhs.getDate().toString() + rhs.getStr(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + case Value::Type::DURATION: { + return lhs.getDate() + rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::TIME: { + switch (rhs.type()) { + case Value::Type::STRING: { + return lhs.getTime().toString() + rhs.getStr(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + case Value::Type::DURATION: { + return lhs.getTime() + rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DATETIME: { + switch (rhs.type()) { + case Value::Type::STRING: { + return lhs.getDateTime().toString() + rhs.getStr(); + } + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + case Value::Type::DURATION: { + return lhs.getDateTime() + rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::LIST: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = lhs.getList(); + auto& list = rhs.getList(); + ret.values.insert(ret.values.end(), list.values.begin(), list.values.end()); + return ret; + } + case Value::Type::BOOL: + case Value::Type::INT: + case Value::Type::FLOAT: + case Value::Type::STRING: + case Value::Type::DATE: + case Value::Type::TIME: + case Value::Type::DATETIME: + case Value::Type::DURATION: + case Value::Type::VERTEX: + case Value::Type::EDGE: + case Value::Type::PATH: + case Value::Type::MAP: + case Value::Type::SET: { + auto ret = lhs.getList(); + ret.emplace_back(rhs); + return ret; + } + case Value::Type::DATASET: { + return Value::kNullBadType; + } + case Value::Type::GEOGRAPHY: { + return Value::kNullBadType; + } + case Value::Type::__EMPTY__: { + return Value::kEmpty; + } + case Value::Type::NULLVALUE: { + return Value::kNullValue; + } + } + DLOG(FATAL) << "Unknown type: " << rhs.type(); + return Value::kNullBadType; + } + case Value::Type::VERTEX: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::EDGE: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::PATH: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::MAP: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::SET: { + switch (rhs.type()) { + case Value::Type::LIST: { + auto ret = rhs.getList(); + ret.values.insert(ret.values.begin(), lhs); + return ret; + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DURATION: { + switch (rhs.type()) { + case Value::Type::DURATION: { + return lhs.getDuration() + rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator-(const Value& lhs, const Value& rhs) { + if (lhs.isNull() || (lhs.empty() && !rhs.isNull())) { + return lhs; + } + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (lhs.type()) { + case Value::Type::INT: { + switch (rhs.type()) { + case Value::Type::INT: { + int64_t lVal = lhs.getInt(); + int64_t rVal = rhs.getInt(); + int64_t res; + if (__builtin_sub_overflow(lVal, rVal, &res)) { + return Value::kNullOverflow; + } + return res; + } + case Value::Type::FLOAT: { + return lhs.getInt() - rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getFloat() - rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getFloat() - rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DATE: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getDate() - rhs.getInt(); + } + case Value::Type::DATE: { + return lhs.getDate().toInt() - rhs.getDate().toInt(); + } + case Value::Type::DURATION: { + return lhs.getDate() - rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::TIME: { + switch (rhs.type()) { + case Value::Type::DURATION: { + return lhs.getTime() - rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DATETIME: { + switch (rhs.type()) { + case Value::Type::DURATION: { + return lhs.getDateTime() - rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::DURATION: { + switch (rhs.type()) { + case Value::Type::DURATION: { + return lhs.getDuration() - rhs.getDuration(); + } + default: { + return Value::kNullBadType; + } + } + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator*(const Value& lhs, const Value& rhs) { + if (lhs.isNull() || (lhs.empty() && !rhs.isNull())) { + return lhs; + } + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (lhs.type()) { + case Value::Type::INT: { + switch (rhs.type()) { + case Value::Type::INT: { + int64_t lVal = lhs.getInt(); + int64_t rVal = rhs.getInt(); + // -1 * min causes overflow + if ((lVal == -1 && rVal == INT64_MIN) || (rVal == -1 && lVal == INT64_MIN)) { + return Value::kNullOverflow; + } + int64_t res; + if (__builtin_mul_overflow(lVal, rVal, &res)) { + return Value::kNullOverflow; + } + return res; + } + case Value::Type::FLOAT: { + return lhs.getInt() * rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getFloat() * rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getFloat() * rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator/(const Value& lhs, const Value& rhs) { + if (lhs.isNull() || (lhs.empty() && !rhs.isNull())) { + return lhs; + } + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (lhs.type()) { + case Value::Type::INT: { + switch (rhs.type()) { + case Value::Type::INT: { + int64_t denom = rhs.getInt(); + if (denom == 0) { + return Value::kNullDivByZero; + } + int64_t lVal = lhs.getInt(); + // INT_MIN/-1 causes overflow + if (lVal == INT64_MIN && denom == -1) { + return Value::kNullOverflow; + } + return lVal / denom; + } + case Value::Type::FLOAT: { + return lhs.getInt() / rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (rhs.type()) { + case Value::Type::INT: { + return lhs.getFloat() / rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getFloat() / rhs.getFloat(); + } + default: { + return Value::kNullBadType; + } + } + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator%(const Value& lhs, const Value& rhs) { + if (lhs.isNull() || (lhs.empty() && !rhs.isNull())) { + return lhs; + } + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (lhs.type()) { + case Value::Type::INT: { + switch (rhs.type()) { + case Value::Type::INT: { + int64_t denom = rhs.getInt(); + if (denom != 0) { + return lhs.getInt() % denom; + } else { + return Value::kNullDivByZero; + } + } + case Value::Type::FLOAT: { + return std::fmod(lhs.getInt(), rhs.getFloat()); + } + default: { + return Value::kNullBadType; + } + } + } + case Value::Type::FLOAT: { + switch (rhs.type()) { + case Value::Type::INT: { + return std::fmod(lhs.getFloat(), rhs.getInt()); + } + case Value::Type::FLOAT: { + return std::fmod(lhs.getFloat(), rhs.getFloat()); + } + default: { + return Value::kNullBadType; + } + } + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator-(const Value& rhs) { + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + switch (rhs.type()) { + case Value::Type::INT: { + int64_t rVal = rhs.getInt(); + if (rVal == INT64_MIN) { + return Value::kNullOverflow; + } + auto val = -rVal; + return val; + } + case Value::Type::FLOAT: { + auto val = -rhs.getFloat(); + return val; + } + case Value::Type::DURATION: { + auto val = -rhs.getDuration(); + return val; + } + default: { + return Value::kNullBadType; + } + } +} + +Value operator!(const Value& rhs) { + if (rhs.isNull() || rhs.empty()) { + return rhs; + } + + if (!rhs.isImplicitBool()) { + return Value::kNullBadType; + } + + auto val = rhs.implicitBool(); + return !val; +} + +bool operator<(const Value& lhs, const Value& rhs) { + auto lType = lhs.type(); + auto rType = rhs.type(); + auto hasNullOrEmpty = (lType | rType) & Value::kEmptyNullType; + auto notSameType = lType != rType; + auto notBothNumeric = ((lType | rType) & Value::kNumericType) != Value::kNumericType; + if (hasNullOrEmpty || (notSameType && notBothNumeric)) { + return lType < rType; + } + + switch (lType) { + case Value::Type::BOOL: { + return lhs.getBool() < rhs.getBool(); + } + case Value::Type::INT: { + switch (rType) { + case Value::Type::INT: { + return lhs.getInt() < rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getInt() < rhs.getFloat(); + } + default: { + return false; + } + } + } + case Value::Type::FLOAT: { + switch (rType) { + case Value::Type::INT: { + return lhs.getFloat() < rhs.getInt(); + } + case Value::Type::FLOAT: { + return lhs.getFloat() < rhs.getFloat(); + } + default: { + return false; + } + } + } + case Value::Type::STRING: { + return lhs.getStr() < rhs.getStr(); + } + case Value::Type::VERTEX: { + return lhs.getVertex() < rhs.getVertex(); + } + case Value::Type::EDGE: { + return lhs.getEdge() < rhs.getEdge(); + } + case Value::Type::PATH: { + return lhs.getPath() < rhs.getPath(); + } + case Value::Type::TIME: { + return lhs.getTime() < rhs.getTime(); + } + case Value::Type::DATE: { + return lhs.getDate() < rhs.getDate(); + } + case Value::Type::DATETIME: { + return lhs.getDateTime() < rhs.getDateTime(); + } + case Value::Type::LIST: { + return lhs.getList() < rhs.getList(); + } + case Value::Type::MAP: + case Value::Type::SET: + case Value::Type::DATASET: { + // TODO: + return false; + } + case Value::Type::GEOGRAPHY: { + return lhs.getGeography() < rhs.getGeography(); + } + case Value::Type::DURATION: { + DLOG(FATAL) << "Duration is not comparable."; + return false; + } + case Value::Type::NULLVALUE: + case Value::Type::__EMPTY__: { + return false; + } + } + DLOG(FATAL) << "Unknown type " << static_cast(lType); + return false; +} + +bool Value::equals(const Value& rhs) const { + auto lType = type(); + auto rType = rhs.type(); + auto hasNullOrEmpty = (lType | rType) & Value::kEmptyNullType; + auto notSameType = lType != rType; + auto notBothNumeric = ((lType | rType) & Value::kNumericType) != Value::kNumericType; + if (hasNullOrEmpty || (notSameType && notBothNumeric)) { + return type() == rhs.type(); + } + + switch (lType) { + case Value::Type::BOOL: { + return getBool() == rhs.getBool(); + } + case Value::Type::INT: { + switch (rType) { + case Value::Type::INT: { + return getInt() == rhs.getInt(); + } + case Value::Type::FLOAT: { + return std::abs(getInt() - rhs.getFloat()) < kEpsilon; + } + default: { + return false; + } + } + } + case Value::Type::FLOAT: { + switch (rType) { + case Value::Type::INT: { + return std::abs(getFloat() - rhs.getInt()) < kEpsilon; + } + case Value::Type::FLOAT: { + return std::abs(getFloat() - rhs.getFloat()) < kEpsilon; + } + default: { + return false; + } + } + } + case Value::Type::STRING: { + return getStr() == rhs.getStr(); + } + case Value::Type::DATE: { + return getDate() == rhs.getDate(); + } + case Value::Type::TIME: { + // TODO(shylock) convert to UTC then compare + return getTime() == rhs.getTime(); + } + case Value::Type::DATETIME: { + // TODO(shylock) convert to UTC then compare + return getDateTime() == rhs.getDateTime(); + } + case Value::Type::VERTEX: { + if (value_.vVal == rhs.value_.vVal) { + return true; + } + return getVertex() == rhs.getVertex(); + } + case Value::Type::EDGE: { + if (value_.eVal == rhs.value_.eVal) { + return true; + } + return getEdge() == rhs.getEdge(); + } + case Value::Type::PATH: { + return getPath() == rhs.getPath(); + } + case Value::Type::LIST: { + return getList() == rhs.getList(); + } + case Value::Type::MAP: { + return getMap() == rhs.getMap(); + } + case Value::Type::SET: { + return getSet() == rhs.getSet(); + } + case Value::Type::DATASET: { + return getDataSet() == rhs.getDataSet(); + } + case Value::Type::GEOGRAPHY: { + return getGeography() == rhs.getGeography(); + } + case Value::Type::DURATION: { + return getDuration() == rhs.getDuration(); + } + case Value::Type::NULLVALUE: + case Value::Type::__EMPTY__: { + return false; + } + } + DLOG(FATAL) << "Unknown type " << static_cast(type()); + return false; +} + +std::size_t Value::hash() const { + switch (type()) { + case Type::__EMPTY__: { + return 0; + } + case Type::NULLVALUE: { + return ~0UL; + } + case Type::BOOL: { + return std::hash()(getBool()); + } + case Type::INT: { + return std::hash()(getInt()); + } + case Type::FLOAT: { + return std::hash()(getFloat()); + } + case Type::STRING: { + return std::hash()(getStr()); + } + case Type::DATE: { + return std::hash()(getDate()); + } + case Type::TIME: { + return std::hash