p70621953
/
openGauss-server
forked from openGauss-Ecosystem/openGauss-server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

主备共享存储

This commit is contained in:
peibaoyi 2022-11-07 14:18:06 +08:00 committed by peibaoyi
parent 90570ff85b
commit bb5b05ec46
349 changed files with 25869 additions and 1355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
build/script/aarch64_opengauss_list
View File

@ -1,4 +1,10 @@
[server]
./bin/dsscmd
./bin/dssserver
./bin/perctrl
./bin/dms_contrl.sh
./bin/dss_clear.sh
./bin/dss_contrl.sh
./bin/gsql
./bin/gaussdb
./bin/gstrace
@ -824,6 +830,8 @@
./lib/libzstd.so*
./lib/libxgboost.so
./lib/libpagecompression.so*
./lib/libdssapi.so
./lib/libdms.so
./include/postgresql/server/postgres_ext.h
./include/postgresql/server/pg_config_os.h

4
build/script/utils/common.sh
View File

@ -59,7 +59,9 @@ select_package_command
#######################################################################
##get os dist version
#######################################################################
if [[ -f "/etc/euleros-release" ]]; then
if [[ -f "/etc/openEuler-release" ]]; then
os_name="openEuler"
elif [[ -f "/etc/euleros-release" ]]; then
os_name="EulerOS"
elif [[ -f "/etc/centos-release" ]]; then
os_name="CentOS"

9
build/script/x86_64_opengauss_list
View File

@ -1,4 +1,10 @@
[server]
./bin/dsscmd
./bin/dssserver
./bin/perctrl
./bin/dms_contrl.sh
./bin/dss_clear.sh
./bin/dss_contrl.sh
./bin/gsql
./bin/gaussdb
./bin/gstrace
@ -821,6 +827,9 @@
./lib/libzstd.so*
./lib/libxgboost.so
./lib/libpagecompression.so*
./lib/libdssapi.so
./lib/libdms.so
./include/postgresql/server/postgres_ext.h
./include/postgresql/server/pg_config_os.h
./include/postgresql/server/pgtime.h

19
cmake/src/build_options.cmake
View File

@ -17,6 +17,12 @@ if(NOT ${PGPORT})
set(PGPORT 5432)
endif()
#this is the default WAL segment size (MB), the old is --with-wal-segsize=16, now -DWAL_SEGSIZE=16
option(WAL_SEGSIZE 16)
if(NOT ${WAL_SEGSIZE})
set(WAL_SEGSIZE 16)
endif()
GET_VERSIONSTR_FROMGIT(GET_PG_VERSION_STR)
#CMake does not allow g++ to compile C files. They are two different languages. You should understand what you are doing.
@ -58,6 +64,7 @@ option(ENABLE_MYSQL_FDW "enable export or import data with mysql,the old is --en
option(ENABLE_ORACLE_FDW "enable export or import data with oracle,the old is --enable-oracle-fdw" OFF)
option(BUILD_BY_CMAKE "the BUILD_BY_CMAKE is new,used in distribute pg_regress.cpp" ON)
option(DEBUG_UHEAP "collect USTORE statistics" OFF)
option(MAX_ALLOC_SEGNUM "max alloc xlog seg num in extreme_rto" 4)
#No matter what to set, the old mppdb aways use ENABLE_THREAD_SAFETY=yes by default defined.
option(ENABLE_THREAD_SAFETY "enable thread safety, the old is --enable-thread-safety" ON)
@ -140,6 +147,17 @@ set(OPTIMIZE_OPTIONS -pipe -pthread -fno-aggressive-loop-optimizations -fno-expe
set(CHECK_OPTIONS -Wmissing-format-attribute -Wno-attributes -Wno-unused-but-set-variable -Wno-write-strings -Wpointer-arith)
set(MACRO_OPTIONS -D_GLIBCXX_USE_CXX11_ABI=0 -DENABLE_GSTRACE -D_GNU_SOURCE -DPGXC -D_POSIX_PTHREAD_SEMANTICS -D_REENTRANT -DSTREAMPLAN -D_THREAD_SAFE ${DB_COMMON_DEFINE})
# Set MAX_ALLOC_SEGNUM size in extreme_rto
if(${WAL_SEGSIZE} LESS 512)
set(MAX_ALLOC_SEGNUM 4)
elseif(${WAL_SEGSIZE} GREATER_EQUAL 512 AND ${WAL_SEGSIZE} LESS 1024)
set(MAX_ALLOC_SEGNUM 2)
elseif(${WAL_SEGSIZE} GREATER_EQUAL 1024)
set(MAX_ALLOC_SEGNUM 1)
else()
message(FATAL_ERROR "error: Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.")
endif()
# libraries need secure options during compling
set(LIB_SECURE_OPTIONS -fPIC -fno-common -fstack-protector)
# libraries need link options during linking
@ -275,6 +293,7 @@ if(${ENABLE_LLVM_COMPILE} STREQUAL "ON")
# LLVM version
execute_process(COMMAND ${LLVM_CONFIG} --version OUTPUT_VARIABLE LLVM_VERSION_STR OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE "." ";" LLVM_VERSION_LIST ${LLVM_VERSION_STR})
message(STATUS "status ENV{LLVM_VERSION_STR}" $ENV{LLVM_VERSION_STR})
list(GET LLVM_VERSION_LIST 0 LLVM_MAJOR_VERSION)
list(GET LLVM_VERSION_LIST 1 LLVM_MINOR_VERSION)
endif()

5
cmake/src/config-in/pg_config.h.in
View File

@ -870,7 +870,10 @@
/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
* * and larger than XLOG_BLCKSZ (preferably, a great deal larger than
* * XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
#define XLOG_SEG_SIZE (16 * 1024 * 1024)
#define XLOG_SEG_SIZE (@WAL_SEGSIZE@ * 1024 * 1024)
/* Number of max alloc xlog segment in extreme_rto, default 4 */
#define MAX_ALLOC_SEGNUM @MAX_ALLOC_SEGNUM@
/* Number of bits in a file offset, on hosts where this is settable. */
#cmakedefine _FILE_OFFSET_BITS

3
cmake/src/config-in/pg_config.h.in.nocheck.aarch64
View File

@ -878,6 +878,9 @@
XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
#define XLOG_SEG_SIZE (16 * 1024 * 1024)
/* Number of max alloc xlog segment in extreme_rto, default 4 */
#define MAX_ALLOC_SEGNUM 4
/* Number of bits in a file offset, on hosts where this is settable. */
/* #undef _FILE_OFFSET_BITS */

3
cmake/src/config-in/pg_config.h.in.nocheck.x86_64
View File

@ -878,6 +878,9 @@
XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
#define XLOG_SEG_SIZE (16 * 1024 * 1024)
/* Number of max alloc xlog segment in extreme_rto, default 4 */
#define MAX_ALLOC_SEGNUM 4
/* Number of bits in a file offset, on hosts where this is settable. */
/* #undef _FILE_OFFSET_BITS */

3
cmake/src/config-in/pg_config_nocheck.h.in
View File

@ -870,6 +870,9 @@
* XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
#define XLOG_SEG_SIZE (16 * 1024 * 1024)
/* Number of max alloc xlog segment in extreme_rto, default 4 */
#define MAX_ALLOC_SEGNUM 4
/* Number of bits in a file offset, on hosts where this is settable. */
/* #undef _FILE_OFFSET_BITS */

13
cmake/src/set_thirdparty_path.cmake
View File

@ -76,6 +76,8 @@ set(HOTPATCH_HOME ${PLATFORM_PATH}/hotpatch)
set(SECURE_HOME ${PLATFORM_PATH}/Huawei_Secure_C/${LIB_UNIFIED_SUPPORT})
set(SECUREDYNAMICLIB_HOME ${PLATFORM_PATH}/Huawei_Secure_C/Dynamic_Lib)
set(DCF_HOME ${COMPONENT_PATH}/dcf)
set(DMS_HOME ${COMPONENT_PATH}/dms)
set(DSS_HOME ${COMPONENT_PATH}/dss)
set(MOCKCPP_HOME ${BUILDTOOLS_PATH}/mockcpp/${LIB_UNIFIED_SUPPORT})
set(GTEST_HOME ${BUILDTOOLS_PATH}/gtest/${LIB_UNIFIED_SUPPORT})
@ -263,6 +265,17 @@ set(ZSTD_LIB_PATH ${ZSTD_HOME}/lib)
set(DCF_INCLUDE_PATH ${DCF_HOME}/include)
set(DCF_LIB_PATH ${DCF_HOME}/lib)
#############################################################################
# dms component
#############################################################################
set(DMS_LIB_PATH ${DMS_HOME}/lib)
#############################################################################
# dss component
#############################################################################
set(DSS_LIB_PATH ${DSS_HOME}/lib)
set(DSS_BIN_PATH ${DSS_HOME}/bin)
#############################################################################
# license manager compnent
#############################################################################

20
configure vendored
View File

@ -3710,8 +3710,12 @@ case ${wal_segsize} in
16) ;;
32) ;;
64) ;;
*) { { $as_echo "$as_me:$LINENO: error: Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64." >&5
$as_echo "$as_me: error: Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64." >&2;}
128) ;;
256) ;;
512) ;;
1024) ;;
*) { { $as_echo "$as_me:$LINENO: error: Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." >&5
$as_echo "$as_me: error: Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." >&2;}
{ (exit 1); exit 1; }; }
esac
{ $as_echo "$as_me:$LINENO: result: ${wal_segsize}MB" >&5
@ -3722,6 +3726,18 @@ cat >>confdefs.h <<_ACEOF
#define XLOG_SEG_SIZE (${wal_segsize} * 1024 * 1024)
_ACEOF
# Set number of MAX_ALLOC_SEGNUM in extreme_rto
if test "${wal_segsize}" -lt 512; then
MAX_ALLOC_SEGNUM=4
elif test "${wal_segsize}" -ge 512 -a "${wal_segsize}" -lt 1024; then
MAX_ALLOC_SEGNUM=2
else
MAX_ALLOC_SEGNUM=1
fi
cat >>confdefs.h <<_ACEOF
#define MAX_ALLOC_SEGNUM ${MAX_ALLOC_SEGNUM}
_ACEOF
#
# C compiler

16
configure.in
View File

@ -381,7 +381,11 @@ case ${wal_segsize} in
16) ;;
32) ;;
64) ;;
*) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64.])
128) ;;
256) ;;
512) ;;
1024) ;;
*) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.])
esac
AC_MSG_RESULT([${wal_segsize}MB])
@ -393,6 +397,16 @@ AC_DEFINE_UNQUOTED([XLOG_SEG_SIZE], [(${wal_segsize} * 1024 * 1024)], [
Changing XLOG_SEG_SIZE requires an initdb.
])
# Set number of MAX_ALLOC_SEGNUM in extreme_rto
if test "${wal_segsize}" -lt 512; then
MAX_ALLOC_SEGNUM=4
elif test "${wal_segsize}" -ge 512 -a "${wal_segsize}" -lt 1024; then
MAX_ALLOC_SEGNUM=2
else
MAX_ALLOC_SEGNUM=1
fi
AC_DEFINE_UNQUOTED(MAX_ALLOC_SEGNUM, $MAX_ALLOC_SEGNUM, [Define the max alloc segnum in extreme_rto.])
#
# C compiler
#

3
contrib/pagehack/CMakeLists.txt
View File

@ -1,5 +1,8 @@
#This is the main CMAKE for build all pagehack.
# pagehack
execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/dss_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_pagehack_SRC)
set(TGT_pagehack_INC
${TGT_pq_INC} ${ZSTD_INCLUDE_PATH} ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SRC_DIR}/lib/gstrace ${PROJECT_SRC_DIR}/lib/page_compression

3
contrib/pagehack/Makefile
View File

@ -1,6 +1,7 @@
# contrib/pagehack/Makefile
MODULE_big = pagehack
OBJS = pagehack.o
OBJS = pagehack.o \
$(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o
# executable program, even there is no database server/client
PROGRAM = pagehack

74
contrib/pagehack/pagehack.cpp
View File

@ -80,6 +80,8 @@
#include "storage/smgr/relfilenode.h"
#include "storage/sinval.h"
#include "storage/smgr/segment.h"
#include "storage/dss/dss_adaptor.h"
#include "storage/file/fio_device.h"
#include "replication/bcm.h"
#include "utils/datetime.h"
#include "utils/memutils.h"
@ -88,6 +90,7 @@
#include "cstore.h"
#include "common/build_query/build_query.h"
#include <libgen.h>
#include "tool_common.h"
#ifdef ENABLE_MULTIPLE_NODES
#include "tsdb/utils/constant_def.h"
#endif
@ -100,6 +103,7 @@
#define CLASS_TYPE_NUM 512
#define TEN 10
typedef unsigned char* binary;
static const char* indents[] = { // 10 tab is enough to used.
"",
@ -192,6 +196,7 @@ static ParseHeapTupleData PgIndexRelTupleParser[] = {ParseToastIndexTupleData};
static int PgHeapRelTupleParserCursor = -1;
static int PgIndexRelTupleParserCursor = -1;
/* For Assert(...) macros. */
THR_LOCAL bool assert_enabled = true;
@ -200,6 +205,7 @@ bool only_vm = false;
bool only_bcm = false;
bool write_back = false;
bool dirty_page = false;
bool enable_dss = false;
int start_item = 1;
int num_item = 0;
int SegNo = 0;
@ -1014,6 +1020,9 @@ static void usage(const char* progname)
" -d only for test, use 0xFF to fill the last half page[4k]\n"
" -z only for undo space/group meta, dump the specified space/group\n"
" -S heap file segment number\n"
"\nDss options:\n"
" -D enable shared storage mode\n"
" -c SOCKETPATH dss connect socket file path\n"
"\nCommon options:\n"
" --help, -h show this help, then exit\n"
" --version, -V output version information, then exit\n");
@ -3195,7 +3204,8 @@ static int parse_uncompressed_page_file(const char *filename, SegmentType type,
BlockNumber number = number_read;
size_t result;
if (NULL == (fd = fopen(filename, "rb+"))) {
fd = fopen(filename, "rb+");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -3422,7 +3432,8 @@ static bool parse_internal_init_file(char* filename)
return false;
}
if (NULL == (fp = fopen(filename, "rb"))) {
fp = fopen(filename, "rb");
if (fp == NULL) {
result = false;
fprintf(stderr, "IO error when opening %s: %s\n", filename, strerror(errno));
goto read_failed;
@ -3653,7 +3664,8 @@ static int parse_filenodemap_file(char* filename)
}
fill_filenode_map(pg_class_map);
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -3738,7 +3750,8 @@ static int parse_cu_file(char* filename, uint64 offset)
errno_t rc = snprintf_s(fullpath, sizeof(fullpath), sizeof(fullpath) - 1, "%s.%d", filename, seg_num);
securec_check_ss_c(rc, "\0", "\0");
if (NULL == (fd = fopen(fullpath, "rb"))) {
fd = fopen(fullpath, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", fullpath, strerror(errno));
return false;
}
@ -3797,7 +3810,8 @@ static int parse_slot_file(char* filename)
size_t readBytes = 0;
pg_crc32 checksum = 0;
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -3894,7 +3908,8 @@ static int parse_gaussdb_state_file(char* filename)
char* BuildModeStr[] = {"node", "auto", "full", "incremental"};
XLogRecPtr lsn;
uint32 term;
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -3956,7 +3971,8 @@ static int parse_pg_control_file(char* filename)
char sysident_str[32];
const char* strftime_fmt = "%c";
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -4091,7 +4107,8 @@ static int parse_clog_file(char* filename)
xid = (uint64)segnum * segnum_xid;
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, strerror(errno));
return false;
}
@ -4146,7 +4163,8 @@ static int parse_csnlog_file(char* filename)
xid = (uint64)segnum * segnum_xid;
if (NULL == (fd = fopen(filename, "rb"))) {
fd = fopen(filename, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", filename, gs_strerror(errno));
return false;
}
@ -4403,7 +4421,7 @@ static bool parse_dw_file(const char* file_name, uint32 start_page, uint32 page_
(void)dirname(meta_full_path);
/* extract the meta name from DW_META_FILE */
rc = strcpy_s(meta_name_tmp, PATH_MAX, DW_META_FILE);
rc = strcpy_s(meta_name_tmp, PATH_MAX, T_DW_META_FILE);
securec_check(rc, "", "");
meta_name = basename(meta_name_tmp);
@ -4414,7 +4432,6 @@ static bool parse_dw_file(const char* file_name, uint32 start_page, uint32 page_
securec_check(rc, "", "");
fd = fopen(meta_full_path, "rb+");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", meta_full_path, strerror(errno));
return false;
@ -4580,8 +4597,7 @@ static bool parse_segment_head(char *filename, uint32 start_page)
}
bool result;
int fd = open(filename, O_RDONLY);
int fd = open(filename, O_RDONLY, S_IRUSR | S_IWUSR);
if (fd < 0) {
fprintf(stderr, "Failed to open %s: %s\n", filename, strerror(errno));
free(buf);
@ -4652,7 +4668,8 @@ static bool parse_dw_single_flush_file(const char* file_name)
char *dw_block = (char *)TYPEALIGN(BLCKSZ, unaligned_buf2);
PageHeader pghr = NULL;
if (NULL == (fd = fopen(file_name, "rb"))) {
fd = fopen(file_name, "rb");
if (fd == NULL) {
fprintf(stderr, "%s: %s\n", file_name, gs_strerror(errno));
free(item);
free(unaligned_buf);
@ -5569,6 +5586,7 @@ int main(int argc, char** argv)
char* filename = NULL;
char* env = NULL;
const char* progname = NULL;
const char* socketpath = NULL;
uint32 start_point = 0;
uint32 num_block = 0;
uint64 cu_offset = 0;
@ -5593,7 +5611,7 @@ int main(int argc, char** argv)
setvbuf(stderr, NULL, _IONBF, 0);
#endif
while ((c = getopt(argc, argv, "bf:o:t:vs:z:n:r:i:I:N:uwdS:")) != -1) {
while ((c = getopt(argc, argv, "bc:Df:o:t:vs:z:n:r:i:I:N:uwdS:")) != -1) {
switch (c) {
case 'f':
filename = optarg;
@ -5699,6 +5717,15 @@ int main(int argc, char** argv)
SegNo = (unsigned int)strtolSafe(optarg, 0);
break;
case 'D':
enable_dss = true;
break;
case 'c':
socketpath = optarg;
enable_dss = true;
break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@ -5721,6 +5748,16 @@ int main(int argc, char** argv)
exit(1);
}
if (enable_dss && socketpath == NULL) {
fprintf(stderr, "Socketpath cannot be NULL when enable dss.\n");
exit(1);
}
if (enable_dss && (filename[0] != '+' || strstr(filename, "/") == NULL)) {
fprintf(stderr, "Filepath should be absolutely when enable dss.\n");
exit(1);
}
if (((start_point != 0) || (num_block != 0)) &&
/* only heap/index/undo/dw */
(hackingtype > HACKING_UNDO && hackingtype != HACKING_DW && hackingtype != HACKING_SEGMENT)) {
@ -5749,6 +5786,13 @@ int main(int argc, char** argv)
pgdata = env;
}
// dss device init
if (dss_device_init(socketpath, enable_dss) != DSS_SUCCESS) {
exit(1);
}
initDataPathStruct(false);
// if heap relation name is given (-r), force hackingtype to be HACKING_HEAP
if (PgHeapRelTupleParserCursor >= 0) {
hackingtype = HACKING_HEAP;

1
contrib/pg_xlogdump/CMakeLists.txt
View File

@ -27,6 +27,7 @@ execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/access/rmgrdesc/undologdesc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/undologdesc.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/access/rmgrdesc/replorigindesc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/replorigindesc.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/smgr/cfs/cfs_mddesc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cfs_mddesc.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/dss_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_xlogdump_SRC)

3
contrib/pg_xlogdump/Makefile
View File

@ -7,7 +7,8 @@ PROGRAM = pg_xlogdump
OBJS = pg_xlogdump.o compat.o xlogreader.o xlogreader_common.o rmgrdesc.o \
$(top_builddir)/src/gausskernel/storage/smgr/cfs/cfs_mddesc.o \
$(top_builddir)/src/lib/pgcommon/libpgcommon.a \
$(RMGRDESCOBJS) $(WIN32RES)
$(RMGRDESCOBJS) $(WIN32RES) \
$(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o
RMGRDESCSOURCES = $(notdir $(wildcard $(top_srcdir)/src/gausskernel/storage/access/rmgrdesc/*desc.cpp))
RMGRDESCOBJS = $(patsubst %.cpp,%.o,$(RMGRDESCSOURCES))

81
contrib/pg_xlogdump/pg_xlogdump.cpp
View File

@ -33,6 +33,8 @@
#include "rmgrdesc.h"
#include "storage/smgr/segment.h"
#include "storage/page_compression.h"
#include "storage/dss/dss_adaptor.h"
#include "storage/file/fio_device.h"
static const char* progname;
@ -44,6 +46,8 @@ typedef struct XLogDumpPrivate {
bool endptr_reached;
char* shareStorageXlogFilePath;
long shareStorageXlogSize;
bool enable_dss;
char* socketpath;
} XLogDumpPrivate;
typedef struct XLogDumpConfig {
@ -77,7 +81,7 @@ typedef struct XLogDumpStats {
} XLogDumpStats;
static void XLogDumpTablePage(XLogReaderState* record, int block_id, RelFileNode rnode, BlockNumber blk);
static void XLogDumpXLogRead(const char* directory, TimeLineID timeline_id, XLogRecPtr startptr, char* buf, Size count);
static void XLogDumpXLogRead(char* directory, TimeLineID timeline_id, XLogRecPtr startptr, char* buf, Size count);
static int XLogDumpReadPage(XLogReaderState* state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr,
char* readBuff, TimeLineID* curFileTLI, char* xlog_path = NULL);
static void XLogDumpCountRecord(XLogDumpConfig* config, XLogDumpStats* stats, XLogReaderState* record);
@ -88,6 +92,7 @@ static void XLogDumpDisplayStats(XLogDumpConfig* config, XLogDumpStats* stats);
static void usage(void);
static int fuzzy_open_file(const char* directory, const char* fname);
static int open_dss_file(char* directory, const char* fname);
static void split_path(char* path, char** dir, char** fname);
static bool verify_directory(const char* directory);
static void print_rmgr_list(void);
@ -224,6 +229,21 @@ static int fuzzy_open_file(const char* directory, const char* fname)
return -1;
}
// for dss storage
static int open_dss_file(char* directory, const char* fname)
{
int fd = -1;
char fpath[MAXPGPATH];
/* directory / fname */
canonicalize_path(directory);
snprintf(fpath, MAXPGPATH, "%s/%s", directory, fname);
if (dss_open_file(fpath, O_RDONLY | PG_BINARY, 0, &fd) == GS_SUCCESS) {
return fd;
}
return -1;
}
/*
* write the full page to disk.
*/
@ -327,7 +347,7 @@ static void XLogDumpReadSharedStorage(char* directory, XLogRecPtr startptr, long
* given timeline, containing the specified record pointer; store the data in
* the passed buffer.
*/
static void XLogDumpXLogRead(const char* directory, TimeLineID timeline_id, XLogRecPtr startptr, char* buf, Size count)
static void XLogDumpXLogRead(char* directory, TimeLineID timeline_id, XLogRecPtr startptr, char* buf, Size count)
{
char* p = NULL;
XLogRecPtr recptr;
@ -359,7 +379,11 @@ static void XLogDumpXLogRead(const char* directory, TimeLineID timeline_id, XLog
XLogFileName(fname, MAXFNAMELEN, timeline_id, sendSegNo);
sendFile = fuzzy_open_file(directory, fname);
if (is_dss_file(directory)) {
sendFile = open_dss_file(directory, fname);
} else {
sendFile = fuzzy_open_file(directory, fname);
}
if (sendFile < 0)
fatal_error("could not find file \"%s\": %s", fname, strerror(errno));
@ -807,6 +831,8 @@ static void usage(void)
printf(" -z, --stats show statistics instead of records\n");
printf(" -v, --verbose show detailed information\n");
printf(" -?, --help show this help, then exit\n");
printf(" --enable-dss enable shared storage mode\n");
printf(" --socketpath=SOCKETPATH dss connect socket file path\n");
}
int main(int argc, char** argv)
@ -823,12 +849,14 @@ int main(int argc, char** argv)
static struct option long_options[] = {{"bkp-details", no_argument, NULL, 'b'},
{"end", required_argument, NULL, 'e'},
{"enable-dss", no_argument, NULL, 1},
{"follow", no_argument, NULL, 'f'},
{"help", no_argument, NULL, '?'},
{"limit", required_argument, NULL, 'n'},
{"path", required_argument, NULL, 'p'},
{"rmgr", required_argument, NULL, 'r'},
{"start", required_argument, NULL, 's'},
{"socketpath", required_argument, NULL, 2},
{"timeline", required_argument, NULL, 't'},
{"write-fpw", no_argument, NULL, 'w'},
{"xid", required_argument, NULL, 'x'},
@ -852,6 +880,8 @@ int main(int argc, char** argv)
dumpprivate.endptr = InvalidXLogRecPtr;
dumpprivate.endptr_reached = false;
dumpprivate.shareStorageXlogFilePath = NULL;
dumpprivate.enable_dss = false;
dumpprivate.socketpath = NULL;
const long defaultShareStorageXlogSize = 512 * 1024 * 1024 * 1024L;
dumpprivate.shareStorageXlogSize = defaultShareStorageXlogSize;
@ -959,6 +989,13 @@ int main(int argc, char** argv)
case 'z':
config.stats = true;
break;
case 1:
dumpprivate.enable_dss = true;
break;
case 2:
dumpprivate.enable_dss = true;
dumpprivate.socketpath = strdup(optarg);
break;
default:
goto bad_argument;
}
@ -969,6 +1006,30 @@ int main(int argc, char** argv)
goto bad_argument;
}
// dss device init
if (dss_device_init(dumpprivate.socketpath, dumpprivate.enable_dss) != DSS_SUCCESS) {
fatal_error("failed to init dss device");
}
if (dumpprivate.enable_dss) {
if (dumpprivate.socketpath == NULL) {
fprintf(stderr, "%s: socketpath cannot be NULL when enable dss\n", progname);
goto bad_argument;
}
char* directory = dumpprivate.inpath == NULL ? argv[optind] : dumpprivate.inpath;
if (directory != NULL) {
if (directory[0] != '+' || strstr(directory, "/") == NULL) {
fprintf(stderr, "%s: xlog file path should be absolutely when enable dss\n", progname);
goto bad_argument;
}
} else {
fprintf(stderr, "%s: xlog file path must be specified when enable dss\n", progname);
goto bad_argument;
}
}
if (dumpprivate.inpath != NULL) {
/* validate path points to directory */
if (!verify_directory(dumpprivate.inpath)) {
@ -998,7 +1059,12 @@ int main(int argc, char** argv)
fatal_error("cannot open directory \"%s\": %s", dumpprivate.inpath, strerror(errno));
}
fd = fuzzy_open_file(dumpprivate.inpath, fname);
if (is_dss_file(dumpprivate.inpath)) {
fd = open_dss_file(dumpprivate.inpath, fname);
} else {
fd = fuzzy_open_file(dumpprivate.inpath, fname);
}
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);
@ -1030,7 +1096,12 @@ int main(int argc, char** argv)
/* ignore directory, already have that */
split_path(argv[optind + 1], &directory, &fname);
fd = fuzzy_open_file(dumpprivate.inpath, fname);
if (is_dss_file(dumpprivate.inpath)) {
fd = open_dss_file(dumpprivate.inpath, fname);
} else {
fd = fuzzy_open_file(dumpprivate.inpath, fname);
}
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);

2
doc/src/sgml/installation.sgmlin
View File

@ -1186,7 +1186,7 @@ su - gaussdb
the size of each individual file in the WAL log. It may be useful
to adjust this size to control the granularity of WAL log shipping.
The default size is 16 megabytes.
The value must be a power of 2 between 1 and 64 (megabytes).
The value must be a power of 2 between 1 and 1024 (megabytes).
Note that changing this value requires an gs_initdb.
</para>
</listitem>

12
src/CMakeLists.txt
View File

@ -193,6 +193,18 @@ endif()
if("${ENABLE_MULTIPLE_NODES}" STREQUAL "OFF")
install(DIRECTORY ${DCF_LIB_PATH} DESTINATION .)
endif()
if(${ENABLE_MULTIPLE_NODES}_${ENABLE_PRIVATEGAUSS} STREQUAL OFF_OFF)
if(EXISTS ${DMS_LIB_PATH})
install(DIRECTORY ${DMS_LIB_PATH} DESTINATION .)
endif()
if(EXISTS ${DSS_LIB_PATH})
install(DIRECTORY ${DSS_LIB_PATH} DESTINATION .)
endif()
if(EXISTS ${DSS_BIN_PATH})
install(DIRECTORY ${DSS_BIN_PATH} DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE OWNER_READ GROUP_READ WORLD_READ OWNER_WRITE)
endif()
endif()
install(DIRECTORY ${ZSTD_LIB_PATH} DESTINATION . PATTERN "*.a" EXCLUDE)
if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON")
install(DIRECTORY ${LIBOBS_LIB_PATH} DESTINATION .)

16
src/bin/gs_guc/cluster_guc.conf
View File

@ -697,6 +697,22 @@ logical_decode_options_default|string|0,0|NULL|NULL|
logical_sender_timeout|int|0,2147483647|ms|NULL|
var_eq_const_selectivity|bool|0,0|NULL|NULL|
enable_save_confirmed_lsn|bool|0,0|NULL|NULL|
ss_enable_dss|bool|0,0|NULL|NULL|
ss_dss_vg_name|string|0,0|NULL|NULL|
ss_dss_conn_path|string|0,0|NULL|NULL|
ss_enable_dms|bool|0,0|NULL|NULL|
ss_enable_catalog_centralized|bool|0,0|NULL|NULL|
ss_enable_reform|bool|0,0|NULL|NULL|
ss_enable_ssl|bool|0,0|NULL|NULL|
ss_enable_log_level|bool|0,0|NULL|NULL|
ss_interconnect_channel_count|int|1,32|NULL|NULL|
ss_work_thread_count|int|16,128|NULL|NULL|
ss_recv_msg_pool_size|int|1024,1048576|kB|NULL|
ss_interconnect_type|string|0,0|NULL|NULL|
ss_instance_id|int|0,63|NULL|NULL|
ss_interconnect_url|string|0,0|NULL|NULL|
ss_rdma_work_config|string|0,0|NULL|NULL|
ss_ock_log_path|string|0,0|NULL|NULL|
[cmserver]
log_dir|string|0,0|NULL|NULL|
log_file_size|int|0,2047|MB|NULL|

24
src/bin/gs_guc/pg_guc.cpp
View File

@ -1297,6 +1297,11 @@ do_gucset(const char *action_type, const char *data_dir)
int func_status = -1;
int result_status = SUCCESS;
int ss_lines_index = 0;
int ss_optvalue_off = 0;
int ss_optvalue_len = 0;
char ss_enable_dss[MAX_VALUE_LEN] = {0x00};
FileLock filelock = {NULL, 0};
UpdateOrAddParameter updateoradd = UPDATE_PARAMETER;
@ -1324,6 +1329,13 @@ do_gucset(const char *action_type, const char *data_dir)
if (NULL == opt_lines)
return FAILURE;
ss_lines_index = find_gucoption(opt_lines, "ss_enable_dss", NULL, NULL, &ss_optvalue_off, &ss_optvalue_len);
if (INVALID_LINES_IDX != ss_lines_index) {
rc = strncpy_s(ss_enable_dss, MAX_VALUE_LEN,
(opt_lines[ss_lines_index] + ss_optvalue_off), (size_t)ss_optvalue_len);
securec_check_c(rc, "\0", "\0");
}
for (i = 0; i < config_param_number; i++)
{
@ -1335,6 +1347,18 @@ do_gucset(const char *action_type, const char *data_dir)
return FAILURE;
}
if (strncmp(ss_enable_dss, "on", strlen("on")) == 0 &&
((strncmp(config_param[i], "archive_mode", strlen("archive_mode")) == 0 &&
strncmp(config_value[i], "on", strlen("on")) == 0) ||
(strncmp(config_param[i], "archive_command", strlen("archive_command")) == 0 &&
config_value[i] != NULL))) {
release_file_lock(&filelock);
freefile(opt_lines);
GS_FREE(tmpAZStr);
write_stderr(_("%s: Not support archive function while DMS and DSS enabled\n"), progname);
return FAILURE;
}
// only when the parameter is synchronous_standby_names, this branch can be reached.
if (g_need_changed && 0 == strncmp(config_param[i], "synchronous_standby_names",
strlen(config_param[i]) > strlen("synchronous_standby_names") ? strlen(config_param[i]) : strlen("synchronous_standby_names"))) {

3
src/bin/initdb/CMakeLists.txt
View File

@ -5,6 +5,7 @@ execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/backend/utils/mb/encnames.cpp ${CMAKE_CURRENT_SOURCE_DIR}/encnames.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/timezone/localtime.cpp ${CMAKE_CURRENT_SOURCE_DIR}/localtime.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/interfaces/libpq/pqsignal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pqsignal.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/dss_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
set(TGT_initdb_SRC
@ -13,6 +14,8 @@ set(TGT_initdb_SRC
${CMAKE_CURRENT_SOURCE_DIR}/initdb.cpp
${CMAKE_CURRENT_SOURCE_DIR}/localtime.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pqsignal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ss_initdb.cpp
${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
set(TGT_initdb_INC

3
src/bin/initdb/Makefile
View File

@ -30,7 +30,8 @@ ifneq "$(MAKECMDGOALS)" "clean"
endif
endif
endif
OBJS= initdb.o findtimezone.o localtime.o encnames.o pqsignal.o $(WIN32RES) $(top_builddir)/src/lib/elog/elog.a
OBJS= initdb.o findtimezone.o ss_initdb.o localtime.o encnames.o pqsignal.o $(WIN32RES) $(top_builddir)/src/lib/elog/elog.a \
$(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o
all: gs_initdb
gs_initdb: $(OBJS) | submake-libpgport

443
src/bin/initdb/initdb.cpp
View File

@ -59,6 +59,11 @@
#include "getopt_long.h"
#include "miscadmin.h"
#include "bin/elog.h"
#include "catalog/pg_control.h"
#include "storage/dss/dss_adaptor.h"
#include "ss_initdb.h"
#include "storage/smgr/smgr.h"
#include "storage/file/fio_device.h"
#ifdef ENABLE_MULTIPLE_NODES
#include "distribute_core.h"
@ -75,6 +80,7 @@
* Note that this macro must be the same to fd.h
*/
#define PG_TEMP_FILES_DIR "pgsql_tmp"
#define SS_PG_TEMP_FILES_DIR "ss_pgsql_tmp"
#define RESULT_LENGTH 20
#define BUF_LENGTH 64
#define PG_CAST_CHAR_LENGTH 1024
@ -148,6 +154,11 @@ static char* lc_time = "";
static char* lc_messages = "";
static const char* default_text_search_config = "";
static char* username = "";
static char* vgname = "";
static char* vgdata = "";
static char* vglog = "";
static char* socketpath = NULL;
static bool enable_dss = false;
static bool pwprompt = false;
static char* pwfilename = NULL;
static const char* authmethodhost = "";
@ -262,9 +273,9 @@ static const char* raw_backend_options = "--single "
static char bin_path[MAXPGPATH];
static char backend_exec[MAXPGPATH];
static void* pg_malloc(size_t size);
void* pg_malloc(size_t size);
static char* xstrdup(const char* s);
static char** replace_token(char** lines, const char* token, const char* replacement);
char** replace_token(char** lines, const char* token, const char* replacement);
#ifndef HAVE_UNIX_SOCKETS
static char** filter_lines_with_token(char** lines, const char* token);
@ -277,7 +288,7 @@ static void pre_sync_fname(char *fname, bool isdir);
static void fsync_fname(char *fname, bool isdir);
#endif
static FILE* popen_check(const char* command, const char* mode);
static void exit_nicely(void);
void exit_nicely(void);
static char* get_id(void);
static char* get_encoding_id(char* encoding_name);
static void set_input(char** dest, const char* filename);
@ -473,7 +484,7 @@ void check_env_value(const char* input_env_value)
* Note that we can't call exit_nicely() on a memory failure, as it calls
* rmtree() which needs memory allocation. So we just exit with a bang.
*/
static void* pg_malloc(size_t size)
void* pg_malloc(size_t size)
{
void* result = NULL;
@ -507,7 +518,7 @@ static char* xstrdup(const char* s)
* This does most of what sed was used for in the shell script, but
* doesn't need any regexp stuff.
*/
static char** replace_token(char** lines, const char* token, const char* replacement)
char** replace_token(char** lines, const char* token, const char* replacement)
{
int numlines = 1;
int i;
@ -802,7 +813,7 @@ pre_sync_fname(char *fname, bool isdir)
#if defined(HAVE_SYNC_FILE_RANGE) || (defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED))
int fd;
fd = open(fname, O_RDONLY | PG_BINARY);
fd = open(fname, O_RDONLY | PG_BINARY, 0);
/*
* Some OSs don't allow us to open directories at all (Windows returns
@ -851,9 +862,9 @@ fsync_fname(char *fname, bool isdir)
* cases here
*/
if (!isdir)
fd = open(fname, O_RDWR | PG_BINARY);
fd = open(fname, O_RDWR | PG_BINARY, 0);
else
fd = open(fname, O_RDONLY | PG_BINARY);
fd = open(fname, O_RDONLY | PG_BINARY, 0);
/*
* Some OSs don't allow us to open directories at all (Windows returns
@ -906,11 +917,42 @@ static FILE* popen_check(const char* command, const char* mode)
return cmdfd;
}
static void rm_dss_dir(char *path)
{
char filepath[MAXPGPATH];
dirent *ent = NULL;
struct stat statbuf;
DIR *dir = opendir(path);
if (dir == NULL) {
return;
}
while ((ent = gs_readdir(dir)) != NULL) {
if (strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0 &&
strcmp(ent->d_name, ".recycle") != 0) {
int nRet = snprintf_s(filepath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", path, ent->d_name);
securec_check_ss_c(nRet, "\0", "\0");
if (lstat(filepath, &statbuf) != 0) {
continue;
}
if (S_ISDIR(statbuf.st_mode)) {
rm_dss_dir(filepath);
} else {
(void)unlink(filepath);
}
}
}
(void)closedir(dir);
}
/*
* clean up any files we created on failure
* if we created the data directory remove it too
*/
static void exit_nicely(void)
void exit_nicely(void)
{
if (!noclean) {
if (made_new_pgdata) {
@ -932,6 +974,17 @@ static void exit_nicely(void)
if (!rmtree(xlog_dir, false))
write_stderr(_("%s: failed to remove contents of transaction log directory\n"), progname);
}
if (enable_dss) {
write_stderr(_("%s: removing dss directory \"%s\"\n"), progname, vgname);
if (strlen(vgdata) > 0) {
rm_dss_dir(vgdata);
}
if (strlen(vglog) > 0) {
rm_dss_dir(vglog);
}
}
/* otherwise died during startup, do nothing! */
} else {
if (made_new_pgdata || found_existing_pgdata)
@ -1228,6 +1281,35 @@ static void CreatePGDefaultTempDir()
exit_nicely();
}
FREE_AND_RESET(path);
/* Create specific pgsql_tmp dir in sharedstorage mode
*
* Files in pgsql_tmp are temporary and may be big in some complicated
* condition, so it's best to storeage it in file system. Therefore we
* will create a specific dir in PGDATA when we enable dss mode. The
* path can not be customized now, but we may remove this limit in the
* future.
*/
if (enable_dss) {
size_t length = strlen(pg_data) + strlen(SS_PG_TEMP_FILES_DIR) + 13;
char *ss_path = (char*)pg_malloc(length);
rc = sprintf_s(ss_path, length,
"%s/%s",
pg_data,
SS_PG_TEMP_FILES_DIR);
securec_check_ss_c(rc, ss_path, "\0");
if (mkdir(ss_path, S_IRWXU) < 0) {
char errBuffer[ERROR_LIMIT_LEN];
write_stderr(_("%s: could not mkdir \"%s\": %s\n"),
progname,
ss_path,
pqStrerror(errno, errBuffer, ERROR_LIMIT_LEN));
FREE_AND_RESET(ss_path);
exit_nicely();
}
FREE_AND_RESET(ss_path);
}
}
/*
@ -1280,13 +1362,16 @@ static void test_config_settings(void)
static const int trial_conns[] = {100, 50, 40, 30, 20, 10};
static const int trial_bufs[] = {
4096, 3584, 3072, 2560, 2048, 1536, 1000, 900, 800, 700, 600, 500, 400, 300, 200, 100, 50};
131072, 4096, 3584, 3072, 2560, 2048, 1536, 1000, 900, 800, 700, 600, 500, 400, 300, 200, 100, 50};
char cmd[MAXPGPATH];
const int connslen = sizeof(trial_conns) / sizeof(int);
const int bufslen = sizeof(trial_bufs) / sizeof(int);
int nRet = 0;
int i, status, test_conns, test_buffs, ok_buffers = 0;
char** conflines;
char path[MAXPGPATH];
char repltok[TZ_STRLEN_MAX + 100];
printf(_("selecting default max_connections ... "));
(void)fflush(stdout);
@ -1323,6 +1408,15 @@ static void test_config_settings(void)
printf("%d\n", n_connections);
nRet = sprintf_s(path, sizeof(path), "%s/postgresql.conf", pg_data);
securec_check_ss_c(nRet, "\0", "\0");
/* read postgresql.conf in pg_data, so the config setting can be retained */
conflines = readfile(path);
nRet = sprintf_s(repltok, sizeof(repltok), "max_connections = %d", n_connections);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#max_connections = 100", repltok);
printf(_("selecting default shared_buffers ... "));
(void)fflush(stdout);
@ -1354,10 +1448,19 @@ static void test_config_settings(void)
}
n_buffers = test_buffs;
if ((n_buffers * (BLCKSZ / 1024)) % 1024 == 0)
if ((n_buffers * (BLCKSZ / 1024)) % 1024 == 0) {
printf("%dMB\n", (n_buffers * (BLCKSZ / 1024)) / 1024);
else
nRet = sprintf_s(repltok, sizeof(repltok), "shared_buffers = %dMB", (n_buffers * (BLCKSZ / 1024)) / 1024);
} else {
printf("%dkB\n", n_buffers * (BLCKSZ / 1024));
nRet = sprintf_s(repltok, sizeof(repltok), "shared_buffers = %dkB", n_buffers * (BLCKSZ / 1024));
}
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#shared_buffers = 32MB", repltok);
writefile(path, conflines);
(void)chmod(path, S_IRUSR | S_IWUSR);
FREE_AND_RESET(conflines);
}
/*
@ -1378,25 +1481,14 @@ static void setup_config(void)
char* buf_default_text_search_config = NULL;
char* buf_nodename = NULL;
char* buf_default_timezone = NULL;
char* buf_socketpath = NULL;
fputs(_("creating configuration files ... "), stdout);
(void)fflush(stdout);
/* postgresql.conf */
conflines = readfile(conf_file);
nRet = sprintf_s(repltok, sizeof(repltok), "max_connections = %d", n_connections);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#max_connections = 100", repltok);
if ((n_buffers * (BLCKSZ / 1024)) % 1024 == 0)
nRet = sprintf_s(repltok, sizeof(repltok), "shared_buffers = %dMB", (n_buffers * (BLCKSZ / 1024)) / 1024);
else
nRet = sprintf_s(repltok, sizeof(repltok), "shared_buffers = %dkB", n_buffers * (BLCKSZ / 1024));
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#shared_buffers = 32MB", repltok);
#if DEF_PGPORT != 5432
nRet = sprintf_s(repltok, sizeof(repltok), "#port = %d", DEF_PGPORT);
securec_check_ss_c(nRet, "\0", "\0");
@ -1481,6 +1573,28 @@ static void setup_config(void)
FREE_AND_RESET(buf_xlog_file_path);
}
if (strlen(vgdata) != 0) {
nRet = sprintf_s(repltok, sizeof(repltok), "ss_dss_vg_name = '%s'", vgdata);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_dss_vg_name = ''", repltok);
}
if (socketpath != NULL) {
buf_socketpath = escape_quotes(socketpath);
nRet = sprintf_s(repltok, sizeof(repltok), "ss_dss_conn_path = '%s'", buf_socketpath);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_dss_conn_path = ''", repltok);
FREE_AND_RESET(buf_socketpath);
}
if (enable_dss) {
nRet = strcpy_s(repltok, sizeof(repltok), "ss_enable_dss = on");
securec_check_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_enable_dss = off", repltok);
conflines = ss_addnodeparmater(conflines);
}
nRet = sprintf_s(path, sizeof(path), "%s/postgresql.conf", pg_data);
securec_check_ss_c(nRet, "\0", "\0");
@ -1722,9 +1836,11 @@ static void bootstrap_template1(void)
securec_check_ss_c(nRet, "\0", "\0");
PG_CMD_OPEN;
for (line = bki_lines; *line != NULL; line++) {
PG_CMD_PUTS(*line);
if (ss_need_mkclusterdir) {
PG_CMD_PUTS(*line);
}
FREE_AND_RESET(*line);
}
@ -3681,6 +3797,7 @@ static void usage(const char* prog_name)
#ifndef ENABLE_MULTIPLE_NODES
printf(_(" -c, --enable-dcf enable DCF mode\n"));
#endif
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" [-D, --pgdata=]DATADIR location for this database cluster\n"));
#ifdef ENABLE_MULTIPLE_NODES
printf(_(" --nodename=NODENAME name of openGauss node initialized\n"));
@ -3688,6 +3805,9 @@ static void usage(const char* prog_name)
#else
printf(_(" --nodename=NODENAME name of single node initialized\n"));
#endif
printf(_(" --vgname=VGNAME name of dss volume group\n"));
printf(_(" --socketpath=SOCKETPATH\n"
" dss connect socket file path\n"));
printf(_(" -E, --encoding=ENCODING set default encoding for new databases\n"));
printf(_(" --locale=LOCALE set default locale for new databases\n"));
printf(_(" --dbcompatibility=DBCOMPATIBILITY set default dbcompatibility for new database\n"));
@ -3771,6 +3891,36 @@ static bool is_file_exist(const char* path)
return isExist;
}
static void parse_vgname_args(char* args)
{
vgname = xstrdup(args);
enable_dss = true;
if (strstr(vgname, "/") != NULL) {
fprintf(stderr, "invalid token \"/\" in vgname");
exit(1);
}
char *comma = strstr(vgname, ",");
if (comma == NULL) {
vgdata = vgname;
vglog = (char *)"";
return;
}
vgdata = xstrdup(vgname);
comma = strstr(vgdata, ",");
comma[0] = '\0';
vglog = comma + 1;
if (strstr(vgdata, ",") != NULL) {
fprintf(stderr, "invalid vgname args, should be two volume group names, example: \"+data,+log\"");
exit(1);
}
if (strstr(vglog, ",") != NULL) {
fprintf(stderr, "invalid vgname args, should be two volume group names, example: \"+data,+log\"");
exit(1);
}
}
int main(int argc, char* argv[])
{
/*
@ -3815,6 +3965,10 @@ int main(int argc, char* argv[])
#endif
{"dbcompatibility", required_argument, NULL, 13},
{"bucketlength", required_argument, NULL, 14},
{"vgname", required_argument, NULL, 15},
{"socketpath", required_argument, NULL, 16},
{"enable-dss", no_argument, NULL, 17},
{"dms_url", required_argument, NULL, 18},
{NULL, 0, NULL, 0}};
int c, i, ret;
@ -3886,9 +4040,9 @@ int main(int argc, char* argv[])
/* process command-line options */
#ifdef ENABLE_LITE_MODE
while ((c = getopt_long(argc, argv, "cdD:E:L:nNU:WA:SsT:X:C:w:H:g:", long_options, &option_index)) != -1) {
while ((c = getopt_long(argc, argv, "cdD:E:L:nNU:WA:SsT:X:C:w:H:g:I:", long_options, &option_index)) != -1) {
#else
while ((c = getopt_long(argc, argv, "cdD:E:L:nU:WA:SsT:X:C:w:H:g:", long_options, &option_index)) != -1) {
while ((c = getopt_long(argc, argv, "cdD:E:L:nU:WA:SsT:X:C:w:H:g:I:", long_options, &option_index)) != -1) {
#endif
#define FREE_NOT_STATIC_ZERO_STRING(s) \
do { \
@ -4073,6 +4227,14 @@ int main(int argc, char* argv[])
check_input_spec_char(optarg);
host_ip = xstrdup(optarg);
break;
case 'I':
if (atoi(optarg) < MIN_INSTANCEID || atoi(optarg) > MAX_INSTANCEID) {
write_stderr(_("unexpected node id specified, valid range is %d - %d.\n"),
MIN_INSTANCEID, MAX_INSTANCEID);
exit(1);
}
ss_nodeid = atoi(optarg);
break;
#ifdef PGXC
case 12:
FREE_NOT_STATIC_ZERO_STRING(nodename);
@ -4093,7 +4255,25 @@ int main(int argc, char* argv[])
}
g_bucket_len = atoi(optarg);
break;
case 15:
FREE_NOT_STATIC_ZERO_STRING(vgname);
FREE_NOT_STATIC_ZERO_STRING(vgdata);
FREE_NOT_STATIC_ZERO_STRING(vglog);
parse_vgname_args(optarg);
break;
case 16:
FREE_NOT_STATIC_ZERO_STRING(socketpath);
socketpath = xstrdup(optarg);
enable_dss = true;
break;
case 17:
enable_dss = true;
break;
case 18:
FREE_NOT_STATIC_ZERO_STRING(ss_nodedatainfo);
check_input_spec_char(optarg);
ss_nodedatainfo = xstrdup(optarg);
break;
default:
/* getopt_long already emitted a complaint */
write_stderr(_("Try \"%s --help\" for more information.\n"), progname);
@ -4101,7 +4281,10 @@ int main(int argc, char* argv[])
}
#undef FREE_NOT_STATIC_ZERO_STRING
}
/* check nodedata.cfg and node_id */
ss_issharedstorage = ss_check_nodedatainfo();
if (default_text_search_config_tmp != NULL)
default_text_search_config = default_text_search_config_tmp;
if (authmethodhost_tmp != NULL)
@ -4460,6 +4643,12 @@ int main(int argc, char* argv[])
(void)umask(S_IRWXG | S_IRWXO);
// dss device init
if (dss_device_init(socketpath, enable_dss) != DSS_SUCCESS) {
write_stderr(_("failed to init dss device"));
exit_nicely();
}
// log output redirect
init_log(PROG_NAME);
@ -4577,10 +4766,21 @@ int main(int argc, char* argv[])
}
}
if (ss_issharedstorage && (ss_check_shareddir(vgdata))) {
ss_need_mkclusterdir = false;
}
/* Create transaction log symlink, if required */
if (strcmp(xlog_dir, "") != 0) {
char linkloc[MAXPGPATH] = {'\0'};
/* check if specify xlog directory in shared storage mode */
if (enable_dss) {
write_stderr(_("%s: can not specify transaction log directory "
"location in shared storage mode\n"), progname);
exit_nicely();
}
/* clean up xlog directory name, check it's absolute */
canonicalize_path(xlog_dir);
if (!is_absolute_path(xlog_dir)) {
@ -4664,46 +4864,50 @@ int main(int argc, char* argv[])
exit_nicely();
#endif
}
if (enable_dss && ss_issharedstorage) {
ss_mkdirdir(ss_nodeid, pg_data, vgdata, vglog, ss_need_mkclusterdir);
} else {
/* Create required subdirectories */
printf(_("creating subdirectories ... in ordinary occasion"));
(void)fflush(stdout);
/* Create required subdirectories */
printf(_("creating subdirectories ... "));
(void)fflush(stdout);
for (i = 0; (unsigned int)(i) < lengthof(subdirs); i++) {
char* path = NULL;
errno_t sret = 0;
size_t len = 0;
for (i = 0; (unsigned int)(i) < lengthof(subdirs); i++) {
char* path = NULL;
errno_t sret = 0;
size_t len = 0;
/*
* -X means using user define xlog directory, and will create symbolic pg_xlog
* under pg_data directory, So no need to create these sub-directories again.
*/
if (pg_strcasecmp(xlog_dir, "") != 0 && (pg_strcasecmp(subdirs[i], "pg_xlog") == 0)) {
continue;
}
/*
* -X means using user define xlog directory, and will create symbolic pg_xlog
* under pg_data directory, So no need to create these sub-directories again.
*/
if (pg_strcasecmp(xlog_dir, "") != 0 && (pg_strcasecmp(subdirs[i], "pg_xlog") == 0)) {
continue;
}
len = strlen(pg_data) + strlen(subdirs[i]) + 2;
path = (char*)pg_malloc(len);
len = strlen(pg_data) + strlen(subdirs[i]) + 2;
path = (char*)pg_malloc(len);
sret = sprintf_s(path, len, "%s/%s", pg_data, subdirs[i]);
securec_check_ss_c(sret, path, "\0");
sret = sprintf_s(path, len, "%s/%s", pg_data, subdirs[i]);
securec_check_ss_c(sret, path, "\0");
/*
* The parent directory already exists, so we only need mkdir() not
* pg_mkdir_p() here, which avoids some failure modes; cf bug #13853.
*/
if (mkdir(path, S_IRWXU) < 0) {
char errBuffer[ERROR_LIMIT_LEN];
fprintf(stderr,
_("%s: could not create directory \"%s\": %s\n"),
progname,
path,
pqStrerror(errno, errBuffer, ERROR_LIMIT_LEN));
/*
* The parent directory already exists, so we only need mkdir() not
* pg_mkdir_p() here, which avoids some failure modes; cf bug #13853.
*/
if (mkdir(path, S_IRWXU) < 0) {
char errBuffer[ERROR_LIMIT_LEN];
fprintf(stderr,
_("%s: could not create directory \"%s\": %s\n"),
progname,
path,
pqStrerror(errno, errBuffer, ERROR_LIMIT_LEN));
FREE_AND_RESET(path);
exit_nicely();
}
FREE_AND_RESET(path);
exit_nicely();
}
FREE_AND_RESET(path);
}
if (strlen(new_xlog_file_path) == 0) {
@ -4738,87 +4942,103 @@ int main(int argc, char* argv[])
}
}
}
/* create or check pg_location path */
mkdirForPgLocationDir();
check_ok();
if (enable_dss) {
size_t total_len = strlen(boot_options) + BUF_LENGTH;
char *options = (char*)pg_malloc(total_len);
errno_t sret = sprintf_s(options, total_len, "%s -c segment_buffers=128MB -G", boot_options);
securec_check_ss_c(sret, options, "\0");
boot_options = options;
total_len = strlen(backend_options) + BUF_LENGTH;
options = (char*)pg_malloc(total_len);
sret = sprintf_s(options, total_len, "%s -c segment_buffers=128MB -G", backend_options);
securec_check_ss_c(sret, options, "\0");
backend_options = options;
}
if (ss_need_mkclusterdir) {
/* create or check pg_location path */
mkdirForPgLocationDir();
check_ok();
}
/* Top level PG_VERSION is checked by bootstrapper, so make it first */
write_version_file(NULL);
create_pg_lockfiles();
/* Select suitable configuration settings */
/* Create all the text config files and select suitable configuration setting */
set_null_conf();
test_config_settings();
/* Now create all the text config files */
setup_config();
test_config_settings();
/* Init undo subsystem meta. */
InitUndoSubsystemMeta();
/* Bootstrap template1 */
bootstrap_template1();
if (ss_need_mkclusterdir) {
/*
* Make the per-database PG_VERSION for template1 only after init'ing it
*/
write_version_file("base/1");
/*
* Make the per-database PG_VERSION for template1 only after init'ing it
*/
write_version_file("base/1");
CreatePGDefaultTempDir();
CreatePGDefaultTempDir();
/* Create the stuff we don't need to use bootstrap mode for */
/* Create the stuff we don't need to use bootstrap mode for */
setup_auth();
get_set_pwd();
setup_auth();
get_set_pwd();
setup_depend();
load_plpgsql();
setup_sysviews();
setup_depend();
load_plpgsql();
setup_sysviews();
#ifdef ENABLE_PRIVATEGAUSS
setup_privsysviews();
setup_privsysviews();
#endif
setup_perfviews();
setup_perfviews();
#ifdef PGXC
/* Initialize catalog information about the node self */
setup_nodeself();
/* Initialize catalog information about the node self */
setup_nodeself();
#endif
setup_description();
setup_description();
setup_collation();
setup_collation();
setup_conversion();
setup_conversion();
setup_dictionary();
setup_dictionary();
setup_privileges();
setup_privileges();
setup_bucketmap_len();
setup_bucketmap_len();
setup_schema();
setup_schema();
load_supported_extension();
load_supported_extension();
setup_update();
setup_update();
#ifndef ENABLE_MULTIPLE_NODES
setup_snapshots();
setup_snapshots();
#endif
vacuum_db();
vacuum_db();
make_template0();
make_template0();
make_postgres();
make_postgres();
#ifdef PGXC
vacuumfreeze("template0");
vacuumfreeze("template1");
vacuumfreeze("postgres");
vacuumfreeze("template0");
vacuumfreeze("template1");
vacuumfreeze("postgres");
#endif
}
#ifdef ENABLE_LITE_MODE
if (do_sync)
@ -4907,10 +5127,10 @@ int main(int argc, char* argv[])
static bool isDirectory(const char* basepath, const char* name)
{
struct stat buf;
char* path = NULL;
int nRet = 0;
size_t len = 0;
struct stat buf;
len = strlen(basepath) + strlen(name) + 2;
path = (char*)pg_malloc(len);
@ -4946,7 +5166,7 @@ static bool isMountDirCorrect(const char* basepath, const char* name)
securec_check_ss_c(nRet, path, "\0");
if ((chk_mount_dir = opendir(path)) != NULL) {
while ((de_mount = gs_readdir(chk_mount_dir)) != NULL) {
while ((de_mount = readdir(chk_mount_dir)) != NULL) {
if (strcmp(".", de_mount->d_name) == 0 || strcmp("..", de_mount->d_name) == 0) {
/* skip this and parent directory */
continue;
@ -4977,17 +5197,22 @@ static void mkdirForPgLocationDir()
char* path = NULL;
int nRet = 0;
size_t len = 0;
char* datadir = pg_data;
len = strlen(pg_data) + strlen("pg_location") + 2;
if (enable_dss) {
datadir = vgdata;
}
len = strlen(datadir) + 1 + strlen("pg_location") + 1;
path = (char*)pg_malloc(len);
nRet = sprintf_s(path, len, "%s/pg_location", pg_data);
nRet = sprintf_s(path, len, "%s/pg_location", datadir);
securec_check_ss_c(nRet, "\0", "\0");
switch (pg_check_dir(path)) {
case 0:
/* directory not there, must create it */
if (pg_mkdir_p(path, S_IRWXU) != 0) {
if (mkdir(path, S_IRWXU) < 0) {
char errBuffer[ERROR_LIMIT_LEN];
write_stderr(_("%s: could not create directory \"%s\": %s\n"),
progname,
@ -5001,7 +5226,7 @@ static void mkdirForPgLocationDir()
case 1:
/* Present but empty, fix permissions and use it */
if (chmod(path, S_IRWXU) != 0) {
if (is_dss_file(path) != 0 && chmod(path, S_IRWXU) != 0) {
char errBuffer[ERROR_LIMIT_LEN];
write_stderr(_("%s: could not change permissions of directory \"%s\": %s\n"),
progname,
@ -5016,7 +5241,7 @@ static void mkdirForPgLocationDir()
case 2:
/* Present and not empty */
{
DIR* chk_pg_location_dir = NULL;
DIR *chk_pg_location_dir = NULL;
struct dirent* de_pg_location = NULL;
if ((chk_pg_location_dir = opendir(path)) != NULL) {

363
src/bin/initdb/ss_initdb.cpp Normal file
View File

@ -0,0 +1,363 @@
/*
* Copyright (c) 2020 Huawei Technologies Co.,Ltd.
*
* openGauss is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
*
* http://license.coscl.org.cn/MulanPSL2
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
* ---------------------------------------------------------------------------------------
*
* ss_initdb.cpp
*
*
* IDENTIFICATION
* src/bin/initdb/ss_initdb.cpp
*
* ---------------------------------------------------------------------------------------
*/
#include "postgres_fe.h"
#include <cipher.h>
#include "access/ustore/undo/knl_uundoapi.h"
#include "access/ustore/undo/knl_uundotxn.h"
#include "libpq/pqsignal.h"
#include "mb/pg_wchar.h"
#include "getaddrinfo.h"
#include "getopt_long.h"
#include "miscadmin.h"
#include "bin/elog.h"
#include "ss_initdb.h"
#include "storage/file/fio_device.h"
static const char* ss_clusterdirs[] = {"+global",
"+base",
"+pg_tblspc",
"+pg_clog",
"+pg_csnlog",
"+pg_multixact",
"+pg_multixact/members",
"+pg_multixact/offsets",
"+pg_twophase",
"+pg_serial"};
static const char* ss_instancedirs[] = {"+pg_xlog",
"+pg_doublewrite"
};
static const char* ss_instanceowndirs[] = {"base",
"base/1",
"global",
"pg_xlog",
"pg_xlog/archive_status",
"undo",
"pg_replslot",
"pg_stat_tmp",
"pg_errorinfo",
"pg_logical",
"pg_llog",
"pg_llog/snapshots",
"pg_llog/mappings",
"pg_clog",
"pg_notify",
"pg_csnlog",
"pg_multixact",
"pg_multixact/members",
"pg_multixact/offsets",
"pg_snapshots"};
static const char* ss_xlogsubdirs[] = {"archive_status"};
/* num of every directory type */
#define SS_CLUSTERDIRS_NUM ARRAY_NUM(ss_clusterdirs)
#define SS_INSTANCEDIRS_NUM ARRAY_NUM(ss_instancedirs)
#define SS_INSTANCEOENDIRS_NUM ARRAY_NUM(ss_instanceowndirs)
#define SS_XLOGSUBIRS_NUM ARRAY_NUM(ss_xlogsubdirs)
char *ss_nodedatainfo = NULL;
int32 ss_nodeid = INVALID_INSTANCEID;
bool ss_issharedstorage = false;
bool ss_need_mkclusterdir = true;
static const char *ss_progname = "ss_initdb";
/*
* pg_ltoa: converts a signed 32-bit integer to its string representation
*
* Caller must ensure that 'a' points to enough memory to hold the result
* (at least 12 bytes, counting a leading sign and trailing NUL).
*/
void pg_ltoa(int32 value, char *a)
{
char *start = a;
bool neg = false;
errno_t ss_rc;
if (a == NULL) {
return;
}
/*
* Avoid problems with the most negative integer not being representable
* as a positive integer.
*/
if (value == (-2147483647 - 1)) {
const int a_len = 12;
ss_rc = memcpy_s(a, a_len, "-2147483648", a_len);
securec_check(ss_rc, "\0", "\0");
return;
} else if (value < 0) {
value = -value;
neg = true;
}
/* Compute the result string backwards. */
do {
int32 remainder;
int32 oldval = value;
value /= 10;
remainder = oldval - value * 10;
*a++ = (char)('0' + remainder);
} while (value != 0);
if (neg) {
*a++ = '-';
}
/* Add trailing NUL byte, and back up 'a' to the last character. */
*a-- = '\0';
/* Reverse string. */
while (start < a) {
char swap = *start;
*start++ = *a;
*a-- = swap;
}
}
/* check dms url when gs_initdb */
bool ss_check_nodedatainfo()
{
bool issharedstorage = false;
if ((ss_nodeid == INVALID_INSTANCEID && ss_nodedatainfo != NULL) ||
(ss_nodeid != INVALID_INSTANCEID && ss_nodedatainfo == NULL)) {
issharedstorage = false;
fprintf(stderr, _("ss_nodeid is invalid or nodedatainfo file not exist or nodedatainfo file is empty.\n"));
exit(1);
}
if (ss_nodeid != INVALID_INSTANCEID && ss_nodedatainfo != NULL) {
issharedstorage = true;
}
return issharedstorage;
}
bool ss_check_existclusterdir(const char *path)
{
for (uint32 i = 0; i < SS_CLUSTERDIRS_NUM; i++) {
if (strcmp(ss_clusterdirs[i] + 1, path) == 0) {
/* skip this and parent directory */
return true;
}
}
return false;
}
bool ss_check_shareddir(char *path)
{
char *datadir = path;
DIR *chk_pg_data_dir = NULL;
struct dirent *file = NULL;
if ((chk_pg_data_dir = opendir(datadir)) != NULL) {
while ((file = readdir(chk_pg_data_dir)) != NULL) {
if (strcmp(".", file->d_name) == 0 || strcmp("..", file->d_name) == 0) {
/* skip this and parent directory */
continue;
} else if (ss_check_existclusterdir(file->d_name)) {
(void)closedir(chk_pg_data_dir);
return true;
}
}
(void)closedir(chk_pg_data_dir);
}
return false;
}
void ss_mkdirdir(int32 node_id, const char *pg_data, const char *vgdata_dir, const char *vglog_dir,
bool need_mkclusterdir)
{
if (node_id == 0 && !need_mkclusterdir) {
printf(_("The dss file needs to be cleared before node 0 init db.\n"));
(void)fflush(stdout);
exit_nicely();
}
/* Create required subdirectories */
printf(_("creating subdirectories ... in shared storage mode ... "));
(void)fflush(stdout);
/* unshared and instance one copy */
ss_createdir(ss_instanceowndirs, SS_INSTANCEOENDIRS_NUM, INVALID_INSTANCEID, pg_data, vgdata_dir, vglog_dir);
/* shared and instance one copy */
ss_createdir(ss_instancedirs, SS_INSTANCEDIRS_NUM, node_id, pg_data, vgdata_dir, vglog_dir);
/* shared and cluster one copy */
if (need_mkclusterdir) {
ss_createdir(ss_clusterdirs, SS_CLUSTERDIRS_NUM, INVALID_INSTANCEID, pg_data, vgdata_dir, vglog_dir);
}
}
void ss_makedirectory(char *path)
{
/*
* The parent directory already exists, so we only need mkdir() not
* pg_mkdir_p() here, which avoids some failure modes; cf bug #13853.
*/
if (mkdir(path, S_IRWXU) < 0) {
char errBuffer[ERROR_LIMIT_LEN];
fprintf(stderr, _("%s: could not create directory \"%s\": %s\n"), ss_progname, path,
pqStrerror(errno, errBuffer, ERROR_LIMIT_LEN));
(void)fflush(stdout);
exit_nicely();
}
}
void ss_makesubdir(char *path, const char **subdir, uint num)
{
size_t len = 0;
errno_t sret = 0;
for (int i = 0; (unsigned int)i < num; i++) {
len = strlen(path) + strlen(subdir[i]) + 1 + 1;
char *subpath = NULL;
subpath = (char *)pg_malloc(len);
sret = sprintf_s(subpath, len, "%s/%s", path, subdir[i]);
securec_check_ss_c(sret, subpath, "\0");
ss_makedirectory(subpath);
FREE_AND_RESET(subpath);
}
}
static char *ss_concat_path(int32 node_id, const char *parent_dir, const char *dir)
{
char *path = NULL;
char *prepath = NULL;
char nodeid_str[MAXPGPATH];
size_t len = strlen(parent_dir) + 2 + strlen(dir);
/* prepared path by connecting vgname and subdir */
prepath = (char *)pg_malloc(len);
errno_t sret = sprintf_s(prepath, len, "%s/%s", parent_dir, dir + 1);
securec_check_ss_c(sret, prepath, "\0");
if (node_id != INVALID_INSTANCEID) {
pg_ltoa(node_id, nodeid_str);
len = len + strlen(nodeid_str);
path = (char *)pg_malloc(len);
/* full path by connecting prepared path and node id */
sret = sprintf_s(path, len, "%s%d", prepath, node_id);
securec_check_ss_c(sret, path, "\0");
} else {
path = (char *)pg_malloc(len);
sret = sprintf_s(path, len, "%s", prepath);
securec_check_ss_c(sret, path, "\0");
}
FREE_AND_RESET(prepath);
return path;
}
void ss_createdir(const char **ss_dirs, int32 num, int32 node_id, const char *pg_data, const char *vgdata_dir,
const char *vglog_dir)
{
int i;
for (i = 0; i < num; i++) {
char *path = NULL;
errno_t sret = 0;
size_t len = 0;
bool is_dss = is_dss_file(ss_dirs[i]);
bool is_xlog = false;
char *link_path = NULL;
if (vglog_dir[0] != '\0' && (pg_strcasecmp(ss_dirs[i], "+pg_xlog") == 0 ||
pg_strcasecmp(ss_dirs[i], "+pg_doublewrite") == 0 ||
pg_strcasecmp(ss_dirs[i], "+pg_notify") == 0 ||
pg_strcasecmp(ss_dirs[i], "+pg_snapshots") == 0)) {
is_xlog = true;
}
if (is_dss) {
if (is_xlog) {
path = ss_concat_path(node_id, vglog_dir, ss_dirs[i]);
link_path = ss_concat_path(node_id, vgdata_dir, ss_dirs[i]);
} else {
path = ss_concat_path(node_id, vgdata_dir, ss_dirs[i]);
}
} else {
len = strlen(pg_data) + strlen(ss_dirs[i]) + 1 + 1;
path = (char *)pg_malloc(len);
sret = sprintf_s(path, len, "%s/%s", pg_data, ss_dirs[i]);
securec_check_ss_c(sret, path, "\0");
}
ss_makedirectory(path);
if (is_xlog) {
symlink(path, link_path);
}
/* create suddirectory of pg_xlog/pg_multixact/pg_llog */
if (is_dss && pg_strcasecmp(ss_dirs[i] + 1, "pg_xlog") == 0) {
ss_makesubdir(path, ss_xlogsubdirs, SS_XLOGSUBIRS_NUM);
}
FREE_AND_RESET(path);
FREE_AND_RESET(link_path);
}
}
/* ss_addnodeparmater
* function: add the extra parameter for share storage for dms during gs_Initdb
* input: conflines char** parameter of postgresql.conf had been added previuosly
* output: conflines char** parameter of postgresql.conf to be added in this function
*/
char **ss_addnodeparmater(char **conflines)
{
if (!ss_issharedstorage) {
return conflines;
}
int nRet = 0;
char repltok[TZ_STRLEN_MAX + 100];
fputs(_("adding dms parameters to configuration files ... "), stdout);
(void)fflush(stdout);
nRet = sprintf_s(repltok, sizeof(repltok), "ss_instance_id = %d", ss_nodeid);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_instance_id = 0", repltok);
nRet = strcpy_s(repltok, sizeof(repltok), "ss_enable_dms = on");
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_enable_dms = off", repltok);
nRet = sprintf_s(repltok, sizeof(repltok), "ss_interconnect_url = '%s'", ss_nodedatainfo);
securec_check_ss_c(nRet, "\0", "\0");
conflines = replace_token(conflines, "#ss_interconnect_url = '0:127.0.0.1:1611'", repltok);
return conflines;
}

61
src/bin/initdb/ss_initdb.h Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2020 Huawei Technologies Co.,Ltd.
*
* openGauss is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
*
* http://license.coscl.org.cn/MulanPSL2
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
* ---------------------------------------------------------------------------------------
*
* ss_initdb.h
*
*
* IDENTIFICATION
* src/bin/initdb/ss_initdb.h
*
* ---------------------------------------------------------------------------------------
*/
#ifndef SS_INITDB_H_
#define SS_INITTDB_H_
#ifndef ERROR_LIMIT_LEN
#define ERROR_LIMIT_LEN 256
#endif
#define ARRAY_NUM(a) (sizeof(a) / sizeof((a)[0]))
extern char **replace_token(char **lines, const char *token, const char *replacement);
extern void exit_nicely(void);
extern void *pg_malloc(size_t size);
extern char* ss_nodedatainfo;
extern int32 ss_nodeid;
extern bool ss_issharedstorage;
extern bool ss_need_mkclusterdir;
/* check dms url when gs_initdb */
extern bool ss_check_nodedatainfo();
extern bool ss_check_existclusterdir(const char* path);
extern bool ss_check_shareddir(char* path);
extern void ss_createdir(const char** ss_dirs, int32 num, int32 node_id, const char* pg_data, const char* vgdata_dir, const char* vglog_dir);
extern void ss_mkdirdir(int32 node_id, const char* pg_data, const char* vgdata_dir, const char* vglog_dir, bool need_mkclusterdir);
extern char** ss_addnodeparmater(char** conflines);
#define FREE_AND_RESET(ptr) \
do { \
if (NULL != (ptr) && reinterpret_cast<char*>(ptr) != static_cast<char*>("")) { \
free(ptr); \
(ptr) = NULL; \
} \
} while (0)
#endif /* SS_INITDB_H */

3
src/bin/pg_basebackup/CMakeLists.txt
View File

@ -80,6 +80,9 @@ target_link_directories(pg_receivexlog PUBLIC
${LIBOPENSSL_LIB_PATH} ${LIBCGROUP_LIB_PATH} ${LIBCURL_LIB_PATH} ${ZLIB_LIB_PATH} ${LIBOBS_LIB_PATH} ${LIBCGROUP_LIB_PATH} ${LIBEDIT_LIB_PATH}
${LZ4_LIB_PATH} ${SECURE_LIB_PATH} ${KERBEROS_LIB_PATH} ${CMAKE_BINARY_DIR}/lib
)
# BIND_NOW
target_link_options(pg_receivexlog PRIVATE "-Wl,-z,relro,-z,now")
install(TARGETS pg_receivexlog RUNTIME DESTINATION bin)
# pg_recvlogical bin

2
src/bin/pg_basebackup/Makefile
View File

@ -18,7 +18,7 @@ subdir = src/bin/pg_basebackup
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(libpq_srcdir) -I$(ZLIB_INCLUDE_PATH) $(CPPFLAGS) -DHAVE_LIBZ -fPIC -fPIE -DFRONTEND -I$(top_builddir)/src/bin/pg_rewind -I$(top_builddir)/src/bin/pg_ctl -I$(LZ4_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) -I${top_builddir}/src/lib/page_compression
override CPPFLAGS := -I$(libpq_srcdir) -I$(ZLIB_INCLUDE_PATH) $(CPPFLAGS) -DHAVE_LIBZ -fPIC -fPIE -DFRONTEND -I$(top_builddir)/src/bin/pg_rewind -I$(top_builddir)/src/bin/pg_ctl -I$(LZ4_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) -I${top_builddir}/src/lib/page_compression -I${top_builddir}/src/include
LDFLAGS += -Wl,-z,relro,-z,now -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH) -L${top_builddir}/src/lib/page_compression
LIBS += -llz4 -lzstd -lpagecompression

6
src/bin/pg_basebackup/pg_basebackup.cpp
View File

@ -32,6 +32,7 @@
#include <sys/types.h>
#include <sys/wait.h>
#include "tool_common.h"
#include "getopt_long.h"
#include "receivelog.h"
#include "streamutil.h"
@ -83,6 +84,7 @@ extern int tblspaceIndex;
extern int standby_message_timeout; /* 10 sec = default */
/* Progress counters */
static uint64 totalsize;
static uint64 totaldone;
@ -1513,13 +1515,13 @@ static void backup_dw_file(const char *target_dir)
char *unaligned_buf = NULL;
/* Delete the dw file, if it exists. */
remove_dw_file(OLD_DW_FILE_NAME, target_dir, real_file_path);
remove_dw_file(T_OLD_DW_FILE_NAME, target_dir, real_file_path);
rc = memset_s(real_file_path, (PATH_MAX + 1), 0, (PATH_MAX + 1));
securec_check_c(rc, "\0", "\0");
/* Delete the dw build file, if it exists. */
remove_dw_file(DW_BUILD_FILE_NAME, target_dir, real_file_path);
remove_dw_file(T_DW_BUILD_FILE_NAME, target_dir, real_file_path);
/* Create the dw build file. */
if ((fd = open(real_file_path, (DW_FILE_FLAG | O_CREAT), DW_FILE_PERM)) < 0) {

4
src/bin/pg_basebackup/pg_receivexlog.cpp
View File

@ -167,7 +167,7 @@ static XLogRecPtr FindStreamingStart(XLogRecPtr currentpos, uint32 currenttimeli
disconnect_and_exit(1);
}
if (statbuf.st_size == XLOG_SEG_SIZE) {
if (statbuf.st_size == (off_t)XLogSegSize) {
/* Completed segment */
if (segno > high_segno) {
high_segno = segno;
@ -277,7 +277,7 @@ static void StreamLog(void)
/*
* Always start streaming at the beginning of a segment
*/
startpos -= startpos % XLOG_SEG_SIZE;
startpos -= startpos % XLogSegSize;
/*
* Start the replication

18
src/bin/pg_basebackup/receivelog.cpp
View File

@ -130,8 +130,8 @@ static int open_walfile(XLogRecPtr startpoint, uint32 timeline, const char* base
MAXFNAMELEN - 1,
"%08X%08X%08X",
timeline,
(uint32)((startpoint / XLOG_SEG_SIZE) / XLogSegmentsPerXLogId),
(uint32)((startpoint / XLOG_SEG_SIZE) % XLogSegmentsPerXLogId));
(uint32)((startpoint / XLogSegSize) / XLogSegmentsPerXLogId),
(uint32)((startpoint / XLogSegSize) % XLogSegmentsPerXLogId));
securec_check_ss_c(nRet, "", "");
nRet = snprintf_s(fn, sizeof(fn), sizeof(fn) - 1, "%s/%s.partial", basedir, namebuf);
@ -160,11 +160,11 @@ static int open_walfile(XLogRecPtr startpoint, uint32 timeline, const char* base
f = -1;
return -1;
}
if (statbuf.st_size == XLogSegSize)
if (statbuf.st_size == (off_t)XLogSegSize)
return f; /* File is open and ready to use */
if (statbuf.st_size != 0) {
pg_log(PG_PRINT,
_("%s: WAL segment %s is %d bytes, should be 0 or %d\n"),
_("%s: WAL segment %s is %d bytes, should be 0 or %lu\n"),
progname,
Lrealpath,
(int)statbuf.st_size,
@ -302,7 +302,7 @@ static bool close_walfile(int walfile, const char* basedir, char* walname, bool
* Rename the .partial file only if we've completed writing the
* whole segment or segment_complete is true.
*/
if (currpos == XLOG_SEG_SIZE || segment_complete) {
if (currpos == (off_t)XLogSegSize || segment_complete) {
char oldfn[MAXPGPATH];
char newfn[MAXPGPATH];
errno_t nRet;
@ -438,7 +438,7 @@ static bool checkForReceiveTimeout(PGconn* conn)
static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const char* basedir, char* cur_wal_file,
uint32 wal_file_timeline, int& walfile, stream_stop_callback stream_stop, PGconn* conn)
{
int xlogoff = block_pos % XLOG_SEG_SIZE;
int xlogoff = block_pos % XLogSegSize;
int bytes_left = len;
int bytes_to_write = 0;
@ -470,8 +470,8 @@ static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const
while (bytes_left) {
/* If crossing a WAL boundary, only write up until we reach XLOG_SEG_SIZE. */
if (xlogoff + bytes_left > XLOG_SEG_SIZE)
bytes_to_write = XLOG_SEG_SIZE - xlogoff;
if (xlogoff + bytes_left > (int)XLogSegSize)
bytes_to_write = (int)XLogSegSize - xlogoff;
else
bytes_to_write = bytes_left;
@ -502,7 +502,7 @@ static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const
xlogoff += bytes_to_write;
/* Did we reach the end of a WAL segment? */
if (block_pos % XLOG_SEG_SIZE == 0) {
if (block_pos % XLogSegSize == 0) {
if (!close_walfile(walfile, basedir, cur_wal_file, false, block_pos)) {
suspendHeartBeatTimer();
/* Error message written in close_walfile() */

4
src/bin/pg_controldata/CMakeLists.txt
View File

@ -1,6 +1,10 @@
#This is the main CMAKE for build all components.
execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/dss_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
set(TGT_controldata_SRC
${CMAKE_CURRENT_SOURCE_DIR}/pg_controldata.cpp
${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
set(TGT_controldata_INC

3
src/bin/pg_controldata/Makefile
View File

@ -22,7 +22,8 @@ ifneq "$(MAKECMDGOALS)" "clean"
endif
endif
endif
OBJS= pg_controldata.o $(WIN32RES) $(top_builddir)/src/lib/elog/elog.a
OBJS= pg_controldata.o $(WIN32RES) $(top_builddir)/src/lib/elog/elog.a \
$(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -fPIC -fPIE

252
src/bin/pg_controldata/pg_controldata.cpp
View File

@ -27,17 +27,29 @@
#include "access/xlog.h"
#include "catalog/pg_control.h"
#include "bin/elog.h"
#include "getopt_long.h"
#include "storage/dss/dss_adaptor.h"
#include "storage/file/fio_device.h"
#define FirstNormalTransactionId ((TransactionId)3)
#define TransactionIdIsNormal(xid) ((xid) >= FirstNormalTransactionId)
static void usage(const char* progname)
static const char *progname;
static bool enable_dss = false;
static void usage(const char* prog_name)
{
printf(_("%s displays control information of a openGauss database cluster.\n\n"), progname);
printf(_("%s displays control information of a openGauss database cluster.\n\n"), prog_name);
printf(_("Usage:\n"));
printf(_(" %s [OPTION] [DATADIR]\n"), progname);
printf(_(" %s [OPTION] [DATADIR]\n"), prog_name);
printf(_("\nOptions:\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_(" -I, --instance-id=INSTANCE_ID\n"));
printf(_(" display information of specified instance (default all)\n"));
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" --socketpath=SOCKETPATH\n"));
printf(_(" dss connect socket file path\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nIf no data directory (DATADIR) is specified, "
"the environment variable PGDATA\nis used.\n"));
#if ((defined(ENABLE_MULTIPLE_NODES)) || (defined(ENABLE_PRIVATEGAUSS)))
@ -87,87 +99,30 @@ static const char* wal_level_str(WalLevel wal_level)
return _("unrecognized wal_level");
}
int main(int argc, char* argv[])
static void exit_safely(int returnCode)
{
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(returnCode);
}
static void display_data(ControlFileData ControlFile, int instance_id)
{
ControlFileData ControlFile;
int fd = -1;
char ControlFilePath[MAXPGPATH];
char* DataDir = NULL;
pg_crc32c crc; /* pg_crc32c as same as pg_crc32 */
time_t time_tmp;
char pgctime_str[128];
char ckpttime_str[128];
char sysident_str[32];
const char* strftime_fmt = "%c";
const char* progname = NULL;
int sret = 0;
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_controldata"));
progname = get_progname(argv[0]);
if (argc > 1) {
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) {
usage(progname);
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) {
#ifdef ENABLE_MULTIPLE_NODES
puts("pg_controldata (PostgreSQL) " PG_VERSION);
#else
puts("pg_controldata (openGauss) " PG_VERSION);
#endif
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(0);
}
/* display instance id in shared storage mode */
if (enable_dss) {
printf(_("\npg_control data (instance id %d)\n\n"), instance_id);
}
if (argc > 1) {
DataDir = argv[1];
} else {
DataDir = getenv("PGDATA");
}
if (DataDir == NULL) {
fprintf(stderr, _("%s: no data directory specified\n"), progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(1);
}
check_env_value_c(DataDir);
sret = snprintf_s(ControlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/global/pg_control", DataDir);
securec_check_ss_c(sret, "\0", "\0");
if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) {
fprintf(
stderr, _("%s: could not open file \"%s\" for reading: %s\n"), progname, ControlFilePath, strerror(errno));
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(2);
}
if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) {
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), progname, ControlFilePath, strerror(errno));
close(fd);
if (progname != NULL) {
free((char*)progname);
progname = NULL;
}
exit(2);
}
close(fd);
/* Check the CRC. */
/* using CRC32C since 923 */
INIT_CRC32C(crc);
@ -231,7 +186,7 @@ int main(int argc, char* argv[])
printf(_("Latest checkpoint's oldestXID: " XID_FMT "\n"), ControlFile.checkPointCopy.oldestXid);
printf(_("Latest checkpoint's oldestXID's DB: %u\n"), ControlFile.checkPointCopy.oldestXidDB);
printf(_("Latest checkpoint's oldestActiveXID: " XID_FMT "\n"), ControlFile.checkPointCopy.oldestActiveXid);
printf(_("Latest checkpoint's remove lsn: %X/%X\n"),
printf(_("Latest checkpoint's remove lsn: %X/%X\n"),
(uint32)(ControlFile.checkPointCopy.remove_seg >> 32),
(uint32)ControlFile.checkPointCopy.remove_seg);
printf(_("Time of latest checkpoint: %s\n"), ckpttime_str);
@ -265,6 +220,151 @@ int main(int argc, char* argv[])
printf(
_("Float8 argument passing: %s\n"), (ControlFile.float8ByVal ? _("by value") : _("by reference")));
printf(_("Database system TimeLine: %u\n"), ControlFile.timeline);
}
int main(int argc, char* argv[])
{
ControlFileData ControlFile;
int fd = -1;
bool display_all = true;
char ControlFilePath[MAXPGPATH];
char* DataDir = NULL;
char* socketpath = NULL;
int sret = 0;
int seekpos;
int option_value;
int option_index;
int display_id;
int ss_nodeid = MIN_INSTANCEID;
off_t ControlFileSize;
static struct option long_options[] = {{"enable-dss", no_argument, NULL, 1},
{"socketpath", required_argument, NULL, 2}};
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_controldata"));
progname = get_progname(argv[0]);
if (argc > 1) {
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) {
usage(progname);
exit_safely(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) {
#ifdef ENABLE_MULTIPLE_NODES
puts("pg_controldata (PostgreSQL) " PG_VERSION);
#else
puts("pg_controldata (openGauss) " PG_VERSION);
#endif
exit_safely(0);
}
}
while ((option_value = getopt_long(argc, argv, "I:V", long_options, &option_index)) != -1) {
switch (option_value) {
case 'I':
if (atoi(optarg) < MIN_INSTANCEID || atoi(optarg) > MAX_INSTANCEID) {
fprintf(stderr, _("%s: unexpected node id specified, valid range is %d - %d\n"),
progname, MIN_INSTANCEID, MAX_INSTANCEID);
exit_safely(1);
}
ss_nodeid = atoi(optarg);
display_all = false;
break;
case 1:
enable_dss = true;
break;
case 2:
enable_dss = true;
socketpath = strdup(optarg);
break;
default:
/* getopt_long already emitted a complaint */
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit_safely(1);
}
}
if (optind < argc) {
DataDir = argv[optind];
} else {
DataDir = getenv("PGDATA");
}
if (DataDir == NULL) {
fprintf(stderr, _("%s: no data directory specified\n"), progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit_safely(1);
}
check_env_value_c(DataDir);
if (enable_dss) {
if (socketpath == NULL) {
fprintf(stderr, _("%s: socketpath cannot be NULL when enable dss\n"), progname);
exit_safely(1);
}
if (DataDir[0] != '+') {
fprintf(stderr, _("%s: DATADIR should be specified when enable dss\n"), progname);
exit_safely(1);
}
}
// dss device init
if (dss_device_init(socketpath, enable_dss) != DSS_SUCCESS) {
fprintf(stderr, _("failed to init dss device\n"));
exit_safely(1);
}
if (enable_dss) {
// in shared storage mode, the cluster contains only one pg_control file
sret = snprintf_s(ControlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", DataDir);
} else {
sret = snprintf_s(ControlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/global/pg_control", DataDir);
}
securec_check_ss_c(sret, "\0", "\0");
fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0);
if (fd < 0) {
fprintf(
stderr, _("%s: could not open file \"%s\" for reading: %s\n"), progname, ControlFilePath, strerror(errno));
exit_safely(2);
}
if ((ControlFileSize = lseek(fd, 0, SEEK_END)) < 0) {
fprintf(stderr, _("%s: could not get \"%s\" size: %s\n"), progname, ControlFilePath, strerror(errno));
close(fd);
exit_safely(2);
}
display_id = ss_nodeid;
seekpos = (int)BLCKSZ * ss_nodeid;
if (seekpos >= ControlFileSize) {
fprintf(stderr, _("%s: cound not read beyond end of file \"%s\", file_size: %ld, instance_id: %d\n"),
progname, ControlFilePath, ControlFileSize, ss_nodeid);
close(fd);
exit_safely(2);
}
do {
if (lseek(fd, (off_t)seekpos, SEEK_SET) < 0) {
fprintf(stderr, _("%s: could not seek in \"%s\" to offset %d: %s\n"),
progname, ControlFilePath, seekpos, strerror(errno));
close(fd);
exit_safely(2);
}
if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) {
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), progname, ControlFilePath, strerror(errno));
close(fd);
exit_safely(2);
}
display_data(ControlFile, display_id);
seekpos += BLCKSZ;
display_id = display_id + 1;
} while (display_all && seekpos < ControlFileSize);
close(fd);
if (progname != NULL) {
free((char*)progname);
progname = NULL;

2
src/bin/pg_ctl/Makefile
View File

@ -16,7 +16,7 @@ subdir = src/bin/pg_ctl
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I$(libpq_srcdir) -I$(ZLIB_INCLUDE_PATH) $(CPPFLAGS) -DHAVE_LIBZ -DFRONTEND -I$(top_builddir)/src/bin/pg_rewind -I${top_builddir}/src/lib/page_compression
override CPPFLAGS := -I$(libpq_srcdir) -I$(ZLIB_INCLUDE_PATH) $(CPPFLAGS) -DHAVE_LIBZ -DFRONTEND -I$(top_builddir)/src/bin/pg_rewind -I${top_builddir}/src/lib/page_compression -I${top_builddir}/src/include
override LDFLAGS += -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH) -L${top_builddir}/src/lib/page_compression
ifeq ($(enable_lite_mode), no)
LIBS += -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss -lpagecompression -lzstd -llz4

8
src/bin/pg_ctl/backup.cpp
View File

@ -41,6 +41,7 @@
#include "backup.h"
#include "logging.h"
#include "tool_common.h"
#include "bin/elog.h"
#include "file_ops.h"
#include "catalog/catalog.h"
@ -52,6 +53,7 @@
#include "fetchmot.h"
#endif
/* Maximum number of digit in integer. Used to allocate memory to copy int to string */
#define MAX_INT_SIZE 20
/* set build receive timeout during master getting in backup mode */
@ -518,7 +520,7 @@ bool StartLogStreamer(
Assert(!XLogRecPtrIsInvalid(param->startptr));
/* Round off to even segment position */
param->startptr -= param->startptr % XLOG_SEG_SIZE;
param->startptr -= param->startptr % XLogSegSize;
#ifndef WIN32
/* Create our background pipe */
@ -2343,7 +2345,7 @@ static bool backup_dw_file(const char* target_dir)
char* unaligned_buf = NULL;
/* Delete the dw file, if it exists. */
rc = snprintf_s(dw_file_path, PATH_MAX, PATH_MAX - 1, "%s/%s", target_dir, OLD_DW_FILE_NAME);
rc = snprintf_s(dw_file_path, PATH_MAX, PATH_MAX - 1, "%s/%s", target_dir, T_OLD_DW_FILE_NAME);
securec_check_ss_c(rc, "\0", "\0");
if (realpath(dw_file_path, real_file_path) == NULL) {
if (real_file_path[0] == '\0') {
@ -2357,7 +2359,7 @@ static bool backup_dw_file(const char* target_dir)
securec_check_c(rc, "\0", "\0");
/* Delete the dw build file, if it exists. */
rc = snprintf_s(dw_file_path, PATH_MAX, PATH_MAX - 1, "%s/%s", target_dir, DW_BUILD_FILE_NAME);
rc = snprintf_s(dw_file_path, PATH_MAX, PATH_MAX - 1, "%s/%s", target_dir, T_DW_BUILD_FILE_NAME);
securec_check_ss_c(rc, "\0", "\0");
if (realpath(dw_file_path, real_file_path) == NULL) {
if (real_file_path[0] == '\0') {

3
src/bin/pg_ctl/pg_ctl.cpp
View File

@ -54,6 +54,7 @@
#include "fetch.h"
#include "common/fe_memutils.h"
#include "logging.h"
#include "tool_common.h"
#ifdef ENABLE_MOT
#include "fetchmot.h"
@ -80,6 +81,7 @@
#define static
#endif
/* PID can be negative for standalone backend */
typedef long pgpid_t;
@ -6720,6 +6722,7 @@ int main(int argc, char** argv)
do_wait = false;
}
initDataPathStruct(false);
SetConfigFilePath();
pg_host = getenv("PGHOST");

18
src/bin/pg_ctl/receivelog.cpp
View File

@ -92,8 +92,8 @@ static int open_walfile(XLogRecPtr startpoint, uint32 timeline, const char* base
MAXFNAMELEN - 1,
"%08X%08X%08X",
timeline,
(uint32)((startpoint / XLOG_SEG_SIZE) / XLogSegmentsPerXLogId),
(uint32)((startpoint / XLOG_SEG_SIZE) % XLogSegmentsPerXLogId));
(uint32)((startpoint / XLogSegSize) / XLogSegmentsPerXLogId),
(uint32)((startpoint / XLogSegSize) % XLogSegmentsPerXLogId));
securec_check_ss_c(nRet, "", "");
nRet = snprintf_s(fn, sizeof(fn), sizeof(fn) - 1, "%s/%s.partial", basedir, namebuf);
@ -122,11 +122,11 @@ static int open_walfile(XLogRecPtr startpoint, uint32 timeline, const char* base
f = -1;
return -1;
}
if (statbuf.st_size == XLogSegSize)
if (statbuf.st_size == (off_t)XLogSegSize)
return f; /* File is open and ready to use */
if (statbuf.st_size != 0) {
pg_log(PG_PRINT,
_("%s: WAL segment %s is %d bytes, should be 0 or %d\n"),
_("%s: WAL segment %s is %d bytes, should be 0 or %lu\n"),
progname,
Lrealpath,
(int)statbuf.st_size,
@ -263,7 +263,7 @@ static bool close_walfile(int walfile, const char* basedir, char* walname, bool
* Rename the .partial file only if we've completed writing the
* whole segment or segment_complete is true.
*/
if (currpos == XLOG_SEG_SIZE || segment_complete) {
if (currpos == (off_t)XLogSegSize || segment_complete) {
char oldfn[MAXPGPATH];
char newfn[MAXPGPATH];
errno_t nRet;
@ -398,7 +398,7 @@ static bool checkForReceiveTimeout(PGconn* conn)
static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const char* basedir, char* cur_wal_file,
uint32 wal_file_timeline, int& walfile, stream_stop_callback stream_stop, PGconn* conn)
{
int xlogoff = block_pos % XLOG_SEG_SIZE;
int xlogoff = block_pos % XLogSegSize;
int bytes_left = len;
int bytes_to_write = 0;
@ -430,8 +430,8 @@ static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const
while (bytes_left) {
/* If crossing a WAL boundary, only write up until we reach XLOG_SEG_SIZE. */
if (xlogoff + bytes_left > XLOG_SEG_SIZE)
bytes_to_write = XLOG_SEG_SIZE - xlogoff;
if (xlogoff + bytes_left > (int)XLogSegSize)
bytes_to_write = (int)XLogSegSize - xlogoff;
else
bytes_to_write = bytes_left;
@ -462,7 +462,7 @@ static int DoWALWrite(const char* wal_buf, int len, XLogRecPtr& block_pos, const
xlogoff += bytes_to_write;
/* Did we reach the end of a WAL segment? */
if (block_pos % XLOG_SEG_SIZE == 0) {
if (block_pos % XLogSegSize == 0) {
if (!close_walfile(walfile, basedir, cur_wal_file, false, block_pos)) {
suspendHeartBeatTimer();
/* Error message written in close_walfile() */

1
src/bin/pg_probackup/CMakeLists.txt
View File

@ -9,6 +9,7 @@ execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/bin/pg_basebackup/receivelog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/receivelog.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/bin/pg_basebackup/streamutil.cpp ${CMAKE_CURRENT_SOURCE_DIR}/streamutil.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/access/transam/xlogreader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/xlogreader.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/dss_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dss_adaptor.cpp
)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC)

5
src/bin/pg_probackup/Makefile
View File

@ -21,7 +21,8 @@ OBJS += backup.o catalog.o configure.o data.o \
OBJS += datapagemap.o receivelog.o streamutil.o \
xlogreader.o pg_lzcompress.o
OBJS += $(top_builddir)/src/lib/pgcommon/libpgcommon.a
OBJS += $(top_builddir)/src/lib/pgcommon/libpgcommon.a \
$(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o
EXTRA_CLEAN = datapagemap.cpp datapagemap.h \
receivelog.cpp receivelog.h streamutil.cpp streamutil.h \
@ -40,7 +41,7 @@ endif
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH)
# add page_compression so .h
LDFLAGS += -L../../lib/page_compression
PG_CPPFLAGS = -I../../lib/page_compression
PG_CPPFLAGS = -I../../lib/page_compression -I../../src/common/port
LIBS += -lpagecompression -lzstd
override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) -DHAVE_LIBZ
PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS}

130
src/bin/pg_probackup/backup.cpp
View File

@ -23,9 +23,12 @@
#include <time.h>
#include <unistd.h>
#include "tool_common.h"
#include "thread.h"
#include "file.h"
#include "common/fe_memutils.h"
#include "storage/file/fio_device.h"
/* list of dirs which will not to be backuped
it will be backuped up in external dirs */
@ -134,9 +137,9 @@ backup_stopbackup_callback(bool fatal, void *userdata)
}
}
static void run_backup_threads(char *external_prefix, char *database_path,
parray *prev_backup_filelist, parray *external_dirs,
PGNodeInfo *nodeInfo, XLogRecPtr prev_backup_start_lsn)
static void run_backup_threads(char *external_prefix, char *database_path, char *dssdata_path,
parray *prev_backup_filelist, parray *external_dirs,
PGNodeInfo *nodeInfo, XLogRecPtr prev_backup_start_lsn)
{
int i;
int nRet = 0;
@ -163,6 +166,8 @@ static void run_backup_threads(char *external_prefix, char *database_path,
securec_check_ss_c(nRet, "\0", "\0");
join_path_components(dirpath, temp, file->rel_path);
}
else if (is_dss_type(file->type))
join_path_components(dirpath, dssdata_path, file->rel_path);
else
join_path_components(dirpath, database_path, file->rel_path);
@ -205,6 +210,8 @@ static void run_backup_threads(char *external_prefix, char *database_path,
arg->nodeInfo = nodeInfo;
arg->from_root = instance_config.pgdata;
arg->to_root = database_path;
arg->src_dss = instance_config.dss.vgdata;
arg->dst_dss = dssdata_path;
arg->external_prefix = external_prefix;
arg->external_dirs = external_dirs;
arg->files_list = backup_files_list;
@ -412,16 +419,47 @@ static void calc_pgdata_bytes()
elog(INFO, "PGDATA size: %s", pretty_bytes);
}
static void add_xlog_files_into_backup_list(const char *database_path)
static void add_xlog_files_into_backup_list(const char *database_path, const char *dssdata_path,
int instance_id, bool enable_dss)
{
int i;
parray *xlog_files_list;
char pg_xlog_path[MAXPGPATH];
char wal_full_path[MAXPGPATH];
const char *parent_path;
/* Scan backup PG_XLOG_DIR */
xlog_files_list = parray_new();
join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
/* link dssdata's pg_xlog to database's pg_xlog */
if (enable_dss) {
char database_xlog[MAXPGPATH];
char dssdata_xlog[MAXPGPATH];
errno_t rc;
rc = snprintf_s(dssdata_xlog, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", dssdata_path, PG_XLOG_DIR, instance_id);
securec_check_ss_c(rc, "\0", "\0");
join_path_components(database_xlog, database_path, PG_XLOG_DIR);
/* dssdata_xlog is already exist, destory it and recreate */
if (rmdir(dssdata_xlog) != 0) {
elog(ERROR, "can not remove xlog dir \"%s\" : %s", dssdata_xlog, strerror(errno));
}
if (symlink(database_xlog, dssdata_xlog) < 0) {
elog(ERROR, "can not link dss xlog dir \"%s\" to database xlog dir \"%s\": %s", dssdata_xlog, database_xlog,
strerror(errno));
}
rc = strcpy_s(pg_xlog_path, MAXPGPATH, dssdata_xlog);
securec_check_c(rc, "\0", "\0");
parent_path = dssdata_path;
} else {
join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
parent_path = database_path;
}
dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, true, 0,
FIO_BACKUP_HOST);
@ -435,6 +473,11 @@ static void add_xlog_files_into_backup_list(const char *database_path)
if (!S_ISREG(file->mode))
continue;
/* refresh file type */
if (enable_dss) {
file->type = DEV_TYPE_DSS;
}
file->crc = pgFileGetCRC(wal_full_path, true, false);
file->write_size = file->size;
@ -444,7 +487,7 @@ static void add_xlog_files_into_backup_list(const char *database_path)
pg_free(file->rel_path);
/* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */
file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path));
file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, parent_path));
file->name = last_dir_separator(file->rel_path);
@ -461,7 +504,7 @@ static void add_xlog_files_into_backup_list(const char *database_path)
static void sync_files(parray *database_map, const char *database_path, parray *external_dirs,
const char *external_prefix, bool no_sync)
const char *dssdata_path, const char *external_prefix, bool no_sync)
{
time_t start_time, end_time;
char pretty_time[20];
@ -472,19 +515,28 @@ static void sync_files(parray *database_map, const char *database_path, parray *
if (current.from_replica && !exclusive_backup)
{
pgFile *pg_control = NULL;
char fullpath[MAXPGPATH];
for (unsigned int i = 0; i < parray_num(backup_files_list); i++)
{
pgFile *tmp_file = (pgFile *)parray_get(backup_files_list, (size_t)i);
if (tmp_file->external_dir_num == 0 &&
(strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0))
(strcmp(tmp_file->name, PG_XLOG_CONTROL_FILE) == 0))
{
pg_control = tmp_file;
break;
}
}
if (!pg_control)
elog(ERROR, "Failed to find file \"%s\" in backup filelist.", XLOG_CONTROL_FILE);
set_min_recovery_point(pg_control, database_path, current.stop_lsn);
if (!pg_control) {
elog(ERROR, "Failed to find file \"%s\" in backup filelist.", T_XLOG_CONTROL_FILE);
}
if (is_dss_type(pg_control->type)) {
join_path_components(fullpath, dssdata_path, pg_control->rel_path);
} else {
join_path_components(fullpath, database_path, pg_control->rel_path);
}
set_min_recovery_point(pg_control, fullpath, current.stop_lsn);
}
/* close and sync page header map */
@ -502,7 +554,7 @@ static void sync_files(parray *database_map, const char *database_path, parray *
/* Add archived xlog files into the list of files of this backup */
if (stream_wal)
{
add_xlog_files_into_backup_list(database_path);
add_xlog_files_into_backup_list(database_path, dssdata_path, instance_config.dss.instance_id, IsDssMode());
}
/* write database map to file and add it to control file */
@ -541,9 +593,7 @@ static void sync_files(parray *database_map, const char *database_path, parray *
continue;
/* construct fullpath */
if (file->external_dir_num == 0)
join_path_components(to_fullpath, database_path, file->rel_path);
else
if (file->external_dir_num != 0)
{
char external_dst[MAXPGPATH];
@ -551,6 +601,10 @@ static void sync_files(parray *database_map, const char *database_path, parray *
file->external_dir_num);
join_path_components(to_fullpath, external_dst, file->rel_path);
}
else if (is_dss_type(file->type))
join_path_components(to_fullpath, dssdata_path, file->rel_path);
else
join_path_components(to_fullpath, database_path, file->rel_path);
if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0)
elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno));
@ -572,6 +626,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
{
int i;
char database_path[MAXPGPATH];
char dssdata_path[MAXPGPATH];
char external_prefix[MAXPGPATH]; /* Temp value. Used as template */
char label[1024];
XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr;
@ -619,6 +674,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
pgBackupGetPath(&current, database_path, lengthof(database_path),
DATABASE_DIR);
pgBackupGetPath(&current, dssdata_path, lengthof(dssdata_path),
DSSDATA_DIR);
pgBackupGetPath(&current, external_prefix, lengthof(external_prefix),
EXTERNAL_DIR);
@ -639,6 +696,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
dir_list_file(backup_files_list, instance_config.pgdata,
true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST, backup_replslots);
/* some files are storage in dss server, list them */
if (IsDssMode()) {
dir_list_file(backup_files_list, instance_config.dss.vgdata,
true, true, false, backup_logs, true, 0, FIO_DSS_HOST);
}
/*
* Get database_map (name to oid) for use in partial restore feature.
* It's possible that we fail and database_map will be NULL.
@ -726,7 +789,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
/*
* Make directories before backup and setup threads at the same time
*/
run_backup_threads(external_prefix, database_path, prev_backup_filelist,
run_backup_threads(external_prefix, database_path, dssdata_path, prev_backup_filelist,
external_dirs, nodeInfo, prev_backup_start_lsn);
/* clean previous backup file list */
@ -739,7 +802,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
/* Notify end of backup */
pg_stop_backup(&current, backup_conn, nodeInfo);
sync_files(database_map, database_path, external_dirs, external_prefix, no_sync);
sync_files(database_map, database_path, external_dirs, dssdata_path, external_prefix, no_sync);
/* be paranoid about instance been from the past */
if (current.backup_mode != BACKUP_MODE_FULL &&
@ -874,10 +937,14 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
/* Initialize PGInfonode */
pgNodeInit(&nodeInfo);
/* vgname of dss is already checked in previous step */
if (!instance_config.pgdata)
elog(ERROR, "required parameter not specified: PGDATA "
"(-D, --pgdata)");
if (IsDssMode() && current.backup_mode != BACKUP_MODE_FULL)
elog(ERROR, "only support full backup when enable dss.");
/* Update backup status and other metainfo. */
current.status = BACKUP_STATUS_RUNNING;
current.start_time = start_time;
@ -889,6 +956,8 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
current.compress_alg = instance_config.compress_alg;
current.compress_level = instance_config.compress_level;
current.storage_type = IsDssMode() ? DEV_TYPE_DSS : DEV_TYPE_FILE;
/* Save list of external directories */
if (instance_config.external_dir_str &&
(pg_strcasecmp(instance_config.external_dir_str, "none") != 0))
@ -2054,12 +2123,7 @@ backup_files(void *arg)
}
/* construct destination filepath */
if (file->external_dir_num == 0)
{
join_path_components(from_fullpath, arguments->from_root, file->rel_path);
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
}
else
if (file->external_dir_num != 0)
{
char external_dst[MAXPGPATH];
char *external_path = (char *)parray_get(arguments->external_dirs,
@ -2072,6 +2136,16 @@ backup_files(void *arg)
join_path_components(to_fullpath, external_dst, file->rel_path);
join_path_components(from_fullpath, external_path, file->rel_path);
}
else if (is_dss_type(file->type))
{
join_path_components(from_fullpath, arguments->src_dss, file->rel_path);
join_path_components(to_fullpath, arguments->dst_dss, file->rel_path);
}
else
{
join_path_components(from_fullpath, arguments->from_root, file->rel_path);
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
}
/* Encountered some strange beast */
if (!S_ISREG(file->mode))
@ -2480,6 +2554,16 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn)
res = pgut_execute(backup_conn, query, 0, NULL);
/* Check that external directories do not contain dsspath */
for (i = 0; i < (int)parray_num(external_list); i++) {
char *external_path = (char *)parray_get(external_list, i);
if (is_dss_file(external_path))
elog(ERROR,
"External directory path (-E option) \"%s\" "
"contains dss path, which is not allow now",
external_path);
}
/* Check successfull execution of query */
if (!res)
elog(ERROR, "Failed to get list of tablespaces");

23
src/bin/pg_probackup/catalog.cpp
View File

@ -877,6 +877,16 @@ pgBackupCreateDir(pgBackup *backup)
backup->database_dir = (char *)pgut_malloc(MAXPGPATH);
join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR);
if (IsDssMode())
{
/* prepare dssdata_dir */
backup->dssdata_dir = (char *)pgut_malloc(MAXPGPATH);
join_path_components(backup->dssdata_dir, backup->root_dir, DSSDATA_DIR);
/* add into subdirs array, which will be create later */
parray_append(subdirs, pg_strdup(DSSDATA_DIR));
}
/* block header map */
init_header_map(backup);
@ -1907,6 +1917,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
if (backup->content_crc != 0)
fio_fprintf(out, "content-crc = %u\n", backup->content_crc);
fio_fprintf(out, "\n#Database Storage type\n");
fio_fprintf(out, "storage-type = %s\n", dev2str(backup->storage_type));
}
@ -2069,14 +2082,15 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
"\"dbOid\":\"%u\"",
"\"dbOid\":\"%u\", \"file_type\":\"%d\"",
file->rel_path, file->write_size, file->mode,
file->is_datafile ? 1 : 0,
file->is_cfs ? 1 : 0,
file->crc,
deparse_compress_alg(file->compress_alg),
file->external_dir_num,
file->dbOid);
file->dbOid,
(int)file->type);
securec_check_ss_c(nRet, "\0", "\0");
len = nRet;
@ -2195,6 +2209,7 @@ readBackupControlFile(const char *path)
char *compress_alg = NULL;
char *recovery_name = NULL;
int parsed_options;
char *storage_type = NULL;
errno_t rc = 0;
ConfigOption options[] =
@ -2229,6 +2244,7 @@ readBackupControlFile(const char *path)
{'s', 0, "note", &backup->note, SOURCE_FILE_STRICT},
{'s', 0, "recovery-name", &recovery_name, SOURCE_FILE_STRICT},
{'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT},
{'s', 0, "storage-type", &storage_type, SOURCE_FILE_STRICT},
{0}
};
@ -2327,6 +2343,9 @@ readBackupControlFile(const char *path)
if (compress_alg)
backup->compress_alg = parse_compress_alg(compress_alg);
if (storage_type)
backup->storage_type = str2dev(storage_type);
return backup;
}

45
src/bin/pg_probackup/configure.cpp
View File

@ -14,6 +14,7 @@
#include "configuration.h"
#include "json.h"
#include "catalog/pg_control.h"
static void assign_log_level_console(ConfigOption *opt, const char *arg);
@ -40,6 +41,7 @@ static void show_configure_json(ConfigOption *opt);
#define OPTION_RETENTION_GROUP "Retention parameters"
#define OPTION_COMPRESS_GROUP "Compression parameters"
#define OPTION_REMOTE_GROUP "Remote access parameters"
#define OPTION_DSS_GROUP "DSS connect parameters"
/*
* Short name should be non-printable ASCII character.
@ -220,6 +222,27 @@ ConfigOption instance_options[] =
&instance_config.remote.libpath, SOURCE_CMD, (OptionSource)0,
OPTION_REMOTE_GROUP, 0, option_get_value
},
/* DSS options */
{
'b', 232, "enable-dss",
&instance_config.dss.enable_dss, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
's', 233, "vgname",
&instance_config.dss.vgname, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
's', 234, "socketpath",
&instance_config.dss.socketpath, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
'i', 235, "instance-id",
&instance_config.dss.instance_id, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{ 0 }
};
@ -333,7 +356,7 @@ init_config(InstanceConfig *config, const char *instance_name)
#if PG_VERSION_NUM >= 110000
config->xlog_seg_size = 0;
#else
config->xlog_seg_size = XLOG_SEG_SIZE;
config->xlog_seg_size = XLogSegSize;
#endif
config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT;
@ -349,6 +372,9 @@ init_config(InstanceConfig *config, const char *instance_name)
config->compress_level = COMPRESS_LEVEL_DEFAULT;
config->remote.proto = (const char*)"ssh";
config->dss.enable_dss = false;
config->dss.instance_id = INVALID_INSTANCEID;
}
/*
@ -515,6 +541,23 @@ readInstanceConfigFile(const char *instance_name)
's', 231, "remote-libpath", &instance->remote.libpath, SOURCE_CMD, (OptionSource)0,
OPTION_REMOTE_GROUP, 0, option_get_value
},
/* DSS connect options */
{
'b', 232, "enable-dss", &instance->dss.enable_dss, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
's', 233, "vgname", &instance->dss.vgname, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
's', 234, "socketpath", &instance->dss.socketpath, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{
'i', 235, "instance-id", &instance->dss.instance_id, SOURCE_CMD, (OptionSource)0,
OPTION_DSS_GROUP, 0, option_get_value
},
{ 0 }
};

111
src/bin/pg_probackup/data.cpp
View File

@ -24,10 +24,12 @@
#include <zlib.h>
#endif
#include "tool_common.h"
#include "thread.h"
#include "common/fe_memutils.h"
#include "lz4.h"
#include "zstd.h"
#include "storage/file/fio_device.h"
/* Union to ease operations on relation pages */
typedef struct DataPage
@ -502,6 +504,10 @@ prepare_page(ConnectionArgs *conn_arg,
blknum, from_fullpath, read_len, BLCKSZ);
else
{
/* If it is in DSS mode, the validation is skipped */
if (IsDssMode())
return PageIsOk;
/* We have BLCKSZ of raw data, validate it */
rc = validate_one_page(page, absolute_blknum,
InvalidXLogRecPtr, page_st,
@ -800,10 +806,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
BackupMode backup_mode, time_t parent_backup_time,
bool missing_ok)
{
fio_location from_location = is_dss_file(from_fullpath) ? FIO_DSS_HOST : FIO_DB_HOST;
/* special treatment for global/pg_control */
if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0)
if (file->external_dir_num == 0 && strcmp(file->name, PG_XLOG_CONTROL_FILE) == 0)
{
copy_pgcontrol_file(from_fullpath, FIO_DB_HOST,
copy_pgcontrol_file(from_fullpath, from_location,
to_fullpath, FIO_BACKUP_HOST, file);
return;
}
@ -815,7 +822,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
file->mtime <= parent_backup_time)
{
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
file->crc = fio_get_crc32(from_fullpath, from_location, false);
/* ...and checksum is the same... */
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@ -825,8 +832,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
}
}
backup_non_data_file_internal(from_fullpath, FIO_DB_HOST,
to_fullpath, file, true);
backup_non_data_file_internal(from_fullpath, from_location, to_fullpath, file, true);
}
/*
@ -901,8 +907,16 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
* At this point we are sure, that something is going to be copied
* Open source file.
*/
join_path_components(from_root, backup->root_dir, DATABASE_DIR);
join_path_components(from_fullpath, from_root, tmp_file->rel_path);
if (is_dss_type(tmp_file->type))
{
join_path_components(from_root, backup->root_dir, DSSDATA_DIR);
join_path_components(from_fullpath, from_root, tmp_file->rel_path);
}
else
{
join_path_components(from_root, backup->root_dir, DATABASE_DIR);
join_path_components(from_fullpath, from_root, tmp_file->rel_path);
}
in = fopen(from_fullpath, PG_BINARY_R);
if (in == NULL)
@ -915,8 +929,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
/* get headers for this file */
if (use_headers && tmp_file->n_headers > 0)
headers = get_data_file_headers(&(backup->hdr_map), tmp_file,
parse_program_version(backup->program_version),
true);
parse_program_version(backup->program_version),
true);
if (use_headers && !headers && tmp_file->n_headers > 0)
elog(ERROR, "Failed to get page headers for file \"%s\"", from_fullpath);
@ -928,13 +942,13 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
* copy the file from backup.
*/
total_write_len += restore_data_file_internal(in, out, tmp_file,
parse_program_version(backup->program_version),
from_fullpath, to_fullpath, dest_file->n_blocks,
use_bitmap ? &(dest_file)->pagemap : NULL,
checksum_map, backup->checksum_version,
/* shiftmap can be used only if backup state precedes the shift */
backup->stop_lsn <= shift_lsn ? lsn_map : NULL,
headers);
parse_program_version(backup->program_version),
from_fullpath, to_fullpath, dest_file->n_blocks,
use_bitmap ? &(dest_file)->pagemap : NULL,
checksum_map, backup->checksum_version,
/* shiftmap can be used only if backup state precedes the shift */
backup->stop_lsn <= shift_lsn ? lsn_map : NULL,
headers);
if (fclose(in) != 0)
elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
@ -984,8 +998,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
* but should never happen in case of blocks from FULL backup.
*/
if (fio_fseek(out, cur_pos_out) < 0)
elog(ERROR, "Cannot seek block %u of \"%s\": %s",
blknum, to_fullpath, strerror(errno));
elog(ERROR, "Cannot seek block %u of \"%s\": %s",
blknum, to_fullpath, strerror(errno));
for (;;)
{
@ -1136,8 +1150,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
cur_pos_in != headers[n_hdr].pos)
{
if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0)
elog(ERROR, "Cannot seek to offset %u of \"%s\": %s",
headers[n_hdr].pos, from_fullpath, strerror(errno));
elog(ERROR, "Cannot seek to offset %u of \"%s\": %s",
headers[n_hdr].pos, from_fullpath, strerror(errno));
cur_pos_in = headers[n_hdr].pos;
}
@ -1224,7 +1238,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath)
{
size_t read_len = 0;
char *buf = (char *)pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */
char buf[STDIO_BUFSIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); /* 64kB buffer, need to be aligned */
/* copy content */
for (;;)
@ -1251,10 +1265,6 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
if (feof(in))
break;
}
pg_free(buf);
}
size_t
@ -1355,15 +1365,17 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
to_fullpath, strerror(errno));
}
if (tmp_file->external_dir_num == 0)
join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR);
else
if (tmp_file->external_dir_num != 0)
{
char external_prefix[MAXPGPATH];
join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR);
makeExternalDirPathByNum(from_root, external_prefix, tmp_file->external_dir_num);
}
else if (is_dss_type(tmp_file->type))
join_path_components(from_root, tmp_backup->root_dir, DSSDATA_DIR);
else
join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR);
join_path_components(from_fullpath, from_root, dest_file->rel_path);
@ -1424,10 +1436,8 @@ bool backup_remote_file(const char *from_fullpath, const char *to_fullpath, pgFi
* it is either small control file or already compressed cfs file.
*/
void
backup_non_data_file_internal(const char *from_fullpath,
fio_location from_location,
const char *to_fullpath, pgFile *file,
bool missing_ok)
backup_non_data_file_internal(const char *from_fullpath, fio_location from_location,
const char *to_fullpath, pgFile *file, bool missing_ok)
{
FILE *in = NULL;
FILE *out = NULL;
@ -1448,9 +1458,12 @@ backup_non_data_file_internal(const char *from_fullpath,
to_fullpath, strerror(errno));
/* update file permission */
if (chmod(to_fullpath, file->mode) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
strerror(errno));
if (!is_dss_file(from_fullpath))
{
if (chmod(to_fullpath, file->mode) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
strerror(errno));
}
/* backup remote file */
if (fio_is_remote(FIO_DB_HOST))
@ -1466,7 +1479,7 @@ backup_non_data_file_internal(const char *from_fullpath,
if (in == NULL)
{
/* maybe deleted, it's not error in case of backup */
if (errno == ENOENT)
if (is_file_delete(errno))
{
if (missing_ok)
{
@ -1648,7 +1661,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
* If file is not found, this is not en error.
* It could have been deleted by concurrent openGauss transaction.
*/
if (errno == ENOENT)
if (is_file_delete(errno))
{
elog(LOG, "File \"%s\" is not found", from_fullpath);
return true;
@ -1709,15 +1722,12 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
int n_hdr = -1;
off_t cur_pos_in = 0;
/* should not be possible */
Assert(!(backup_version >= 20400 && file->n_headers <= 0));
in = fopen(fullpath, PG_BINARY_R);
if (in == NULL)
elog(ERROR, "Cannot open file \"%s\": %s",
fullpath, strerror(errno));
elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno));
headers = get_data_file_headers(hdr_map, file, backup_version, false);
@ -1732,7 +1742,6 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
/* calc CRC of backup file */
INIT_FILE_CRC32(use_crc32c, crc);
/* read and validate pages one by one */
while (true)
{
@ -1837,10 +1846,10 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
const char *errormsg = NULL;
uncompressed_size = do_decompress(page.data, BLCKSZ,
compressed_page.data,
compressed_size,
file->compress_alg,
&errormsg);
compressed_page.data,
compressed_size,
file->compress_alg,
&errormsg);
if (uncompressed_size < 0 && errormsg != NULL)
{
elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s",
@ -1862,14 +1871,12 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
return false;
}
rc = validate_one_page(page.data,
file->segno * RELSEG_SIZE + blknum,
stop_lsn, &page_st, checksum_version);
rc = validate_one_page(page.data, file->segno * RELSEG_SIZE + blknum,
stop_lsn, &page_st, checksum_version);
}
else
rc = validate_one_page(compressed_page.data,
file->segno * RELSEG_SIZE + blknum,
stop_lsn, &page_st, checksum_version);
rc = validate_one_page(compressed_page.data, file->segno * RELSEG_SIZE + blknum,
stop_lsn, &page_st, checksum_version);
switch (rc)
{

119
src/bin/pg_probackup/dir.cpp
View File

@ -25,6 +25,7 @@
#include "configuration.h"
#include "common/fe_memutils.h"
#include "PageCompression.h"
#include "storage/file/fio_device.h"
/*
* The contents of these directories are removed or recreated during server
@ -126,6 +127,7 @@ static char check_in_tablespace(pgFile *file, bool in_tablespace);
static char check_db_dir(pgFile *file);
static char check_digit_file(pgFile *file);
static char check_nobackup_dir(pgFile *file);
static char check_in_dss(pgFile *file, int include_id);
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool follow_symlink, bool backup_logs,
bool skip_hidden, int external_dir_num, fio_location location,
@ -160,7 +162,7 @@ dir_create_dir(const char *dir, mode_t mode)
/* Create directory */
if (mkdir(dir, mode) == -1)
{
if (errno == EEXIST) /* already exist */
if (is_file_exist(errno)) /* already exist */
return 0;
elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno));
}
@ -179,13 +181,14 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
if (fio_stat(path, &st, follow_symlink, location) < 0)
{
/* file not found is not an error case */
if (errno == ENOENT)
if (is_file_delete(errno))
return NULL;
elog(ERROR, "cannot stat file \"%s\": %s", path,
strerror(errno));
}
file = pgFileInit(rel_path);
file->type = fio_device_type(path);
file->size = st.st_size;
file->mode = st.st_mode;
file->mtime = st.st_mtime;
@ -244,7 +247,7 @@ pgFileDelete(mode_t mode, const char *full_path)
{
if (rmdir(full_path) == -1)
{
if (errno == ENOENT)
if (is_file_delete(errno))
return;
else if (errno == ENOTDIR) /* could be symbolic link */
goto delete_file;
@ -258,7 +261,7 @@ pgFileDelete(mode_t mode, const char *full_path)
delete_file:
if (remove(full_path) == -1)
{
if (errno == ENOENT)
if (is_file_delete(errno))
return;
elog(ERROR, "Cannot remove file \"%s\": %s", full_path,
strerror(errno));
@ -274,7 +277,7 @@ pgFileDelete(mode_t mode, const char *full_path)
pg_crc32
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
FILE *fp = NULL;
pg_crc32 crc = 0;
char *buf;
size_t len = 0;
@ -285,7 +288,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
fp = fopen(file_path, PG_BINARY_R);
if (fp == NULL)
{
if (errno == ENOENT)
if (is_file_delete(errno))
{
if (missing_ok)
{
@ -294,8 +297,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
}
}
elog(ERROR, "Cannot open file \"%s\": %s",
file_path, strerror(errno));
elog(ERROR, "Cannot open file \"%s\": %s", file_path, strerror(errno));
}
/* disable stdio buffering */
@ -591,6 +593,7 @@ dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots)
int sscanf_res;
char ret;
bool in_tablespace = false;
char check_res;
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path);
@ -700,6 +703,12 @@ dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots)
return CHECK_FALSE;
}
/* skip other instance files in dss mode */
check_res = check_in_dss(file, instance_config.dss.instance_id);
if (check_res != CHECK_TRUE) {
return check_res;
}
ret = check_in_tablespace(file, in_tablespace);
if (ret != -1) {
return ret;
@ -708,6 +717,47 @@ dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots)
return check_db_dir(file);
}
static char check_in_dss(pgFile *file, int include_id)
{
char instance_id[MAX_INSTANCEID_LEN];
char top_path[MAXPGPATH];
errno_t rc = EOK;
int move = 0;
if (!is_dss_type(file->type)) {
return CHECK_TRUE;
}
/* step1 : skip other instance owner file or dir */
strlcpy(top_path, file->rel_path, sizeof(top_path));
get_top_path(top_path);
rc = snprintf_s(instance_id, sizeof(instance_id), sizeof(instance_id) - 1, "%d", include_id);
securec_check_ss_c(rc, "\0", "\0");
move = (int)strlen(top_path) - (int)strlen(instance_id);
if (move > 0 && move < MAXPGPATH && strcmp(top_path + move, instance_id) != 0) {
char tail = top_path[strlen(top_path) - 1];
/* Is this file or dir belongs to other instance? */
if (tail >= '0' && tail <= '9') {
return CHECK_FALSE;
}
}
/* step2: recheck dir is in the exclude list, include id will be considered */
if (S_ISDIR(file->mode)) {
for (int i = 0; pgdata_exclude_dir[i]; i++) {
int len = (int)strlen(pgdata_exclude_dir[i]);
if (strncmp(top_path, pgdata_exclude_dir[i], len) == 0 &&
strcmp(top_path + len, instance_id) == 0) {
return CHECK_EXCLUDE_FALSE;
}
}
}
return CHECK_TRUE;
}
static char check_in_tablespace(pgFile *file, bool in_tablespace)
{
if (in_tablespace)
@ -741,11 +791,15 @@ if (in_tablespace)
{
file->tblspcOid = DEFAULTTABLESPACE_OID;
int ret = sscanf_s(file->rel_path, "base/%u/", &(file->dbOid));
if (ret == -1)
elog(INFO, "Cannot parse path \"%s\"", file->rel_path);
if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0)
file->is_database = true;
/* skip "base" itself */
if (strcmp(file->name, "base") != 0)
{
int ret = sscanf_s(file->rel_path, "base/%u/", &(file->dbOid));
if (ret == -1)
elog(INFO, "Cannot parse path \"%s\"", file->rel_path);
if (S_ISDIR(file->mode))
file->is_database = true;
}
}
return -1;
@ -949,6 +1003,12 @@ bool SkipSomeDirFile(pgFile *file, struct dirent *dent, bool skipHidden)
if (S_ISDIR(file->mode) && (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) {
return false;
}
/* Skip recycle in dss mode */
if (is_dss_type(file->type) && strcmp(dent->d_name, ".recycle") == 0)
{
elog(WARNING, "Skip .recycle");
return false;
}
/* skip hidden files and directories */
if (skipHidden && file->name[0] == '.') {
elog(WARNING, "Skip hidden file: '%s'", file->name);
@ -958,7 +1018,7 @@ bool SkipSomeDirFile(pgFile *file, struct dirent *dent, bool skipHidden)
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) {
if (!S_ISDIR(file->mode) && !S_ISREG(file->mode) && !S_ISLNK(file->mode)) {
elog(WARNING, "Skip '%s': unexpected file format", file->name);
return false;
}
@ -986,7 +1046,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
dir = fio_opendir(parent_dir, location);
if (dir == NULL)
{
if (errno == ENOENT)
if (is_file_delete(errno))
{
/* Maybe the directory was removed */
return;
@ -1046,7 +1106,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
backup_logs, skip_hidden, external_dir_num, location, backup_replslots);
}
if (errno && errno != ENOENT)
if (errno && !is_file_delete(errno))
{
int errno_tmp = errno;
fio_closedir(dir);
@ -1233,7 +1293,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
* original directory created as symlink to it.
*/
elog(LOG, "Restore directories and symlinks...");
elog(LOG, "Restore directories and symlinks... in %s", data_dir);
/* create directories */
for (i = 0; i < parray_num(dest_files); i++)
@ -1241,6 +1301,10 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
char parent_dir[MAXPGPATH];
pgFile *dir = (pgFile *) parray_get(dest_files, i);
/* skip undesirable type */
if (is_dss_type(dir->type) != fio_is_dss(location))
continue;
if (!S_ISDIR(dir->mode))
continue;
@ -1248,6 +1312,12 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
if (dir->external_dir_num != 0)
continue;
if (is_dss_type(dir->type) && is_ss_xlog(dir->rel_path)) {
ss_createdir(dir->rel_path, instance_config.dss.vgdata,
instance_config.dss.vglog);
continue;
}
/* tablespace_map exists */
if (links)
{
@ -1702,7 +1772,8 @@ dir_read_file_list(const char *root, const char *external_prefix,
dbOid, /* used for partial restore */
hdr_crc,
hdr_off,
hdr_size;
hdr_size,
file_type;
pgFile *file = nullptr;
COMP_FILE_CRC32(true, content_crc, buf, strlen(buf));
@ -1716,6 +1787,7 @@ dir_read_file_list(const char *root, const char *external_prefix,
get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false);
get_control_value(buf, "dbOid", NULL, &dbOid, false);
get_control_value(buf, "file_type", NULL, &file_type, true);
file = pgFileInit(path);
file->write_size = (int64) write_size;
@ -1729,6 +1801,7 @@ dir_read_file_list(const char *root, const char *external_prefix,
file->compress_alg = parse_compress_alg(compress_alg_string);
file->external_dir_num = external_dir_num;
file->dbOid = dbOid ? dbOid : 0;
file->type = (device_type_t)file_type;
/*
* Optional fields
@ -1805,7 +1878,7 @@ dir_is_empty(const char *path, fio_location location)
if (dir == NULL)
{
/* Directory in path doesn't exist */
if (errno == ENOENT)
if (is_file_delete(errno))
return true;
elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno));
}
@ -1818,6 +1891,10 @@ dir_is_empty(const char *path, fio_location location)
strcmp(dir_ent->d_name, "..") == 0)
continue;
/* Skip recycle in dss mode */
if (fio_is_dss(location) && strcmp(dir_ent->d_name, ".recycle") == 0)
continue;
/* Directory is not empty */
fio_closedir(dir);
return false;
@ -1838,7 +1915,7 @@ fileExists(const char *path, fio_location location)
{
struct stat buf;
if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT)
if (fio_stat(path, &buf, true, location) == -1 && is_file_delete(errno))
return false;
else if (!S_ISREG(buf.st_mode))
return false;
@ -1846,7 +1923,7 @@ fileExists(const char *path, fio_location location)
return true;
}
size_t
off_t
pgFileSize(const char *path)
{
struct stat buf;

5
src/bin/pg_probackup/fetch.cpp
View File

@ -27,16 +27,13 @@
*
*/
char *
slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fio_location location)
slurpFile(const char *fullpath, size_t *filesize, bool safe, fio_location location)
{
int fd;
char *buffer;
struct stat statbuf;
char fullpath[MAXPGPATH];
int len;
join_path_components(fullpath, datadir, path);
if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1)
{
if (safe)

52
src/bin/pg_probackup/file.cpp
View File

@ -15,6 +15,7 @@
#include "file.h"
#include "storage/checksum.h"
#include "storage/file/fio_device.h"
#include "common/fe_memutils.h"
#define PRINTF_BUF_SIZE 1024
@ -211,6 +212,12 @@ bool fio_is_remote_simple(fio_location location)
return is_remote;
}
/* Check if specified location is for current node */
bool fio_is_dss(fio_location location)
{
return location == FIO_DSS_HOST;
}
/* Try to read specified amount of bytes unless error or EOF are encountered */
ssize_t fio_read_all(int fd, void* buf, size_t size)
{
@ -546,7 +553,7 @@ int fio_fprintf(FILE* f, char const* format, ...)
return rc;
}
/* Flush stream data (does nothing for remote file) */
/* Flush stream data (does nothing for remote file and dss file) */
int fio_fflush(FILE* f)
{
int rc = 0;
@ -697,9 +704,22 @@ int fio_seek(int fd, off_t offs)
/* Write data to stdio file */
size_t fio_fwrite(FILE* f, void const* buf, size_t size)
{
return fio_is_remote_file(f)
? fio_write(fio_fileno(f), buf, size)
: fwrite(buf, 1, size, f);
if (fio_is_remote_file(f))
{
return (size_t)fio_write(fio_fileno(f), buf, size);
}
else if (is_dss_file_dec(f))
{
/* size must be multiples of ALIGNOF_BUFFER in dss */
char align_buf[size] __attribute__((__aligned__(ALIGNOF_BUFFER))); /* need to be aligned */
errno_t rc = memcpy_s(align_buf, size, buf, size);
securec_check_c(rc, "\0", "\0");
return dss_fwrite_file(align_buf, 1, size, f);
}
else
{
return fwrite(buf, 1, size, f);
}
}
/* Write data to the file */
@ -728,10 +748,7 @@ int32
fio_decompress(void* dst, void const* src, size_t size, int compress_alg)
{
const char *errormsg = NULL;
int32 uncompressed_size = do_decompress(dst, BLCKSZ,
src,
size,
(CompressAlg)compress_alg, &errormsg);
int32 uncompressed_size = do_decompress(dst, BLCKSZ, src, size, (CompressAlg)compress_alg, &errormsg);
if (uncompressed_size < 0 && errormsg != NULL)
{
elog(WARNING, "An error occured during decompressing block: %s", errormsg);
@ -766,12 +783,13 @@ ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compres
}
else
{
/* operate is same in local mode and dss mode */
char uncompressed_buf[BLCKSZ];
int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg);
return (uncompressed_size < 0)
? uncompressed_size
: fwrite(uncompressed_buf, 1, uncompressed_size, f);
: fio_fwrite(f, uncompressed_buf, uncompressed_size);
}
}
@ -983,6 +1001,11 @@ int fio_sync(char const* path, fio_location location)
return 0;
}
else if (is_dss_file(path))
{
/* nothing to do in dss mode, data are already sync to disk */
return 0;
}
else
{
int fd;
@ -1027,7 +1050,7 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decomp
else
{
#ifdef HAVE_LIBZ
if (decompress)
if (decompress && !IsDssMode())
return pgFileGetCRCgz(file_path, true, true);
else
#endif
@ -1080,6 +1103,7 @@ int fio_mkdir(const char* path, int mode, fio_location location)
}
else
{
/* operate is same in local mode and dss mode */
return dir_create_dir(path, mode);
}
}
@ -1113,7 +1137,7 @@ int fio_chmod(char const* path, int mode, fio_location location)
*/
static void fio_load_file(int out, char const* path)
{
int fd = open(path, O_RDONLY);
int fd = open(path, O_RDONLY, 0);
fio_header hdr;
void* buf = NULL;
@ -2098,8 +2122,8 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version,
}
else
{
lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks,
shift_lsn, segmentno);
/* operate is same in local mode and dss mode */
lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, shift_lsn, segmentno);
}
return lsn_map;
@ -2152,6 +2176,7 @@ pid_t fio_check_postmaster(const char *pgdata, fio_location location)
return hdr.arg;
}
else
/* operate is same in local mode and dss mode */
return check_postmaster(pgdata);
}
@ -2188,6 +2213,7 @@ fio_delete(mode_t mode, const char *fullpath, fio_location location)
}
else
/* operate is same in local mode and dss mode */
pgFileDelete(mode, fullpath);
}

4
src/bin/pg_probackup/file.h
View File

@ -74,7 +74,8 @@ typedef enum
FIO_LOCAL_HOST, /* data is locate at local host */
FIO_DB_HOST, /* data is located at Postgres server host */
FIO_BACKUP_HOST, /* data is located at backup host */
FIO_REMOTE_HOST /* date is located at remote host */
FIO_REMOTE_HOST, /* date is located at remote host */
FIO_DSS_HOST /* data is located at dss storage, it can be visit in local host */
} fio_location;
#define FIO_FDMAX 64
@ -101,6 +102,7 @@ extern __thread int fio_stdin;
/* Check if FILE handle is local or remote (created by FIO) */
#define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX)
extern bool fio_is_dss(fio_location location);
extern ssize_t fio_read_all(int fd, void* buf, size_t size);
extern ssize_t fio_write_all(int fd, void const* buf, size_t size);
extern void fio_redirect(int in, int out, int err);

2
src/bin/pg_probackup/file_gz.cpp
View File

@ -158,7 +158,7 @@ fio_gzread(gzFile f, void *buf, unsigned size)
gz->strm.next_in = gz->buf;
}
rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in,
gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in);
gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in);
if (rc > 0)
{
gz->strm.avail_in += rc;

46
src/bin/pg_probackup/help.cpp
View File

@ -86,6 +86,8 @@ void help_pg_probackup(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s del-instance -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
@ -111,6 +113,8 @@ void help_pg_probackup(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s set-backup -B backup-path --instance=instance_name -i backup-id\n"), PROGRAM_NAME);
@ -153,6 +157,8 @@ void help_pg_probackup(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--ttl=interval] [--expire-time=time]\n"));
printf(_(" [--backup-pg-replslot]\n"));
printf(_(" [--help]\n"));
@ -169,6 +175,8 @@ void help_pg_probackup(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--log-level-console=log-level-console]\n"));
printf(_(" [--log-level-file=log-level-file]\n"));
printf(_(" [--log-filename=log-filename]\n"));
@ -237,6 +245,9 @@ static void help_add_instance(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n"));
@ -255,7 +266,13 @@ static void help_add_instance(void)
printf(_(" --remote-user=username user name for ssh connection (default: current user)\n"));
printf(_(" --remote-libpath=libpath library path on remote host\n"));
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n"));
printf(_("\n DSS options:\n"));
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" --vgname=\"vgdata,vglog\" name of dss volume group\n"));
printf(_(" --instance-id=instance_id instance id of backup node\n"));
printf(_(" --socketpath=socketpath dss connect socket file path\n\n"));
}
static void help_del_instance(void)
@ -288,6 +305,9 @@ static void help_set_config(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" --instance=instance_name name of the instance\n"));
@ -348,7 +368,13 @@ static void help_set_config(void)
printf(_(" --remote-user=username user name for ssh connection (default: current user)\n"));
printf(_(" --remote-libpath=libpath library path on remote host\n"));
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n"));
printf(_("\n DSS options:\n"));
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" --vgname=\"vgdata,vglog\" name of dss volume group\n"));
printf(_(" --instance-id=instance_id instance id of backup node\n"));
printf(_(" --socketpath=socketpath dss connect socket file path\n\n"));
}
static void help_set_backup(void)
@ -420,6 +446,8 @@ static void help_backup(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--ttl=interval] [--expire-time=time]\n\n"));
printf(_(" [--backup-pg-replslot]\n"));
@ -507,6 +535,12 @@ static void help_backup(void)
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n"));
printf(_("\n DSS options:\n"));
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" --vgname=\"vgdata,vglog\" name of dss volume group\n"));
printf(_(" --instance-id=instance_id instance id of backup node\n"));
printf(_(" --socketpath=socketpath dss connect socket file path\n"));
printf(_("\n Pinning options:\n"));
printf(_(" --ttl=interval pin backup for specified amount of time; 0 unpin\n"));
printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n"));
@ -529,6 +563,8 @@ static void help_restore(void)
printf(_(" [--remote-path=path] [--remote-user=username]\n"));
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--enable-dss] [--instance-id=instance_id]\n"));
printf(_(" [--vgname=\"vgdata,vglog\"] [--socketpath=socketpath]\n"));
printf(_(" [--log-level-console=log-level-console]\n"));
printf(_(" [--log-level-file=log-level-file]\n"));
printf(_(" [--log-filename=log-filename]\n"));
@ -577,6 +613,12 @@ static void help_restore(void)
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n"));
printf(_("\n DSS options:\n"));
printf(_(" --enable-dss enable shared storage mode\n"));
printf(_(" --vgname=\"vgdata,vglog\" name of dss volume group\n"));
printf(_(" --instance-id=instance_id instance id of backup node\n"));
printf(_(" --socketpath=socketpath dss connect socket file path\n"));
printf(_("\n Logging options:\n"));
printf(_(" --log-level-console=log-level-console\n"));
printf(_(" level for console logging (default: info)\n"));

2
src/bin/pg_probackup/init.cpp
View File

@ -125,7 +125,7 @@ do_add_instance(InstanceConfig *instance)
config_set_opt(instance_options, &instance_config.remote.ssh_config,
SOURCE_DEFAULT);
/* pgdata was set through command line */
/* pgdata and vgname were set through command line */
do_set_config(true);
elog(INFO, "Instance '%s' successfully inited", instance_name);

3
src/bin/pg_probackup/merge.cpp
View File

@ -15,6 +15,7 @@
#include "thread.h"
#include "common/fe_memutils.h"
#include "storage/file/fio_device.h"
typedef struct
{
@ -1393,7 +1394,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
if (out == NULL)
elog(ERROR, "Cannot open merge target file \"%s\": %s",
to_fullpath_tmp1, strerror(errno));
setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE);
setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE);
/* restore file into temp file */
restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1,

4
src/bin/pg_probackup/parsexlog.cpp
View File

@ -1123,7 +1123,7 @@ static int read_requested_page(XLogReaderData *reader_data, char *readBuf,
}
}
#ifdef HAVE_LIBZ
else
else if (!IsDssMode())
{
if (fio_gzseek(reader_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1)
{
@ -1733,7 +1733,7 @@ CleanupXLogPageRead(XLogReaderState *xlogreader)
reader_data->xlogfile = -1;
}
#ifdef HAVE_LIBZ
else if (reader_data->gz_xlogfile != NULL)
else if (reader_data->gz_xlogfile != NULL && !IsDssMode())
{
fio_gzclose(reader_data->gz_xlogfile);
reader_data->gz_xlogfile = NULL;

84
src/bin/pg_probackup/pg_probackup.cpp
View File

@ -17,10 +17,13 @@
#include <sys/stat.h>
#include "tool_common.h"
#include "configuration.h"
#include "thread.h"
#include <time.h>
#include "common/fe_memutils.h"
#include "storage/file/fio_device.h"
#include "storage/dss/dss_adaptor.h"
const char *PROGRAM_NAME = NULL; /* PROGRAM_NAME_FULL without .exe suffix
* if any */
@ -149,6 +152,7 @@ static void opt_backup_mode(ConfigOption *opt, const char *arg);
static void opt_show_format(ConfigOption *opt, const char *arg);
static void compress_init(void);
static void dss_init(void);
/*
* Short name should be non-printable ASCII character.
@ -755,7 +759,7 @@ int main(int argc, char *argv[])
/* Initialize logger */
init_logger(backup_path, &instance_config.logger);
/* command was initialized for a few commands */
if (command)
{
@ -782,6 +786,13 @@ int main(int argc, char *argv[])
!is_absolute_path(instance_config.pgdata))
elog(ERROR, "-D, --pgdata must be an absolute path");
/* prepare pgdata of g_datadir struct */
if (instance_config.pgdata != NULL)
{
errno_t rc = strcpy_s(g_datadir.pg_data, strlen(instance_config.pgdata) + 1, instance_config.pgdata);
securec_check_c(rc, "\0", "\0");
}
#if PG_VERSION_NUM >= 110000
/* Check xlog-seg-size option */
if (instance_name &&
@ -814,6 +825,10 @@ int main(int argc, char *argv[])
/* compress_init */
compress_init();
dss_init();
initDataPathStruct(IsDssMode());
/* do actual operation */
return do_actual_operate();
}
@ -909,3 +924,70 @@ compress_init(void)
elog(ERROR, "Multithread backup does not support pglz compression");
}
}
static void dss_init(void)
{
if (instance_config.dss.enable_dss) {
/* skip in some special backup modes */
if (backup_subcmd == DELETE_CMD || backup_subcmd == DELETE_INSTANCE_CMD) {
return;
}
/* register for dssapi */
if (dss_device_init(instance_config.dss.socketpath, instance_config.dss.enable_dss) != DSS_SUCCESS) {
elog(ERROR, "fail to init dss device");
return;
}
if (IsSshProtocol()) {
elog(ERROR, "Remote operations on dss mode are not supported");
}
parse_vgname_args(instance_config.dss.vgname);
/* Check dss connect */
if (!dss_exist_dir(instance_config.dss.vgdata)) {
elog(ERROR, "Could not connect dssserver, vgdata: \"%s\", socketpath: \"%s\", check and retry later.",
instance_config.dss.vgdata, instance_config.dss.socketpath);
}
if (strlen(instance_config.dss.vglog) && !dss_exist_dir(instance_config.dss.vglog)) {
elog(ERROR, "Could not connect dssserver, vglog: \"%s\", socketpath: \"%s\", check and retry later.",
instance_config.dss.vglog, instance_config.dss.socketpath);
}
/* Check backup instance id in shared storage mode */
int id = instance_config.dss.instance_id;
if (id < MIN_INSTANCEID || id > MAX_INSTANCEID) {
elog(ERROR, "Instance id must be specified in dss mode, valid range is %d - %d.",
MIN_INSTANCEID, MAX_INSTANCEID);
}
if (backup_subcmd != RESTORE_CMD) {
off_t size = 0;
char xlog_control_path[MAXPGPATH];
join_path_components(xlog_control_path, instance_config.dss.vgdata, PG_XLOG_CONTROL_FILE);
if ((size = dss_get_file_size(xlog_control_path)) == INVALID_DEVICE_SIZE) {
elog(ERROR, "Could not get \"%s\" size: %s", xlog_control_path, strerror(errno));
}
if (size < (off_t)BLCKSZ * id) {
elog(ERROR, "Cound not read beyond end of file \"%s\", file_size: %ld, instance_id: %d\n",
xlog_control_path, size, id);
}
}
/* Prepare some g_datadir parameters */
g_datadir.instance_id = id;
errno_t rc = strcpy_s(g_datadir.dss_data, strlen(instance_config.dss.vgdata) + 1, instance_config.dss.vgdata);
securec_check_c(rc, "\0", "\0");
rc = strcpy_s(g_datadir.dss_log, strlen(instance_config.dss.vglog) + 1, instance_config.dss.vglog);
securec_check_c(rc, "\0", "\0");
XLogSegmentSize = DSS_XLOG_SEG_SIZE;
instance_config.xlog_seg_size = DSS_XLOG_SEG_SIZE;
}
}

2
src/bin/pg_probackup/pg_probackupa.h
View File

@ -53,11 +53,13 @@ extern const char *PROGRAM_FULL_PATH;
/* Directory/File names */
#define DATABASE_DIR "database"
#define DSSDATA_DIR "dssdata"
#define BACKUPS_DIR "backups"
#define PG_XLOG_DIR "pg_xlog"
#define PG_LOG_DIR "pg_log"
#define PG_TBLSPC_DIR "pg_tblspc"
#define PG_GLOBAL_DIR "global"
#define PG_XLOG_CONTROL_FILE "pg_control"
#define BACKUP_CONTROL_FILE "backup.control"
#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf"
#define BACKUP_CATALOG_PID "backup.pid"

21
src/bin/pg_probackup/pg_probackupb.h
View File

@ -56,6 +56,7 @@ typedef struct pgFile_t
pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */
off_t hdr_off; /* offset in header map */
int hdr_size; /* offset in header map */
device_type_t type; /* file device type */
} pgFile;
typedef struct page_map_entry
@ -128,6 +129,16 @@ typedef struct ArchiveOptions
const char *user;
} ArchiveOptions;
typedef struct DssOptions
{
bool enable_dss;
int instance_id;
const char *vgname;
char *vglog;
char *vgdata;
char *socketpath;
} DssOptions;
/*
* An instance configuration. It can be stored in a configuration file or passed
* from command line.
@ -168,6 +179,9 @@ typedef struct InstanceConfig
/* Archive description */
ArchiveOptions archive;
/* DSS conntct parameters */
DssOptions dss;
} InstanceConfig;
extern ConfigOption instance_options[];
@ -268,6 +282,8 @@ struct pgBackup
backup_path/instance_name/backup_id */
char *database_dir; /* Full path to directory with data files:
backup_path/instance_name/backup_id/database */
char *dssdata_dir; /* Full path to directory with dss data files:
backup_path/instance_name/backup_id/database/dssdata */
parray *files; /* list of files belonging to this backup
* must be populated explicitly */
char *note;
@ -278,6 +294,9 @@ struct pgBackup
/* map used for access to page headers */
HeaderMap hdr_map;
/* device type */
device_type_t storage_type;
};
/* Recovery target for restore and validate subcommands */
@ -334,6 +353,8 @@ typedef struct
const char *from_root;
const char *to_root;
const char *src_dss;
const char *dst_dss;
const char *external_prefix;
parray *files_list;

21
src/bin/pg_probackup/pg_probackupc.h
View File

@ -26,6 +26,7 @@
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".gz.part") == 0)
#define IsDssMode() (instance_config.dss.enable_dss == true)
#define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0)
/* directory options */
@ -145,8 +146,7 @@ extern int do_delete_instance(void);
extern void do_delete_status(InstanceConfig *instance_config, const char *status);
/* in fetch.c */
extern char *slurpFile(const char *datadir,
const char *path,
extern char *slurpFile(const char *fullpath,
size_t *filesize,
bool safe,
fio_location location);
@ -280,7 +280,7 @@ extern int dir_create_dir(const char *path, mode_t mode);
extern bool dir_is_empty(const char *path, fio_location location);
extern bool fileExists(const char *path, fio_location location);
extern size_t pgFileSize(const char *path);
extern off_t pgFileSize(const char *path);
extern pgFile *pgFileNew(const char *path, const char *rel_path,
bool follow_symlink, int external_dir_num,
@ -327,9 +327,8 @@ extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint
const char *from_fullpath, const char *to_fullpath, int nblocks,
datapagemap_t *map, PageState *checksum_map, int checksum_version,
datapagemap_t *lsn_map, BackupPageHeader2 *headers);
extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
pgFile *dest_file, FILE *out, const char *to_fullpath,
bool already_exists);
extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out,
const char *to_fullpath, bool already_exists);
extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
const char *from_fullpath, const char *to_fullpath);
extern bool create_empty_file(fio_location from_location, const char *to_root,
@ -381,10 +380,14 @@ extern XLogRecPtr get_checkpoint_location(PGconn *conn);
extern uint64 get_system_identifier(const char *pgdata_path);
extern uint64 get_remote_system_identifier(PGconn *conn);
extern uint32 get_data_checksum_version(bool safe);
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
extern pg_crc32c get_pgcontrol_checksum(const char *fullpath);
extern uint32 get_xlog_seg_size(char *pgdata_path);
extern void get_redo(const char *pgdata_path, RedoParams *redo);
extern void set_min_recovery_point(pgFile *file, const char *backup_path,
extern void parse_vgname_args(const char* args);
extern bool is_ss_xlog(const char *ss_dir);
extern void ss_createdir(const char *ss_dir, const char *vgdata, const char *vglog);
extern char* xstrdup(const char* s);
extern void set_min_recovery_point(pgFile *file, const char *fullpath,
XLogRecPtr stop_backup_lsn);
extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
const char *to_fullpath, fio_location to_location, pgFile *file);
@ -392,6 +395,8 @@ extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_loc
extern void time2iso(char *buf, size_t len, time_t time);
extern const char *status2str(BackupStatus status);
extern BackupStatus str2status(const char *status);
extern const char *dev2str(device_type_t type);
extern device_type_t str2dev(const char *dev);
extern const char *base36enc(long unsigned int value);
extern char *base36enc_dup(long unsigned int value);
extern long unsigned int base36dec(const char *text);

92
src/bin/pg_probackup/restore.cpp
View File

@ -19,6 +19,7 @@
#include "thread.h"
#include "common/fe_memutils.h"
#include "catalog/catalog.h"
#include "storage/file/fio_device.h"
#define RESTORE_ARRAY_LEN 100
@ -31,6 +32,7 @@ typedef struct
parray *parent_chain;
bool skip_external_dirs;
const char *to_root;
const char *to_dss;
size_t restored_bytes;
bool use_bitmap;
IncrRestoreMode incremental_mode;
@ -55,8 +57,8 @@ static void set_orphan_status(parray *backups, pgBackup *parent_backup);
static void pg12_recovery_config(pgBackup *backup, bool add_include);
static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
pgRestoreParams *params,
const char *pgdata_path, bool no_sync);
pgRestoreParams *params, const char *pgdata_path,
const char *dssdata_path, bool no_sync);
static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
IncrRestoreMode incremental_mode);
static pgBackup *find_backup_range(parray *backups,
@ -83,6 +85,7 @@ static void threads_handle(pthread_t *threads,
parray *parent_chain,
pgRestoreParams *params,
const char *pgdata_path,
const char *dssdata_path,
bool use_bitmap,
size_t total_bytes);
static void sync_restored_files(parray *dest_files,
@ -159,6 +162,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
const char *action = (const char *)(params->is_restore ? "Restore":"Validate");
parray *parent_chain = NULL;
bool pgdata_is_empty = true;
bool dssdata_is_empty = true;
bool tblspaces_are_empty = true;
if (params->is_restore)
@ -166,6 +170,29 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
if (instance_config.pgdata == NULL)
elog(ERROR,
"required parameter not specified: PGDATA (-D, --pgdata)");
if (IsDssMode())
{
/* do not support increment restore in dss mode */
if (params->incremental_mode != INCR_NONE)
{
elog(ERROR, "Incremental restore is not support when enable dss");
}
if (!dir_is_empty(instance_config.dss.vgdata, FIO_DSS_HOST))
{
dssdata_is_empty = false;
elog(ERROR, "Restore destination is not empty: \"%s\"",
instance_config.dss.vgdata);
}
if (!dir_is_empty(instance_config.dss.vglog, FIO_DSS_HOST))
{
dssdata_is_empty = false;
elog(ERROR, "Restore destination is not empty: \"%s\"",
instance_config.dss.vglog);
}
}
/* Check if restore destination empty */
if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST))
{
@ -228,7 +255,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
{
check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty);
if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty)
if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty && dssdata_is_empty)
{
elog(INFO, "Destination directory and tablespace directories are empty, "
"disable incremental restore");
@ -393,8 +420,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
*/
if (params->is_restore)
{
restore_chain(dest_backup, parent_chain,
params, instance_config.pgdata, no_sync);
restore_chain(dest_backup, parent_chain, params, instance_config.pgdata,
instance_config.dss.vgdata, no_sync);
/* Create recovery.conf with given recovery target parameters */
create_recovery_conf(target_backup_id, rt, dest_backup, params);
@ -684,8 +711,8 @@ static XLogRecPtr determine_shift_lsn(pgBackup *dest_backup)
*/
void
restore_chain(pgBackup *dest_backup, parray *parent_chain,
pgRestoreParams *params,
const char *pgdata_path, bool no_sync)
pgRestoreParams *params, const char *pgdata_path,
const char *dssdata_path, bool no_sync)
{
int i;
char timestamp[100];
@ -780,6 +807,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
params->incremental_mode != INCR_NONE,
FIO_DB_HOST);
/* some file is in dssserver */
if (IsDssMode())
create_data_directories(dest_files, instance_config.dss.vgdata,
dest_backup->root_dir, true,
params->incremental_mode != INCR_NONE,
FIO_DSS_HOST);
/*
* Restore dest_backup external directories.
*/
@ -847,9 +881,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) *
num_threads);
threads_handle(threads, threads_args, dest_backup,
dest_files, pgdata_files, external_dirs, parent_chain,
params, pgdata_path, use_bitmap, total_bytes);
threads_handle(threads, threads_args, dest_backup, dest_files,
pgdata_files, external_dirs, parent_chain, params,
pgdata_path, dssdata_path, use_bitmap, total_bytes);
/* Close page header maps */
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
@ -1003,6 +1037,7 @@ static void threads_handle(pthread_t *threads,
parray *parent_chain,
pgRestoreParams *params,
const char *pgdata_path,
const char *dssdata_path,
bool use_bitmap,
size_t total_bytes)
{
@ -1036,6 +1071,7 @@ static void threads_handle(pthread_t *threads,
arg->parent_chain = parent_chain;
arg->skip_external_dirs = params->skip_external_dirs;
arg->to_root = pgdata_path;
arg->to_dss = dssdata_path;
arg->use_bitmap = use_bitmap;
arg->incremental_mode = params->incremental_mode;
arg->shift_lsn = params->shift_lsn;
@ -1102,6 +1138,10 @@ static void sync_restored_files(parray *dest_files,
params->skip_external_dirs)
continue;
/* skip dss files, which do not need sync */
if (is_dss_type(dest_file->type))
continue;
/* construct fullpath */
if (dest_file->external_dir_num == 0)
{
@ -1160,6 +1200,7 @@ restore_files(void *arg)
char to_fullpath[MAXPGPATH];
FILE *out = NULL;
char *out_buf = (char *)pgut_malloc(STDIO_BUFSIZE);
fio_location out_location;
restore_files_arg *arguments = (restore_files_arg *) arg;
@ -1208,14 +1249,18 @@ restore_files(void *arg)
continue;
/* set fullpath of destination file */
if (dest_file->external_dir_num == 0)
join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path);
else
if (dest_file->external_dir_num != 0)
{
char *external_path = (char *)parray_get(arguments->dest_external_dirs,
dest_file->external_dir_num - 1);
join_path_components(to_fullpath, external_path, dest_file->rel_path);
}
else if (is_dss_type(dest_file->type))
{
join_path_components(to_fullpath, arguments->to_dss, dest_file->rel_path);
}
else
join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path);
if (arguments->incremental_mode != INCR_NONE &&
parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc))
@ -1223,6 +1268,7 @@ restore_files(void *arg)
already_exists = true;
}
out_location = is_dss_type(dest_file->type) ? FIO_DSS_HOST : FIO_DB_HOST;
/*
* Handle incremental restore case for data files.
* If file is already exists in pgdata, then
@ -1237,13 +1283,13 @@ restore_files(void *arg)
{
lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version,
dest_file->n_blocks, arguments->shift_lsn,
dest_file->segno * RELSEG_SIZE, FIO_DB_HOST);
dest_file->segno * RELSEG_SIZE, out_location);
}
else if (arguments->incremental_mode == INCR_CHECKSUM)
{
checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version,
dest_file->n_blocks, arguments->dest_backup->stop_lsn,
dest_file->segno * RELSEG_SIZE, FIO_DB_HOST);
dest_file->segno * RELSEG_SIZE, out_location);
}
}
@ -1253,20 +1299,20 @@ restore_files(void *arg)
* if file do not exist
*/
if ((already_exists && dest_file->write_size == 0) || !already_exists)
out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST);
out = fio_fopen(to_fullpath, PG_BINARY_W, out_location);
/*
* If file already exists and dest size is not zero,
* then open it for reading and writing.
*/
else
out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST);
out = fio_fopen(to_fullpath, PG_BINARY_R "+", out_location);
if (out == NULL)
elog(ERROR, "Cannot open restore target file \"%s\": %s",
to_fullpath, strerror(errno));
/* update file permission */
if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1)
if (fio_chmod(to_fullpath, dest_file->mode, out_location) == -1)
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
strerror(errno));
@ -1283,8 +1329,8 @@ restore_files(void *arg)
if (!fio_is_remote_file(out))
setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE);
/* Destination file is data file */
arguments->restored_bytes += restore_data_file(arguments->parent_chain,
dest_file, out, to_fullpath,
arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file,
out, to_fullpath,
arguments->use_bitmap, checksum_map,
arguments->shift_lsn, lsn_map, true);
}
@ -1295,8 +1341,9 @@ restore_files(void *arg)
setvbuf(out, NULL, _IONBF, BUFSIZ);
/* Destination file is nonedata file */
arguments->restored_bytes += restore_non_data_file(arguments->parent_chain,
arguments->dest_backup, dest_file, out, to_fullpath,
already_exists);
arguments->dest_backup,
dest_file, out,
to_fullpath, already_exists);
}
done:
@ -2041,3 +2088,4 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
if (!success)
elog(ERROR, "Incremental restore is impossible");
}

38
src/bin/pg_probackup/show.cpp
View File

@ -36,6 +36,7 @@ typedef struct ShowBackendRow
char zratio[20];
char start_lsn[20];
char stop_lsn[20];
char type[20];
const char *status;
} ShowBackendRow;
@ -56,11 +57,12 @@ typedef struct ShowArchiveRow
static void show_instance_start(void);
static void show_instance_end(void);
static void show_instance(const char *instance_name, time_t requested_backup_id, bool show_name);
static void show_instance(InstanceConfig *instance, time_t requested_backup_id, bool show_name);
static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup);
static int show_backup(const char *instance_name, time_t requested_backup_id);
static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name);
static void show_instance_plain(const char *instance_name, device_type_t instance_type,
parray *backup_list, bool show_name);
static void show_instance_json(const char *instance_name, parray *backup_list);
static void show_instance_archive(InstanceConfig *instance);
@ -110,7 +112,7 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive
if (show_archive)
show_instance_archive(instance);
else
show_instance(instance->name, INVALID_BACKUP_ID, true);
show_instance(instance, INVALID_BACKUP_ID, true);
}
show_instance_end();
@ -124,17 +126,17 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive
requested_backup_id == INVALID_BACKUP_ID)
{
show_instance_start();
InstanceConfig *instance = readInstanceConfigFile(instance_name);
if (show_archive)
{
InstanceConfig *instance = readInstanceConfigFile(instance_name);
if (instance == NULL) {
return 0;
}
show_instance_archive(instance);
}
else
show_instance(instance_name, requested_backup_id, false);
show_instance(instance, requested_backup_id, false);
show_instance_end();
@ -324,14 +326,18 @@ show_instance_end(void)
* Show brief meta information about all backups in the backup instance.
*/
static void
show_instance(const char *instance_name, time_t requested_backup_id, bool show_name)
show_instance(InstanceConfig *instance, time_t requested_backup_id, bool show_name)
{
parray *backup_list;
const char *instance_name;
device_type_t instance_type;
instance_name = instance->name;
instance_type = instance->dss.enable_dss ? DEV_TYPE_DSS : DEV_TYPE_FILE;
backup_list = catalog_get_backup_list(instance_name, requested_backup_id);
if (show_format == SHOW_PLAIN)
show_instance_plain(instance_name, backup_list, show_name);
show_instance_plain(instance_name, instance_type, backup_list, show_name);
else if (show_format == SHOW_JSON)
show_instance_json(instance_name, backup_list);
else
@ -553,18 +559,18 @@ static void process_time(pgBackup *backup, ShowBackendRow *row)
* Show instance backups in plain format.
*/
static void
show_instance_plain(const char *instance_name, parray *backup_list, bool show_name)
show_instance_plain(const char *instance_name, device_type_t instance_type, parray *backup_list, bool show_name)
{
#define SHOW_FIELDS_COUNT 14
#define SHOW_FIELDS_COUNT 15
int i;
const char *names[SHOW_FIELDS_COUNT] =
{ "Instance", "Version", "ID", "Recovery Time",
"Mode", "WAL Mode", "TLI", "Time", "Data", "WAL",
"Zratio", "Start LSN", "Stop LSN", "Status" };
"Zratio", "Start LSN", "Stop LSN", "Type", "Status" };
const char *field_formats[SHOW_FIELDS_COUNT] =
{ " %-*s ", " %-*s ", " %-*s ", " %-*s ",
" %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", " %*s ",
" %*s ", " %-*s ", " %-*s ", " %-*s "};
" %*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s "};
uint32 widths[SHOW_FIELDS_COUNT];
uint32 widths_sum = 0;
ShowBackendRow *rows = NULL;
@ -690,6 +696,12 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
widths[cur] = Max(widths[cur], strlen(row->stop_lsn));
cur++;
/* Type (FILE OR DSS) */
rc = snprintf_s(row->type, lengthof(row->type), lengthof(row->type) - 1, "%s", dev2str(instance_type));
securec_check_ss_c(rc, "\0", "\0");
widths[cur] = Max(widths[cur], (uint32)strlen(row->type));
cur++;
/* Status */
row->status = status2str(backup->status);
widths[cur] = Max(widths[cur], strlen(row->status));
@ -778,6 +790,10 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na
row->stop_lsn);
cur++;
appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
row->type);
cur++;
appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur],
row->status);
cur++;

304
src/bin/pg_probackup/util.cpp
View File

@ -19,7 +19,9 @@
#include <unistd.h>
#include <sys/stat.h>
#include "tool_common.h"
#include "common/fe_memutils.h"
#include "storage/file/fio_device.h"
static const char *statusName[] =
{
@ -36,6 +38,14 @@ static const char *statusName[] =
"CORRUPT"
};
static const char *devTypeName[] =
{
"FILE",
"DSS",
"UNKNOWN",
"UNKNOWN"
};
uint32 NUM_65536 = 65536;
uint32 NUM_10000 = 10000;
@ -84,12 +94,10 @@ static void
checkControlFile(ControlFileData *ControlFile)
{
pg_crc32c crc;
/* Calculate CRC */
INIT_CRC32C(crc);
COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
/* Then compare it */
if (!EQ_CRC32C(crc, ControlFile->crc))
elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n"
@ -104,21 +112,75 @@ checkControlFile(ControlFileData *ControlFile)
"the PostgreSQL installation would be incompatible with this data directory.");
}
static void checkSSControlFile(ControlFileData* ControlFile, char* last, size_t size)
{
pg_crc32c crc;
/* Calculate CRC */
INIT_CRC32C(crc);
COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc));
COMP_CRC32C(crc, last, size);
FIN_CRC32C(crc);
ControlFile->crc = crc;
if ((ControlFile->pg_control_version % NUM_65536 == 0 || ControlFile->pg_control_version % NUM_65536 > NUM_10000) &&
ControlFile->pg_control_version / NUM_65536 != 0)
elog(ERROR, "possible byte ordering mismatch\n"
"The byte ordering used to store the pg_control file might not match the one\n"
"used by this program. In that case the results below would be incorrect, and\n"
"the PostgreSQL installation would be incompatible with this data directory.");
}
/*
* Verify control file contents in the buffer src, and copy it to *ControlFile.
*/
static void
digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
{
if (size != PG_CONTROL_SIZE)
elog(ERROR, "unexpected control file size %d, expected %d",
(int) size, PG_CONTROL_SIZE);
errno_t rc;
char* oldSrc = src;
char* tmpDssSrc;
size_t clearSize = 0;
bool dssMode = IsDssMode();
int instanceId = instance_config.dss.instance_id;
int64 instanceId64 = (int64) instanceId;
size_t instanceIdSize = (size_t) instanceId;
size_t compareSize = PG_CONTROL_SIZE;
/* control file contents need special handle in dss mode */
if (dssMode) {
// dms support (MAX_INSTANCEID + 1) instance, and last page for all control.
compareSize = 1 + MAX_INSTANCEID - MIN_INSTANCEID;
compareSize = (compareSize + 1) * PG_CONTROL_SIZE;
src += instanceId64 * PG_CONTROL_SIZE;
// in here, we clear all control page except instance page and last page.
if (instanceId != MIN_INSTANCEID) {
clearSize = instanceIdSize * PG_CONTROL_SIZE;
rc = memset_s(oldSrc, clearSize, 0, clearSize);
securec_check_c(rc, "\0", "\0");
}
if (instanceId != MAX_INSTANCEID) {
clearSize = (size_t) (MAX_INSTANCEID - instanceIdSize) * PG_CONTROL_SIZE;
tmpDssSrc = oldSrc;
tmpDssSrc += (instanceId64 + 1) * PG_CONTROL_SIZE;
rc = memset_s(tmpDssSrc, clearSize, 0, clearSize);
securec_check_c(rc, "\0", "\0");
}
}
if (size != compareSize)
elog(ERROR, "unexpected control file size %d, expected %d",
(int) size, compareSize);
errno_t rc = memcpy_s(ControlFile, sizeof(ControlFileData), src, sizeof(ControlFileData));
rc = memcpy_s(ControlFile, sizeof(ControlFileData), src, sizeof(ControlFileData));
securec_check_c(rc, "\0", "\0");
/* Additional checks on control file */
checkControlFile(ControlFile);
if (dssMode) {
tmpDssSrc = oldSrc;
tmpDssSrc += (MAX_INSTANCEID + 1) * PG_CONTROL_SIZE;
checkSSControlFile(ControlFile, tmpDssSrc, PG_CONTROL_SIZE);
} else {
checkControlFile(ControlFile);
}
}
/*
@ -136,7 +198,6 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo
/* Write pg_control */
fd = fio_open(path,
O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location);
if (fd < 0)
elog(ERROR, "Failed to open file: %s", path);
@ -150,6 +211,30 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo
pg_free(buffer);
}
/*
* Write Dss buffer to pg_control
*/
static void
writeDssControlFile(char* src, size_t srcLen, const char *path, fio_location location)
{
int fd;
/* Write pg_control */
fd = fio_open(path,
O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location);
if (fd < 0)
elog(ERROR, "Failed to open file: %s", path);
if (fio_write(fd, src, srcLen) != (ssize_t)srcLen)
elog(ERROR, "Failed to overwrite file: %s", path);
if (fio_flush(fd) != 0)
elog(ERROR, "Failed to sync file: %s", path);
if (fio_close(fd) != 0) {
elog(ERROR, "Failed to close file: %s", path);
}
}
/*
* Utility shared by backup and restore to fetch the current timeline
* used by a node.
@ -189,10 +274,17 @@ get_current_timeline_from_control(bool safe)
ControlFileData ControlFile;
char *buffer;
size_t size;
fio_location location;
/* First fetch file... */
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size,
safe, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, instance_config.pgdata, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
if (safe && buffer == NULL)
return 0;
@ -233,8 +325,16 @@ get_checkpoint_location(PGconn *conn)
char *buffer;
size_t size;
ControlFileData ControlFile;
fio_location location;
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, instance_config.pgdata, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@ -248,9 +348,18 @@ get_system_identifier(const char *pgdata_path)
ControlFileData ControlFile;
char *buffer;
size_t size;
fio_location location;
/* First fetch file... */
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, pgdata_path, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
if (buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
@ -283,8 +392,17 @@ get_remote_system_identifier(PGconn *conn)
char *buffer;
size_t size;
ControlFileData ControlFile;
fio_location location;
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, instance_config.pgdata, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@ -299,15 +417,24 @@ get_xlog_seg_size(char *pgdata_path)
ControlFileData ControlFile;
char *buffer;
size_t size;
fio_location location;
/* First fetch file... */
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, pgdata_path, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
return ControlFile.xlog_seg_size;
#else
return (uint32) XLOG_SEG_SIZE;
return (uint32) XLogSegSize;
#endif
}
@ -317,10 +444,17 @@ get_data_checksum_version(bool safe)
ControlFileData ControlFile;
char *buffer;
size_t size;
fio_location location;
/* First fetch file... */
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size,
safe, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, instance_config.pgdata, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
if (buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
@ -330,15 +464,14 @@ get_data_checksum_version(bool safe)
}
pg_crc32c
get_pgcontrol_checksum(const char *pgdata_path)
get_pgcontrol_checksum(const char *fullpath)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST);
/* First fetch file in backup dir ... */
buffer = slurpFile(fullpath, &size, false, FIO_BACKUP_HOST);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@ -351,9 +484,17 @@ get_redo(const char *pgdata_path, RedoParams *redo)
ControlFileData ControlFile;
char *buffer;
size_t size;
fio_location location;
/* First fetch file... */
buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, pgdata_path, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
@ -376,21 +517,103 @@ get_redo(const char *pgdata_path, RedoParams *redo)
}
void
parse_vgname_args(const char* args)
{
char *vgname = xstrdup(args);
if (strstr(vgname, "/") != NULL)
elog(ERROR, "invalid token \"/\" in vgname");
char *comma = strstr(vgname, ",");
if (comma == NULL) {
instance_config.dss.vgdata = vgname;
instance_config.dss.vglog = const_cast<char*>("");
return;
}
instance_config.dss.vgdata = xstrdup(vgname);
comma = strstr(instance_config.dss.vgdata, ",");
comma[0] = '\0';
instance_config.dss.vglog = comma + 1;
if (strstr(instance_config.dss.vgdata, ",") != NULL)
elog(ERROR, "invalid vgname args, should be two volume group names, example: \"+data,+log\"");
if (strstr(instance_config.dss.vglog, ",") != NULL)
elog(ERROR, "invalid vgname args, should be two volume group names, example: \"+data,+log\"");
}
bool
is_ss_xlog(const char *ss_dir)
{
char ss_xlog[MAXPGPATH] = {0};
char ss_doublewrite[MAXPGPATH] = {0};
char ss_notify[MAXPGPATH] = {0};
char ss_snapshots[MAXPGPATH] = {0};
int rc = EOK;
int instance_id = instance_config.dss.instance_id;
rc = sprintf_s(ss_xlog, sizeof(ss_xlog), "%s%d", "pg_xlog", instance_id);
securec_check_ss_c(rc, "\0", "\0");
rc = sprintf_s(ss_doublewrite, sizeof(ss_doublewrite), "%s%d", "pg_doublewrite", instance_id);
securec_check_ss_c(rc, "\0", "\0");
rc = sprintf_s(ss_notify, sizeof(ss_notify), "%s%d", "pg_notify", instance_id);
securec_check_ss_c(rc, "\0", "\0");
rc = sprintf_s(ss_snapshots, sizeof(ss_snapshots), "%s%d", "pg_snapshots", instance_id);
securec_check_ss_c(rc, "\0", "\0");
if (IsDssMode() && strlen(instance_config.dss.vglog) &&
(pg_strcasecmp(ss_dir, ss_xlog) == 0 ||
pg_strcasecmp(ss_dir, ss_doublewrite) == 0 ||
pg_strcasecmp(ss_dir, ss_notify) == 0 ||
pg_strcasecmp(ss_dir, ss_notify) == 0)) {
return true;
}
return false;
}
void
ss_createdir(const char *ss_dir, const char *vgdata, const char *vglog)
{
char path[MAXPGPATH] = {0};
char link_path[MAXPGPATH] = {0};
int rc = EOK;
rc = sprintf_s(link_path, sizeof(link_path), "%s/%s", vgdata, ss_dir);
securec_check_ss_c(rc, "\0", "\0");
rc = sprintf_s(path, sizeof(path), "%s/%s", vglog, ss_dir);
securec_check_ss_c(rc, "\0", "\0");
dir_create_dir(path, DIR_PERMISSION);
if (symlink(path, link_path) < 0) {
elog(ERROR, "can not link dss xlog dir \"%s\" to dss xlog dir \"%s\": %s", link_path, path,
strerror(errno));
}
}
/*
* Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint
* 'as-is' is not to be trusted.
*/
void
set_min_recovery_point(pgFile *file, const char *backup_path,
set_min_recovery_point(pgFile *file, const char *fullpath,
XLogRecPtr stop_backup_lsn)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
char fullpath[MAXPGPATH];
fio_location location;
/* First fetch file content */
buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
location = is_dss_file(T_XLOG_CONTROL_FILE) ? FIO_DSS_HOST : FIO_DB_HOST;
if (IsDssMode()) {
buffer = slurpFile(T_XLOG_CONTROL_FILE, &size, false, location);
} else {
char xlog_ctl_file_path[MAXPGPATH] = {'\0'};
join_path_components(xlog_ctl_file_path, instance_config.pgdata, T_XLOG_CONTROL_FILE);
buffer = slurpFile(xlog_ctl_file_path, &size, false, location);
}
digestControlFile(&ControlFile, buffer, size);
elog(LOG, "Current minRecPoint %X/%X",
@ -410,9 +633,6 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
FIN_CRC32C(ControlFile.crc);
/* overwrite pg_control */
errno_t rc = snprintf_s(fullpath, sizeof(fullpath), sizeof(fullpath) - 1,
"%s/%s", backup_path, XLOG_CONTROL_FILE);
securec_check_ss_c(rc, "\0", "\0");
writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST);
/* Update pg_control checksum in backup_list */
@ -432,17 +652,19 @@ copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
char *buffer;
size_t size;
buffer = slurpFile(from_fullpath, "", &size, false, from_location);
buffer = slurpFile(from_fullpath, &size, false, from_location);
digestControlFile(&ControlFile, buffer, size);
file->crc = ControlFile.crc;
file->read_size = size;
file->write_size = size;
file->uncompressed_size = size;
writeControlFile(&ControlFile, to_fullpath, to_location);
/* Write pg_control */
if (is_dss_type(file->type)) {
writeDssControlFile(buffer, size, to_fullpath, to_location);
} else {
writeControlFile(&ControlFile, to_fullpath, to_location);
}
pg_free(buffer);
}
@ -522,6 +744,20 @@ str2status(const char *status)
return BACKUP_STATUS_INVALID;
}
const char *dev2str(device_type_t type)
{
return devTypeName[type];
}
device_type_t str2dev(const char *dev)
{
for (int i = 0; i < (int)DEV_TYPE_NUM; i++) {
if (pg_strcasecmp(dev, devTypeName[i]) == 0)
return (device_type_t)i;
}
return DEV_TYPE_INVALID;
}
bool
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno)
{

22
src/bin/pg_probackup/validate.cpp
View File

@ -16,6 +16,7 @@
#include "thread.h"
#include "common/fe_memutils.h"
#include "storage/file/fio_device.h"
static void *pgBackupValidateFiles(void *arg);
static void do_validate_instance(void);
@ -26,6 +27,7 @@ static bool skipped_due_to_lock = false;
typedef struct
{
const char *base_path;
const char *dss_path;
parray *files;
bool corrupted;
XLogRecPtr stop_lsn;
@ -88,6 +90,16 @@ bool pre_check_backup(pgBackup *backup)
return false;
}
/* Check backup storage mode suitable */
if (IsDssMode() != is_dss_type(backup->storage_type))
{
elog(WARNING, "Backup %s is not suit for instance %s, because they have different "
"storage type. Change it to CORRUPT and skip validation.",
base36enc((long unsigned int)backup->start_time), instance_name);
write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true);
return false;
}
if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE ||
backup->status == BACKUP_STATUS_MERGING)
elog(INFO, "Validating backup %s", base36enc(backup->start_time));
@ -109,6 +121,7 @@ void
pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
{
char base_path[MAXPGPATH];
char dss_path[MAXPGPATH];
char external_prefix[MAXPGPATH];
parray *files = NULL;
bool corrupted = false;
@ -122,6 +135,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
return;
join_path_components(base_path, backup->root_dir, DATABASE_DIR);
join_path_components(dss_path, backup->root_dir, DSSDATA_DIR);
join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
files = get_backup_filelist(backup, false);
@ -158,6 +172,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
validate_files_arg *arg = &(threads_args[i]);
arg->base_path = base_path;
arg->dss_path = dss_path;
arg->files = files;
arg->corrupted = false;
arg->backup_mode = backup->backup_mode;
@ -257,7 +272,7 @@ void check_crc(pgFile *file, char *file_fullpath, validate_files_arg *arguments)
if (arguments->backup_version >= 20025 &&
strcmp(file->name, "pg_control") == 0 &&
!file->external_dir_num)
crc = get_pgcontrol_checksum(arguments->base_path);
crc = get_pgcontrol_checksum(file_fullpath);
else
crc = pgFileGetCRC(file_fullpath,
arguments->backup_version <= 20021 ||
@ -277,6 +292,9 @@ void check_crc(pgFile *file, char *file_fullpath, validate_files_arg *arguments)
* check page headers, checksums (if enabled)
* and compute checksum of the file
*/
if (IsDssMode()) {
return;
}
if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn,
arguments->checksum_version,
arguments->backup_version,
@ -348,6 +366,8 @@ pgBackupValidateFiles(void *arg)
makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
join_path_components(file_fullpath, temp, file->rel_path);
}
else if (is_dss_type(file->type))
join_path_components(file_fullpath, arguments->dss_path, file->rel_path);
else
join_path_components(file_fullpath, arguments->base_path, file->rel_path);

2
src/bin/pg_resetxlog/Makefile
View File

@ -27,7 +27,7 @@ OBJS= pg_resetxlog.o $(WIN32RES)
all: pg_resetxlog
override CXXFLAGS += -fPIE
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -I${top_builddir}/src/include
CFLAGS += -Wl,-z,relro,-z,now -fPIE
LDFLAGS += -pie

17
src/bin/pg_resetxlog/pg_resetxlog.cpp
View File

@ -49,6 +49,7 @@
#include <getopt.h>
#endif
#include "tool_common.h"
#include "access/transam.h"
#include "access/tuptoaster.h"
#include "access/multixact.h"
@ -239,6 +240,8 @@ int main(int argc, char* argv[])
exit(1);
}
initDataPathStruct(false);
/*
* Check for a postmaster lock file --- if there is one, refuse to
* proceed, on grounds we might be interfering with a live installation.
@ -364,7 +367,7 @@ static bool ReadControlFile(void)
pg_crc32 crc;
errno_t rc = 0;
if ((fd = open(XLOG_CONTROL_FILE, O_RDONLY | PG_BINARY, 0)) < 0) {
if ((fd = open(T_XLOG_CONTROL_FILE, O_RDONLY | PG_BINARY, 0)) < 0) {
/*
* If pg_control is not there at all, or we can't read it, the odds
* are we've been handed a bad DataDir path, so give up. User can do
@ -373,14 +376,14 @@ static bool ReadControlFile(void)
fprintf(stderr,
_("%s: could not open file \"%s\" for reading: %s\n"),
progname,
XLOG_CONTROL_FILE,
T_XLOG_CONTROL_FILE,
strerror(errno));
if (errno == ENOENT)
fprintf(stderr,
_("If you are sure the data directory path is correct, execute\n"
" touch %s\n"
"and try again.\n"),
XLOG_CONTROL_FILE);
T_XLOG_CONTROL_FILE);
exit(1);
}
@ -394,7 +397,7 @@ static bool ReadControlFile(void)
}
len = read(fd, buffer, PG_CONTROL_SIZE);
if (len < 0) {
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), progname, XLOG_CONTROL_FILE, strerror(errno));
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), progname, T_XLOG_CONTROL_FILE, strerror(errno));
free(buffer);
buffer = NULL;
close(fd);
@ -494,7 +497,7 @@ static void GuessControlValues(void)
ControlFile.blcksz = BLCKSZ;
ControlFile.relseg_size = RELSEG_SIZE;
ControlFile.xlog_blcksz = XLOG_BLCKSZ;
ControlFile.xlog_seg_size = XLOG_SEG_SIZE;
ControlFile.xlog_seg_size = XLogSegSize;
ControlFile.nameDataLen = NAMEDATALEN;
ControlFile.indexMaxKeys = INDEX_MAX_KEYS;
ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
@ -637,9 +640,9 @@ static void RewriteControlFile(void)
rc = memcpy_s(buffer, PG_CONTROL_SIZE, &ControlFile, sizeof(ControlFileData));
securec_check_c(rc, "", "");
unlink(XLOG_CONTROL_FILE);
unlink(T_XLOG_CONTROL_FILE);
fd = open(XLOG_CONTROL_FILE, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR);
fd = open(T_XLOG_CONTROL_FILE, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR);
if (fd < 0) {
fprintf(stderr, _("%s: could not create pg_control file: %s\n"), progname, strerror(errno));
exit(1);

2
src/bin/pg_rewind/Makefile
View File

@ -18,7 +18,7 @@ include $(top_builddir)/src/Makefile.global
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -I$(top_builddir)/src/bin/pg_ctl -I${top_builddir}/src/lib/page_compression
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) -I$(top_builddir)/src/bin/pg_ctl -I${top_builddir}/src/lib/page_compression -I${top_builddir}/src/include
override LDFLAGS := -L../../src/lib/page_compression
ifneq "$(MAKECMDGOALS)" "clean"
ifneq "$(MAKECMDGOALS)" "distclean"

2
src/bin/pg_rewind/parsexlog.cpp
View File

@ -463,7 +463,7 @@ XLogRecPtr getValidCommonLSN(XLogRecPtr checkLsn, XLogRecPtr maxLsn)
XLByteToPrevSeg(checkLsn, checkLogSegNo);
if (maxLogSegNo > (checkLogSegNo + 1)) {
for (loopLogSegNo = (checkLogSegNo + 1); loopLogSegNo <= maxLogSegNo; loopLogSegNo++) {
startLsn = loopLogSegNo * XLOG_SEG_SIZE;
startLsn = loopLogSegNo * XLogSegSize;
curLsn = InvalidXLogRecPtr;
curLsn = XLogFindNextRecord(xlogreader, startLsn);
if (!XLogRecPtrIsInvalid(curLsn)) {

6
src/bin/pg_rewind/pg_rewind.cpp
View File

@ -34,6 +34,8 @@
#include "common/build_query/build_query.h"
#include "bin/elog.h"
#include "pg_build.h"
#include "tool_common.h"
#define FORMATTED_TS_LEN 128
#define BUILD_PID "gs_build.pid"
@ -718,7 +720,7 @@ static void rewind_dw_file()
char* unaligned_buf = NULL;
/* Delete the dw file, if it exists. */
rc = snprintf_s(dw_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", datadir_target, OLD_DW_FILE_NAME);
rc = snprintf_s(dw_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", datadir_target, T_OLD_DW_FILE_NAME);
securec_check_ss_c(rc, "\0", "\0");
if (realpath(dw_file_path, real_file_path) == NULL) {
if (real_file_path[0] == '\0') {
@ -734,7 +736,7 @@ static void rewind_dw_file()
securec_check_c(rc, "\0", "\0");
/* Delete the dw build file, if it exists. */
rc = snprintf_s(dw_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", datadir_target, DW_BUILD_FILE_NAME);
rc = snprintf_s(dw_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", datadir_target, T_DW_BUILD_FILE_NAME);
securec_check_ss_c(rc, "\0", "\0");
if (realpath(dw_file_path, real_file_path) == NULL) {
if (real_file_path[0] == '\0') {

3
src/bin/psql/print.cpp
View File

@ -1951,7 +1951,8 @@ void printTableInit(
content->ncolumns = ncolumns;
content->nrows = nrows;
if (ncolumns * nrows + 1 <= 0) {
int64 res = (int64)ncolumns * (int64)nrows + 1L;
if (res >= (int64)PG_INT32_MAX) {
fprintf(stderr, _("Error: Integer overflow when select execution.\n"));
exit(EXIT_FAILURE);
}

237
src/common/backend/catalog/catalog.cpp
View File

@ -203,6 +203,15 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
{
int pathlen;
char* path = NULL;
char* datadir = (char *)palloc(MAXPGPATH);
if (ENABLE_DSS && rnode.dbNode != 1) {
errno_t rc = snprintf_s(datadir, MAXPGPATH, MAXPGPATH - 1, "%s/",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name);
securec_check_ss(rc, "\0", "\0");
} else {
datadir[0] = '\0';
}
// Column store
if (IsValidColForkNum(forknum)) {
@ -217,56 +226,117 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
if (rnode.spcNode == GLOBALTABLESPACE_OID) {
/* Shared system relations live in {datadir}/global */
Assert(rnode.dbNode == 0);
Assert(IsHeapFileNode(rnode));
if (!ENABLE_DSS) {
Assert(rnode.dbNode == 0);
}
Assert(backend == InvalidBackendId);
pathlen = 7 + OIDCHARS + 1 + FORKNAMECHARS + 1;
pathlen = (int)strlen(GLOTBSDIR) + OIDCHARS + 1 + FORKNAMECHARS + 1;
path = (char*)palloc(pathlen);
if (forknum != MAIN_FORKNUM) {
rc = snprintf_s(path, pathlen, pathlen - 1, "global/%u_%s", rnode.relNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%s/%u_%s", GLOTBSDIR, rnode.relNode, forkNames[forknum]);
} else {
rc = snprintf_s(path, pathlen, pathlen - 1, "global/%u", rnode.relNode);
rc = snprintf_s(path, pathlen, pathlen - 1, "%s/%u", GLOTBSDIR, rnode.relNode);
}
securec_check_ss(rc, "\0", "\0");
} else if (rnode.spcNode == DEFAULTTABLESPACE_OID) {
/* The default tablespace is {datadir}/base */
if (backend == InvalidBackendId) {
pathlen = 5 + OIDCHARS + 1 + OIDCHARS + 1 + OIDCHARS + 1 + FORKNAMECHARS + 1 + OIDCHARS + 2;
pathlen = strlen(datadir) + 1 + 5 + OIDCHARS + 1 + OIDCHARS + 1 +
OIDCHARS + 1 + FORKNAMECHARS + 1 + OIDCHARS + 2;
path = (char*)palloc(pathlen);
if (forknum != MAIN_FORKNUM) {
if (!IsBucketFileNode(rnode)) {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/%u_%s",
rnode.dbNode, rnode.relNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/%u_%s",
datadir, rnode.dbNode, rnode.relNode, forkNames[forknum]);
} else {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/%u_b%d_%s",
rnode.dbNode, rnode.relNode, rnode.bucketNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/%u_b%d_%s",
datadir, rnode.dbNode, rnode.relNode, rnode.bucketNode, forkNames[forknum]);
}
} else {
if (!IsBucketFileNode(rnode)) {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/%u", rnode.dbNode, rnode.relNode);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/%u",
datadir, rnode.dbNode, rnode.relNode);
} else {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/%u_b%d",
rnode.dbNode, rnode.relNode, rnode.bucketNode);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/%u_b%d",
datadir, rnode.dbNode, rnode.relNode, rnode.bucketNode);
}
}
securec_check_ss(rc, "\0", "\0");
} else {
/* OIDCHARS will suffice for an integer, too */
Assert(!IsBucketFileNode(rnode));
pathlen = 5 + OIDCHARS + 2 + OIDCHARS + 1 + OIDCHARS + 1 + FORKNAMECHARS + 1;
pathlen = (int)strlen(datadir) + 1 + 5 + OIDCHARS + 2 + OIDCHARS + 1 +
OIDCHARS + 1 + FORKNAMECHARS + 1;
path = (char*)palloc(pathlen);
if (forknum != MAIN_FORKNUM) {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/t%d_%u_%s",
rnode.dbNode, backend, rnode.relNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/t%d_%u_%s",
datadir, rnode.dbNode, backend, rnode.relNode, forkNames[forknum]);
} else {
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u/t%d_%u", rnode.dbNode, backend, rnode.relNode);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u/t%d_%u",
datadir, rnode.dbNode, backend, rnode.relNode);
}
securec_check_ss(rc, "\0", "\0");
}
} else if (ENABLE_DSS) {
if (backend == InvalidBackendId) {
pathlen = strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) + 1 +
OIDCHARS + 1 + OIDCHARS + 1 + OIDCHARS + 2 + OIDCHARS + 1 + FORKNAMECHARS + 1;
path = (char*)palloc(pathlen);
if (forknum != MAIN_FORKNUM)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u/%u_%s",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
rnode.relNode,
forkNames[forknum]);
else
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u/%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
rnode.relNode);
securec_check_ss(rc, "\0", "\0");
} else {
pathlen = 9 + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) + 1 + OIDCHARS + 2
+ OIDCHARS + 1 + OIDCHARS + 1 + FORKNAMECHARS + 1;
path = (char*)palloc(pathlen);
if (forknum != MAIN_FORKNUM)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u/t%d_%u_%s",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
backend,
rnode.relNode,
forkNames[forknum]);
else
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u/t%d_%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
backend,
rnode.relNode);
securec_check_ss(rc, "\0", "\0");
}
} else {
/* All other tablespaces are accessed via symlinks */
if (backend == InvalidBackendId) {
pathlen = 9 + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) + 1 +
pathlen = strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) + 1 +
#ifdef PGXC
/* Postgres-XC tablespaces include node name */
strlen(g_instance.attr.attr_common.PGXCNodeName) + 1 +
@ -276,20 +346,23 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
#ifdef PGXC
if (forknum != MAIN_FORKNUM) {
if (!IsBucketFileNode(rnode)) {
rc = snprintf_s(path, pathlen, pathlen - 1, "pg_tblspc/%u/%s_%s/%u/%u_%s",
rnode.spcNode, TABLESPACE_VERSION_DIRECTORY, g_instance.attr.attr_common.PGXCNodeName,
rnode.dbNode, rnode.relNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%s/%u/%s_%s/%u/%u_%s",
TBLSPCDIR, rnode.spcNode, TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName, rnode.dbNode,
rnode.relNode, forkNames[forknum]);
} else {
rc = snprintf_s(path, pathlen, pathlen - 1, "pg_tblspc/%u/%s_%s/%u/%u_b%d_%s",
rnode.spcNode, TABLESPACE_VERSION_DIRECTORY, g_instance.attr.attr_common.PGXCNodeName,
rnode.dbNode, rnode.relNode, rnode.bucketNode, forkNames[forknum]);
rc = snprintf_s(path, pathlen, pathlen - 1, "%s/%u/%s_%s/%u/%u_b%d_%s",
TBLSPCDIR, rnode.spcNode, TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName, rnode.dbNode,
rnode.relNode, rnode.bucketNode, forkNames[forknum]);
}
} else {
if (!IsBucketFileNode(rnode)) {
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s_%s/%u/%u",
"%s/%u/%s_%s/%u/%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
@ -299,7 +372,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s_%s/%u/%u_b%d",
"%s/%u/%s_%s/%u/%u_b%d",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
@ -314,7 +388,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s/%u/%u_%s",
"%s/%u/%s/%u/%u_%s",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
@ -324,7 +399,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s/%u/%u",
"%s/%u/%s/%u/%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
@ -344,7 +420,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s_%s/%u/t%d_%u_%s",
"%s/%u/%s_%s/%u/t%d_%u_%s",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
@ -356,7 +433,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s_%s/%u/t%d_%u",
"%s/%u/%s_%s/%u/t%d_%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
@ -369,7 +447,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s/%u/t%d_%u_%s",
"%s/%u/%s/%u/t%d_%u_%s",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
@ -380,7 +459,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s/%u/t%d_%u",
"%s/%u/%s/%u/t%d_%u",
TBLSPCDIR,
rnode.spcNode,
TABLESPACE_VERSION_DIRECTORY,
rnode.dbNode,
@ -391,6 +471,8 @@ char* relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum)
}
}
}
pfree(datadir);
return path;
}
@ -479,8 +561,24 @@ RelFileNodeForkNum relpath_to_filenode(char* path)
char* tmptoken = NULL;
parsepath = pstrdup(path);
token = strtok_r(parsepath, "/", &tmptoken);
if (ENABLE_DSS && parsepath[0] == '+') {
char *tmppath = NULL;
char *newpath = parsepath;
uint32 pathsize = (uint32)strlen(parsepath);
uint32 homesize = (uint32)strlen(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name);
if ((pathsize <= homesize + 1) ||
(strncmp(path, g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, homesize) != 0)) {
pfree(parsepath);
ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid relation file path %s.", path)));
}
newpath += homesize + 1;
tmppath = pstrdup(newpath);
pfree(parsepath);
parsepath = tmppath;
}
token = strtok_r(parsepath, "/", &tmptoken);
if (NULL == tmptoken || '\0' == *tmptoken) {
pfree(parsepath);
ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid relation file path %s.", path)));
@ -541,13 +639,22 @@ RelFileNodeForkNum relpath_to_filenode(char* path)
}
char tblspcversiondir[MAXPGPATH];
int errorno = snprintf_s(tblspcversiondir,
MAXPGPATH,
MAXPGPATH - 1,
"%s_%s",
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
securec_check_ss(errorno, "\0", "\0");
if (ENABLE_DSS) {
int errorno = snprintf_s(tblspcversiondir,
MAXPGPATH,
MAXPGPATH - 1,
"%s",
TABLESPACE_VERSION_DIRECTORY);
securec_check_ss(errorno, "\0", "\0");
} else {
int errorno = snprintf_s(tblspcversiondir,
MAXPGPATH,
MAXPGPATH - 1,
"%s_%s",
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
securec_check_ss(errorno, "\0", "\0");
}
/* skip tablespaces which not belong to us. */
if (0 != strncmp(token, tblspcversiondir, strlen(tblspcversiondir) + 1)) {
pfree(parsepath);
@ -576,26 +683,49 @@ char* GetDatabasePath(Oid dbNode, Oid spcNode)
int pathlen;
char* path = NULL;
errno_t rc = EOK;
char* datadir = (char *)palloc(MAXPGPATH);
if (ENABLE_DSS && dbNode != 1) {
rc = snprintf_s(datadir, MAXPGPATH, MAXPGPATH - 1, "%s/",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name);
securec_check_ss(rc, "\0", "\0");
} else {
datadir[0] = '\0';
}
if (spcNode == GLOBALTABLESPACE_OID) {
/* Shared system relations live in {datadir}/global */
Assert(dbNode == 0);
pathlen = 6 + 1;
pathlen = (int)strlen(GLOTBSDIR) + 1;
path = (char*)palloc(pathlen);
rc = snprintf_s(path, pathlen, pathlen - 1, "global");
rc = snprintf_s(path, pathlen, pathlen - 1, "%s", GLOTBSDIR);
securec_check_ss(rc, "\0", "\0");
} else if (spcNode == DEFAULTTABLESPACE_OID) {
/* The default tablespace is {datadir}/base */
pathlen = 5 + OIDCHARS + 1;
pathlen = (int)strlen(datadir) + 1 + 5 + OIDCHARS + 1;
path = (char*)palloc(pathlen);
rc = snprintf_s(path, pathlen, pathlen - 1, "base/%u", dbNode);
rc = snprintf_s(path, pathlen, pathlen - 1, "%sbase/%u", datadir, dbNode);
securec_check_ss(rc, "\0", "\0");
} else if (ENABLE_DSS) {
/* All other tablespaces are accessed via symlinks */
pathlen = (int)strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 + (int)strlen(TABLESPACE_VERSION_DIRECTORY) +
1 + OIDCHARS + 1;
path = (char*)palloc(pathlen);
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u",
TBLSPCDIR,
spcNode,
TABLESPACE_VERSION_DIRECTORY,
dbNode);
securec_check_ss(rc, "\0", "\0");
} else {
/* All other tablespaces are accessed via symlinks */
pathlen = 9 + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) +
pathlen = (int)strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 + (int)strlen(TABLESPACE_VERSION_DIRECTORY) +
#ifdef PGXC
/* Postgres-XC tablespaces include node name in path */
strlen(g_instance.attr.attr_common.PGXCNodeName) + 1 +
(int)strlen(g_instance.attr.attr_common.PGXCNodeName) + 1 +
#endif
1 + OIDCHARS + 1;
path = (char*)palloc(pathlen);
@ -603,17 +733,26 @@ char* GetDatabasePath(Oid dbNode, Oid spcNode)
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"pg_tblspc/%u/%s_%s/%u",
"%s/%u/%s_%s/%u",
TBLSPCDIR,
spcNode,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
dbNode);
#else
rc =
snprintf_s(path, pathlen, pathlen - 1, "pg_tblspc/%u/%s/%u", spcNode, TABLESPACE_VERSION_DIRECTORY, dbNode);
rc = snprintf_s(path,
pathlen,
pathlen - 1,
"%s/%u/%s/%u",
TBLSPCDIR,
spcNode,
TABLESPACE_VERSION_DIRECTORY,
dbNode);
#endif
securec_check_ss(rc, "\0", "\0");
}
pfree(datadir);
return path;
}

55
src/common/backend/catalog/heap.cpp
View File

@ -525,6 +525,13 @@ Relation heap_create(const char* relname, Oid relnamespace, Oid reltablespace, O
Relation rel;
bool isbucket = false;
if (IsInitdb && EnableInitDBSegment) {
/* store tables in segment storage as all possible while initdb */
if (relpersistence == RELPERSISTENCE_PERMANENT) {
storage_type = SEGMENT_PAGE;
}
}
/* The caller must have provided an OID for the relation. */
Assert(OidIsValid(relid));
@ -578,7 +585,17 @@ Relation heap_create(const char* relname, Oid relnamespace, Oid reltablespace, O
}
break;
}
if (ENABLE_DSS && !partitioned_relation) {
/*
* when we store systable to segment, we should allocate segment header page
* for all objects, to avoid some issues, like: pg_table_size for view.
* if the view has no segment header page, the seg_totalblocks' call will crash,
* because of read_head_buffer's magic number check fail.
*/
create_storage = true;
}
/*
* Never allow a pg_class entry to explicitly specify the database's
* default tablespace in reltablespace; force it to zero instead. This
@ -599,17 +616,21 @@ Relation heap_create(const char* relname, Oid relnamespace, Oid reltablespace, O
if (u_sess->proc_cxt.IsBinaryUpgrade) {
if (!partitioned_relation && storage_type == SEGMENT_PAGE) {
isbucket = BUCKET_OID_IS_VALID(bucketOid) && !newcbi;
Oid database_id = (ConvertToRelfilenodeTblspcOid(reltablespace) == GLOBALTABLESPACE_OID) ?
InvalidOid : u_sess->proc_cxt.MyDatabaseId;
relfilenode = seg_alloc_segment(ConvertToRelfilenodeTblspcOid(reltablespace),
u_sess->proc_cxt.MyDatabaseId, isbucket, relfilenode);
database_id, isbucket, relfilenode);
}
} else {
create_storage = false;
}
} else if (storage_type == SEGMENT_PAGE && !partitioned_relation) {
Assert(reltablespace != GLOBALTABLESPACE_OID);
} else if ((storage_type == SEGMENT_PAGE && !partitioned_relation) ||
(storage_type == SEGMENT_PAGE && ENABLE_DSS && create_storage)) {
isbucket = BUCKET_OID_IS_VALID(bucketOid) && !newcbi;
Oid database_id = (ConvertToRelfilenodeTblspcOid(reltablespace) == GLOBALTABLESPACE_OID) ?
InvalidOid : u_sess->proc_cxt.MyDatabaseId;
relfilenode = (Oid)seg_alloc_segment(ConvertToRelfilenodeTblspcOid(reltablespace),
u_sess->proc_cxt.MyDatabaseId, isbucket, InvalidBlockNumber);
database_id, isbucket, InvalidBlockNumber);
ereport(LOG, (errmsg("Segment Relation %s(%u) set relfilenode %u xid %lu", relname, relid, relfilenode,
GetCurrentTransactionIdIfAny())));
} else {
@ -2479,6 +2500,15 @@ static Oid AddNewRelationType(const char* typname, Oid typeNamespace, Oid new_re
InvalidOid); /* rowtypes never have a collation */
}
static Datum AddSegmentOption(Datum relOptions)
{
DefElem *def = makeDefElem(pstrdup("segment"), (Node *)makeString((char *)"on"));
List* optsList = untransformRelOptions(relOptions);
optsList = lappend(optsList, def);
return transformRelOptions((Datum)0, optsList, NULL, NULL, false, false);
}
/* --------------------------------
* heap_create_with_catalog
*
@ -2532,6 +2562,21 @@ Oid heap_create_with_catalog(const char *relname, Oid relnamespace, Oid reltable
bool relhasbucket = false;
bool relhasuids = false;
if (IsInitdb && EnableInitDBSegment) {
if (relpersistence == RELPERSISTENCE_UNLOGGED) {
relpersistence = RELPERSISTENCE_PERMANENT;
ereport(WARNING,
(errmsg("Store unlogged table in segment when enable system table segment")));
}
/* store tables in segment storage as all possible while initdb */
if (relpersistence == RELPERSISTENCE_PERMANENT &&
(relkind != RELKIND_SEQUENCE && relkind != RELKIND_LARGE_SEQUENCE)) {
storage_type = SEGMENT_PAGE;
reloptions = AddSegmentOption(reloptions);
}
}
pg_class_desc = heap_open(RelationRelationId, RowExclusiveLock);
/*

4
src/common/backend/catalog/namespace.cpp
View File

@ -4444,6 +4444,10 @@ static void InitTempTableNamespace(void)
ereport(ERROR,
(errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("cannot create temporary tables during recovery")));
if (SSIsServerModeReadOnly()) {
ereport(ERROR, (errmsg("cannot create temporary tables at Standby with DMS enabled")));
}
timeLineId = get_controlfile_timeline();
tempID = __sync_add_and_fetch(&gt_tempID_seed, 1);

5
src/common/backend/catalog/objectaddress.cpp
View File

@ -185,6 +185,11 @@ static const ObjectPropertyType* get_object_property_data(Oid class_id);
ObjectAddress get_object_address(
ObjectType objtype, List* objname, List* objargs, Relation* relp, LOCKMODE lockmode, bool missing_ok)
{
if (ENABLE_DMS && (objtype == OBJECT_PUBLICATION || objtype == OBJECT_PUBLICATION_NAMESPACE ||
objtype == OBJECT_PUBLICATION_REL || objtype == OBJECT_SUBSCRIPTION)) {
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Not support Publication and Subscription while DMS and DSS enabled")));
}
ObjectAddress address;
ObjectAddress old_address = {InvalidOid, InvalidOid, 0};
Relation relation = NULL;

4
src/common/backend/catalog/pg_hashbucket.cpp
View File

@ -1161,8 +1161,10 @@ HeapTuple HbktModifyRelationRelfilenode(HeapTuple reltup, DataTransferType trans
newrelfilenode = GetNewRelFileNode(indexrel->rd_rel->reltablespace, NULL, indexrel->rd_rel->relpersistence);
bucketNode = InvalidBktId;
} else {
Oid database_id = (ConvertToRelfilenodeTblspcOid(indexrel->rd_rel->reltablespace) == GLOBALTABLESPACE_OID) ?
InvalidOid : u_sess->proc_cxt.MyDatabaseId;
newrelfilenode = seg_alloc_segment(ConvertToRelfilenodeTblspcOid(indexrel->rd_rel->reltablespace),
u_sess->proc_cxt.MyDatabaseId, isBucket, InvalidBlockNumber);
database_id, isBucket, InvalidBlockNumber);
bucketNode = SegmentBktId;
}
rnode = indexrel->rd_node;

20
src/common/backend/catalog/storage.cpp
View File

@ -51,6 +51,7 @@
#include "utils/rel_gs.h"
#include "utils/snapmgr.h"
#include "utils/syscache.h"
#include "ddes/dms/ss_transaction.h"
#ifdef ENABLE_MULTIPLE_NODES
#include "tsdb/cache/part_cachemgr.h"
@ -843,6 +844,10 @@ void smgrDoDropBufferUsingScan(bool isCommit)
}
}
DropRelFileNodeAllBuffersUsingScan(rnodes, rnode_len);
if (ENABLE_DMS && SS_PRIMARY_MODE && rnode_len > 0) {
SSBCastDropRelAllBuffer(rnodes, rnode_len);
}
}
void smgrDoDropBufferUsingHashTbl(bool isCommit)
@ -850,6 +855,9 @@ void smgrDoDropBufferUsingHashTbl(bool isCommit)
PendingRelDelete* pending = NULL;
PendingRelDelete* next = NULL;
int rnode_len = 0;
RelFileNode rnodes[DROP_BUFFER_USING_HASH_DEL_REL_NUM_THRESHOLD];
int nestLevel = GetCurrentTransactionNestLevel();
HTAB* relfilenode_hashtbl = relfilenode_hashtbl_create();
int enter_cnt = 0;
@ -863,11 +871,23 @@ void smgrDoDropBufferUsingHashTbl(bool isCommit)
(void)hash_search(relfilenode_hashtbl, &(pending->relnode), HASH_ENTER, &found);
if (!found) {
enter_cnt++;
if (ENABLE_DMS && SS_PRIMARY_MODE) {
if (rnode_len >= DROP_BUFFER_USING_HASH_DEL_REL_NUM_THRESHOLD) {
SSBCastDropRelAllBuffer(rnodes, rnode_len);
rnode_len = 0;
}
rnodes[rnode_len++] = pending->relnode;
}
}
}
}
}
}
if (ENABLE_DMS && SS_PRIMARY_MODE && rnode_len > 0) {
SSBCastDropRelAllBuffer(rnodes, rnode_len);
}
/* At least one relnode founded */
if (enter_cnt > 0) {

4
src/common/backend/catalog/toasting.cpp
View File

@ -678,6 +678,10 @@ static void InitLobTempToastNamespace(void)
ereport(ERROR,
(errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("cannot create temporary tables during recovery")));
if (SSIsServerModeReadOnly()) {
ereport(ERROR, (errmsg("cannot create temporary tables at Standby with DMS enabled")));
}
timeLineId = get_controlfile_timeline();
tempID = __sync_add_and_fetch(&gt_tempID_seed, 1);

12
src/common/backend/libpq/auth.cpp
View File

@ -706,7 +706,8 @@ void ClientAuthentication(Port* port)
if (IsRoleExist(port->user_name) && GetRoleOid(port->user_name) != INITIAL_USER_ID) {
Oid roleid = GetRoleOid(port->user_name);
USER_STATUS rolestatus;
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE ||
(ENABLE_DMS && !SS_MY_INST_IS_MASTER)) {
rolestatus = GetAccountLockedStatusFromHashTable(roleid);
} else {
rolestatus = GetAccountLockedStatus(roleid);
@ -714,7 +715,8 @@ void ClientAuthentication(Port* port)
if (UNLOCK_STATUS != rolestatus) {
errno_t errorno = EOK;
bool unlocked = false;
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE ||
(ENABLE_DMS && !SS_MY_INST_IS_MASTER)) {
unlocked = UnlockAccountToHashTable(roleid, false, false);
} else {
unlocked = TryUnlockAccount(roleid, false, false);
@ -738,7 +740,8 @@ void ClientAuthentication(Port* port)
(errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), errmsg("The account has been locked.")));
}
} else if (status == STATUS_OK) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE ||
(ENABLE_DMS && !SS_MY_INST_IS_MASTER)) {
(void)UnlockAccountToHashTable(roleid, false, true);
} else {
(void)TryUnlockAccount(roleid, false, true);
@ -747,7 +750,8 @@ void ClientAuthentication(Port* port)
/* if password is not right, send signal to try lock the account*/
if (status == STATUS_WRONG_PASSWORD) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) {
if (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE ||
(ENABLE_DMS && !SS_MY_INST_IS_MASTER)) {
UpdateFailCountToHashTable(roleid, 1, false);
} else {
TryLockAccount(roleid, 1, false);

65
src/common/backend/parser/gram.y
View File

@ -2027,7 +2027,17 @@ OptSchemaEltList:
;
OptBlockchainWith:
WITH BLOCKCHAIN { $$ = true; }
WITH BLOCKCHAIN
{
if (ENABLE_DMS) {
const char* message = "BLOCKCHAIN is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("BLOCKCHAIN is not supported while DMS and DSS enabled")));
}
$$ = true;
}
| /* EMPTY */ { $$ = false; }
;
@ -2049,7 +2059,17 @@ AlterSchemaStmt:
;
OptAlterToBlockchain:
WITH BLOCKCHAIN { $$ = true; }
WITH BLOCKCHAIN
{
if (ENABLE_DMS) {
const char* message = "BLOCKCHAIN is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("BLOCKCHAIN is not supported while DMS and DSS enabled")));
}
$$ = true;
}
| WITHOUT BLOCKCHAIN { $$ = false; }
;
@ -8656,6 +8676,11 @@ CreateMatViewStmt:
errmsg("It's not supported to specify distribute key on incremental materialized views")));
}
#endif
if (ENABLE_DMS) {
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("matview is not supported while DMS and DSS enabled.")));
}
$6->ivm = $3;
$$ = (Node *) ctas;
}
@ -16641,14 +16666,28 @@ TransactionStmt:
$$ = (Node *)n;
}
| PREPARE TRANSACTION Sconst
{
{
if (ENABLE_DMS) {
const char* message = "PREPARE TRANSACTION is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("PREPARE TRANSACTION is not supported while DMS and DSS enabled")));
}
TransactionStmt *n = makeNode(TransactionStmt);
n->kind = TRANS_STMT_PREPARE;
n->gid = $3;
$$ = (Node *)n;
}
| COMMIT PREPARED Sconst
{
{
if (ENABLE_DMS) {
const char* message = "COMMIT TRANSACTION is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COMMIT TRANSACTION is not supported while DMS and DSS enabled")));
}
TransactionStmt *n = makeNode(TransactionStmt);
n->kind = TRANS_STMT_COMMIT_PREPARED;
n->gid = $3;
@ -16656,7 +16695,14 @@ TransactionStmt:
$$ = (Node *)n;
}
| COMMIT PREPARED Sconst WITH Sconst
{
{
if (ENABLE_DMS) {
const char* message = "COMMIT TRANSACTION is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COMMIT TRANSACTION is not supported while DMS and DSS enabled")));
}
TransactionStmt *n = makeNode(TransactionStmt);
n->kind = TRANS_STMT_COMMIT_PREPARED;
n->gid = $3;
@ -16664,7 +16710,14 @@ TransactionStmt:
$$ = (Node *)n;
}
| ROLLBACK PREPARED Sconst
{
{
if (ENABLE_DMS) {
const char* message = "ROLLBACK TRANSACTION is not supported while DMS and DSS enabled";
InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc);
ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("ROLLBACK TRANSACTION is not supported while DMS and DSS enabled")));
}
TransactionStmt *n = makeNode(TransactionStmt);
n->kind = TRANS_STMT_ROLLBACK_PREPARED;
n->gid = $3;

103
src/common/backend/utils/adt/dbsize.cpp
View File

@ -71,6 +71,7 @@
#include "storage/cstore/cstore_compress.h"
#include "storage/page_compression.h"
#include "vecexecutor/vecnodes.h"
#include "storage/file/fio_device.h"
#ifdef PGXC
static Datum pgxc_database_size(Oid dbOid);
@ -159,6 +160,7 @@ static int64 calculate_database_size(Oid dbOid)
{
int64 totalsize;
DIR* dirdesc = NULL;
char* dssdir = NULL;
struct dirent* direntry = NULL;
char dirpath[MAXPGPATH] = {'\0'};
char pathname[MAXPGPATH] = {'\0'};
@ -170,16 +172,30 @@ static int64 calculate_database_size(Oid dbOid)
if (aclresult != ACLCHECK_OK)
aclcheck_error(aclresult, ACL_KIND_DATABASE, get_and_check_db_name(dbOid));
/* Get the vgname in DSS mode */
if (ENABLE_DSS)
dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name;
/* Shared storage in pg_global is not counted */
/* Include pg_default storage */
rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "base/%u", dbOid);
securec_check_ss(rc, "\0", "\0");
if (ENABLE_DSS && dbOid != 1) {
rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "%s/base/%u", dssdir, dbOid);
securec_check_ss(rc, "", "");
} else {
rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "base/%u", dbOid);
securec_check_ss(rc, "\0", "\0");
}
totalsize = db_dir_size(pathname);
/* Scan the non-default tablespaces */
rc = snprintf_s(dirpath, MAXPGPATH, MAXPGPATH - 1, "pg_tblspc");
securec_check_ss(rc, "\0", "\0");
if (ENABLE_DSS) {
rc = snprintf_s(dirpath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_tblspc", dssdir);
securec_check_ss(rc, "", "");
} else {
rc = snprintf_s(dirpath, MAXPGPATH, MAXPGPATH - 1, "pg_tblspc");
securec_check_ss(rc, "\0", "\0");
}
dirdesc = AllocateDir(dirpath);
if (NULL == dirdesc)
ereport(ERROR, (errcode_for_file_access(), errmsg("could not open tablespace directory \"%s\": %m", dirpath)));
@ -192,16 +208,26 @@ static int64 calculate_database_size(Oid dbOid)
#ifdef PGXC
/* openGauss tablespaces include node name in path */
rc = snprintf_s(pathname,
MAXPGPATH,
MAXPGPATH - 1,
"pg_tblspc/%s/%s_%s/%u",
direntry->d_name,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
dbOid);
securec_check_ss(rc, "\0", "\0");
if (ENABLE_DSS) {
rc = snprintf_s(pathname,
MAXPGPATH,
MAXPGPATH - 1,
"pg_tblspc/%s/%s/%u",
direntry->d_name,
TABLESPACE_VERSION_DIRECTORY,
dbOid);
securec_check_ss(rc, "\0", "\0");
} else {
rc = snprintf_s(pathname,
MAXPGPATH,
MAXPGPATH - 1,
"pg_tblspc/%s/%s_%s/%u",
direntry->d_name,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName,
dbOid);
securec_check_ss(rc, "\0", "\0");
}
#else
rc = snprintf_s(pathname,
MAXPGPATH,
@ -212,6 +238,13 @@ static int64 calculate_database_size(Oid dbOid)
dbOid);
securec_check_ss(rc, "\0", "\0");
#endif
/* Get the path in DSS mode */
if (ENABLE_DSS) {
char temp_path[MAXPGPATH];
rc = snprintf_s(temp_path, MAXPGPATH, MAXPGPATH - 1, "%s", pathname);
rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dssdir, temp_path);
securec_check_ss(rc, "", "");
}
totalsize += db_dir_size(pathname);
}
@ -592,21 +625,37 @@ static int64 calculate_tablespace_size(Oid tblspcOid)
errdetail("Please calculate size of DFS tablespace \"%s\" on coordinator node.",
get_tablespace_name(tblspcOid))));
}
if (tblspcOid == DEFAULTTABLESPACE_OID)
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "base");
else if (tblspcOid == GLOBALTABLESPACE_OID)
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "global");
else
if (tblspcOid == DEFAULTTABLESPACE_OID) {
if (ENABLE_DSS) {
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "%s/base",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name);
} else {
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "base");
}
} else if (tblspcOid == GLOBALTABLESPACE_OID) {
if (ENABLE_DSS) {
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "%s/global",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name);
} else {
rc = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH - 1, "global");
}
} else
#ifdef PGXC
/* openGauss tablespaces include node name in path */
rc = snprintf_s(tblspcPath,
MAXPGPATH,
MAXPGPATH - 1,
"pg_tblspc/%u/%s_%s",
tblspcOid,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
if (ENABLE_DSS) {
rc = snprintf_s(
tblspcPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_tblspc/%u/%s",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name,
tblspcOid, TABLESPACE_VERSION_DIRECTORY);
} else {
rc = snprintf_s(tblspcPath,
MAXPGPATH,
MAXPGPATH - 1,
"pg_tblspc/%u/%s_%s",
tblspcOid,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
}
#else
rc = snprintf_s(
tblspcPath, MAXPGPATH, MAXPGPATH - 1, "pg_tblspc/%u/%s", tblspcOid, TABLESPACE_VERSION_DIRECTORY);

16
src/common/backend/utils/adt/genfile.cpp
View File

@ -38,6 +38,7 @@
#include "utils/lsyscache.h"
#include "catalog/pg_partition_fn.h"
#include "storage/cfs/cfs_buffers.h"
#include "storage/file/fio_device.h"
typedef struct {
char* location;
@ -1402,7 +1403,7 @@ Datum pg_ls_waldir(PG_FUNCTION_ARGS)
if (!superuser() && !isMonitoradmin(GetUserId()))
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("only system/monitor admin can check WAL directory!")));
return pg_ls_dir_files(fcinfo, XLOGDIR, false);
return pg_ls_dir_files(fcinfo, SS_XLOGDIR, false);
}
/*
@ -1415,8 +1416,11 @@ static Datum pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc)
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("only system/monitor admin can check pgsql_tmp directory!")));
if (!SearchSysCacheExists1(TABLESPACEOID, ObjectIdGetDatum(tblspc)))
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("tablespace with OID %u does not exist", tblspc)));
if (OidIsValid(tblspc)) {
if (!SearchSysCacheExists1(TABLESPACEOID, ObjectIdGetDatum(tblspc)))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("tablespace with OID %u does not exist", tblspc)));
}
TempTablespacePath(path, tblspc);
return pg_ls_dir_files(fcinfo, path, true);
@ -1428,7 +1432,11 @@ static Datum pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc)
*/
Datum pg_ls_tmpdir_noargs(PG_FUNCTION_ARGS)
{
return pg_ls_tmpdir(fcinfo, DEFAULTTABLESPACE_OID);
if (!ENABLE_DSS) {
return pg_ls_tmpdir(fcinfo, DEFAULTTABLESPACE_OID);
} else {
return pg_ls_tmpdir(fcinfo, InvalidOid);
}
}
/*

59
src/common/backend/utils/adt/misc.cpp
View File

@ -38,6 +38,7 @@
#include "storage/pmsignal.h"
#include "storage/proc.h"
#include "storage/procarray.h"
#include "storage/file/fio_device.h"
#include "utils/lsyscache.h"
#include "tcop/tcopprot.h"
#include "utils/acl.h"
@ -626,37 +627,50 @@ Datum pg_tablespace_databases(PG_FUNCTION_ARGS)
fctx = (ts_db_fctx*)palloc(sizeof(ts_db_fctx));
/*
* size = tablespace dirname length + dir sep char + oid + terminator
*/
#ifdef PGXC
/* openGauss tablespaces also include node name in path */
location_len = 9 + 1 + OIDCHARS + 1 + strlen(g_instance.attr.attr_common.PGXCNodeName) + 1 +
strlen(TABLESPACE_VERSION_DIRECTORY) + 1;
fctx->location = (char*)palloc(location_len);
#else
location_len = 9 + 1 + OIDCHARS + 1 + strlen(TABLESPACE_VERSION_DIRECTORY) + 1 fctx->location =
(char*)palloc(location_len);
#endif
if (tablespaceOid == GLOBALTABLESPACE_OID) {
fctx->dirdesc = NULL;
ereport(WARNING, (errmsg("global tablespace never has databases")));
} else {
if (tablespaceOid == DEFAULTTABLESPACE_OID)
ss_rc = sprintf_s(fctx->location, location_len, "base");
else
#ifdef PGXC
/* openGauss tablespaces also include node name in path */
if (tablespaceOid == DEFAULTTABLESPACE_OID) {
location_len = (int)strlen(DEFTBSDIR) + 1;
fctx->location = (char*)palloc(location_len);
ss_rc = sprintf_s(fctx->location, (size_t)location_len, "%s", DEFTBSDIR);
} else if (ENABLE_DSS) {
location_len = (int)strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 +
(int)strlen(TABLESPACE_VERSION_DIRECTORY) + 1;
fctx->location = (char*)palloc(location_len);
ss_rc = sprintf_s(fctx->location,
location_len,
"pg_tblspc/%u/%s_%s",
"%s/%u/%s",
TBLSPCDIR,
tablespaceOid,
TABLESPACE_VERSION_DIRECTORY);
} else {
#ifdef PGXC
/* openGauss tablespaces also include node name in path */
location_len = (int)strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 +
(int)strlen(TABLESPACE_VERSION_DIRECTORY) + 1 +
(int)strlen(g_instance.attr.attr_common.PGXCNodeName) + 1;
fctx->location = (char*)palloc(location_len);
ss_rc = sprintf_s(fctx->location,
location_len,
"%s/%u/%s_%s",
TBLSPCDIR,
tablespaceOid,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
#else
ss_rc = sprintf_s(
fctx->location, location_len, "pg_tblspc/%u/%s", tablespaceOid, TABLESPACE_VERSION_DIRECTORY);
location_len = (int)strlen(TBLSPCDIR) + 1 + OIDCHARS + 1 +
(int)strlen(TABLESPACE_VERSION_DIRECTORY) + 1;
fctx->location = (char*)palloc(location_len);
ss_rc = sprintf_s(fctx->location,
location_len,
"%s/%u/%s",
TBLSPCDIR,
tablespaceOid,
TABLESPACE_VERSION_DIRECTORY);
#endif
}
securec_check_ss(ss_rc, "\0", "\0");
fctx->dirdesc = AllocateDir(fctx->location);
@ -690,7 +704,7 @@ Datum pg_tablespace_databases(PG_FUNCTION_ARGS)
/* if database subdir is empty, don't report tablespace as used */
/* size = path length + dir sep char + file name + terminator */
int sub_len = strlen(fctx->location) + 1 + strlen(de->d_name) + 1;
int sub_len = (int)strlen(fctx->location) + 1 + (int)strlen(de->d_name) + 1;
subdir = (char*)palloc(sub_len);
ss_rc = sprintf_s(subdir, sub_len, "%s/%s", fctx->location, de->d_name);
securec_check_ss(ss_rc, "\0", "\0");
@ -743,7 +757,8 @@ Datum pg_tablespace_location(PG_FUNCTION_ARGS)
* Find the location of the tablespace by reading the symbolic link that
* is in pg_tblspc/<oid>.
*/
errno_t ss_rc = snprintf_s(sourcepath, sizeof(sourcepath), sizeof(sourcepath) - 1, "pg_tblspc/%u", tablespaceOid);
errno_t ss_rc = snprintf_s(sourcepath, sizeof(sourcepath), sizeof(sourcepath) - 1,
"%s/%u", TBLSPCDIR, tablespaceOid);
securec_check_ss(ss_rc, "\0", "\0");
rllen = readlink(sourcepath, targetpath, sizeof(targetpath));

5
src/common/backend/utils/adt/pgundostatfuncs.cpp
View File

@ -2438,6 +2438,11 @@ Datum gs_undo_dump_parsepage_mv(PG_FUNCTION_ARGS)
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported view in multiple nodes mode.")));
PG_RETURN_VOID();
#else
if (ENABLE_DSS) {
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported view when enable dss.")));
PG_RETURN_VOID();
}
/* check user's right */
const char fName[MAXFNAMELEN] = "gs_undo_dump_parsepage_mv";
CheckUser(fName);

4
src/common/backend/utils/cache/catcache.cpp vendored
View File

@ -1163,6 +1163,10 @@ void InitCatCachePhase2(CatCache* cache, bool touch_index)
*/
bool IndexScanOK(int cache_id)
{
if (SS_STANDBY_MODE) {
return false;
}
switch (cache_id) {
case INDEXRELID:

11
src/common/backend/utils/cache/knl_localsysdbcache.cpp vendored
View File

@ -43,7 +43,8 @@ public:
}
~LSCCloseCheck()
{
Assert(!EnableGlobalSysCache() || m_lsc_closed || g_instance.distribute_test_param_instance->elevel == PANIC);
Assert(!EnableGlobalSysCache() || m_lsc_closed || t_thrd.role == DMS_WORKER ||
g_instance.distribute_test_param_instance->elevel == PANIC);
}
void setCloseFlag(bool value)
{
@ -213,6 +214,14 @@ void CreateLocalSysDBCache()
#endif
}
}
#if defined(USE_ASSERT_CHECKING) && !defined(ENABLE_MEMORY_CHECK)
void CloseLSCCheck()
{
lsc_close_check.setCloseFlag(true);
}
#endif
static void ReleaseBadPtrList(bool isCommit);
static void ThreadNodeGroupCallback(Datum arg, int cacheid, uint32 hashvalue)
{

4
src/common/backend/utils/cache/partcache.cpp vendored
View File

@ -1769,8 +1769,10 @@ void PartitionSetNewRelfilenode(Relation parent, Partition part, TransactionId f
/* segment storage */
Assert(parent->storage_type == SEGMENT_PAGE);
isbucket = BUCKET_OID_IS_VALID(parent->rd_bucketoid) && !RelationIsCrossBucketIndex(parent);
Oid database_id = (ConvertToRelfilenodeTblspcOid(part->pd_part->reltablespace) == GLOBALTABLESPACE_OID) ?
InvalidOid : u_sess->proc_cxt.MyDatabaseId;
newrelfilenode = seg_alloc_segment(ConvertToRelfilenodeTblspcOid(part->pd_part->reltablespace),
u_sess->proc_cxt.MyDatabaseId, isbucket, InvalidBlockNumber);
database_id, isbucket, InvalidBlockNumber);
}

80
src/common/backend/utils/cache/relcache.cpp vendored
View File

@ -181,6 +181,7 @@
#include "storage/page_compression.h"
#include "storage/smgr/smgr.h"
#include "storage/smgr/segment.h"
#include "storage/file/fio_device.h"
#include "threadpool/threadpool.h"
#include "storage/tcap.h"
#include "utils/array.h"
@ -3077,7 +3078,11 @@ extern void formrdesc(const char* relationName, Oid relationReltype, bool isshar
if (IsBootstrapProcessingMode())
RelationMapUpdateMap(RelationGetRelid(relation), RelationGetRelid(relation), isshared, true);
relation->storage_type = HEAP_DISK;
if (t_thrd.shemem_ptr_cxt.ControlFile->bootstrap_segment) {
relation->storage_type = SEGMENT_PAGE;
} else {
relation->storage_type = HEAP_DISK;
}
/*
* initialize the relation lock manager information
@ -4724,8 +4729,10 @@ void RelationSetNewRelfilenode(Relation relation, TransactionId freezeXid, Multi
} else {
/* segment storage */
isbucket = BUCKET_OID_IS_VALID(relation->rd_bucketoid) && !RelationIsCrossBucketIndex(relation);
Oid database_id = (ConvertToRelfilenodeTblspcOid(relation->rd_rel->reltablespace) == GLOBALTABLESPACE_OID) ?
InvalidOid : u_sess->proc_cxt.MyDatabaseId;
newrelfilenode = seg_alloc_segment(ConvertToRelfilenodeTblspcOid(relation->rd_rel->reltablespace),
u_sess->proc_cxt.MyDatabaseId, isbucket, InvalidBlockNumber);
database_id, isbucket, InvalidBlockNumber);
}
// We must consider cudesc relation and delta relation when it is a CStore relation
@ -7022,6 +7029,10 @@ struct PublicationActions* GetRelationPublicationActions(Relation relation)
*/
static bool load_relcache_init_file(bool shared)
{
if (ENABLE_DMS) {
return false;
}
FILE* fp = NULL;
char initfilename[MAXPGPATH];
Relation* rels = NULL;
@ -7033,7 +7044,8 @@ static bool load_relcache_init_file(bool shared)
rc = snprintf_s(initfilename,
sizeof(initfilename),
sizeof(initfilename) - 1,
"global/%s.%u",
"%s/%s.%u",
GLOTBSDIR,
RELCACHE_INIT_FILENAME,
GRAND_VERSION_NUM);
else
@ -7416,6 +7428,10 @@ read_failed:
*/
static void write_relcache_init_file(bool shared)
{
if (ENABLE_DSS || SS_STANDBY_MODE) {
return;
}
FILE* fp = NULL;
char tempfilename[MAXPGPATH];
char finalfilename[MAXPGPATH];
@ -7444,7 +7460,8 @@ static void write_relcache_init_file(bool shared)
rc = snprintf_s(tempfilename,
sizeof(tempfilename),
sizeof(tempfilename) - 1,
"global/%s.%u.%lu",
"%s/%s.%u.%lu",
GLOTBSDIR,
RELCACHE_INIT_FILENAME,
GRAND_VERSION_NUM,
t_thrd.proc_cxt.MyProcPid);
@ -7452,7 +7469,8 @@ static void write_relcache_init_file(bool shared)
rc = snprintf_s(finalfilename,
sizeof(finalfilename),
sizeof(finalfilename) - 1,
"global/%s.%u",
"%s/%s.%u",
GLOTBSDIR,
RELCACHE_INIT_FILENAME,
GRAND_VERSION_NUM);
securec_check_ss(rc, "\0", "\0");
@ -7675,12 +7693,12 @@ void RelationCacheInitFilePreInvalidate(void)
if (unlink(initfilename) < 0) {
/*
* The file might not be there if no backend has been started since
* the last removal. But complain about failures other than ENOENT.
* Fortunately, it's not too late to abort the transaction if we can't
* get rid of the would-be-obsolete init file.
*/
if (errno != ENOENT)
* The file might not be there if no backend has been started since
* the last removal. But complain about failures other than ENOENT.
* Fortunately, it's not too late to abort the transaction if we can't
* get rid of the would-be-obsolete init file.
*/
if (!FILE_POSSIBLY_DELETED(errno))
ereport(ERROR, (errcode_for_file_access(), errmsg("could not remove cache file \"%s\": %m", initfilename)));
}
}
@ -7711,17 +7729,18 @@ void RelationCacheInitFileRemove(void)
* We zap the shared cache file too. In theory it can't get out of sync
* enough to be a problem, but in data-corruption cases, who knows ...
*/
rc = snprintf_s(path, sizeof(path), sizeof(path) - 1, "global/%s.%u", RELCACHE_INIT_FILENAME, GRAND_VERSION_NUM);
rc = snprintf_s(path, sizeof(path), sizeof(path) - 1, "%s/%s.%u",
GLOTBSDIR, RELCACHE_INIT_FILENAME, GRAND_VERSION_NUM);
securec_check_ss(rc, "\0", "\0");
unlink_initfile(path);
/* Scan everything in the default tablespace */
RelationCacheInitFileRemoveInDir("base");
RelationCacheInitFileRemoveInDir(DEFTBSDIR);
/* Scan the tablespace link directory to find non-default tablespaces */
dir = AllocateDir(tblspcdir);
dir = AllocateDir(TBLSPCDIR);
if (dir == NULL) {
ereport(LOG, (errmsg("could not open tablespace link directory \"%s\": %m", tblspcdir)));
ereport(LOG, (errmsg("could not open tablespace link directory \"%s\": %m", TBLSPCDIR)));
return;
}
@ -7730,17 +7749,32 @@ void RelationCacheInitFileRemove(void)
/* Scan the tablespace dir for per-database dirs */
#ifdef PGXC
/* Postgres-XC tablespaces include node name in path */
if (ENABLE_DSS) {
rc = snprintf_s(path,
sizeof(path),
sizeof(path) - 1,
"%s/%s/%s",
TBLSPCDIR,
de->d_name,
TABLESPACE_VERSION_DIRECTORY);
} else {
rc = snprintf_s(path,
sizeof(path),
sizeof(path) - 1,
"%s/%s/%s_%s",
TBLSPCDIR,
de->d_name,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
}
#else
rc = snprintf_s(path,
sizeof(path),
sizeof(path) - 1,
"%s/%s/%s_%s",
tblspcdir,
"%s/%s/%s",
TBLSPCDIR,
de->d_name,
TABLESPACE_VERSION_DIRECTORY,
g_instance.attr.attr_common.PGXCNodeName);
#else
rc = snprintf_s(
path, sizeof(path), sizeof(path) - 1, "%s/%s/%s", tblspcdir, de->d_name, TABLESPACE_VERSION_DIRECTORY);
TABLESPACE_VERSION_DIRECTORY);
#endif
securec_check_ss(rc, "\0", "\0");
RelationCacheInitFileRemoveInDir(path);
@ -7788,7 +7822,7 @@ static void unlink_initfile(const char* initfilename)
{
if (unlink(initfilename) < 0) {
/* It might not be there, but log any error other than ENOENT */
if (errno != ENOENT)
if (!FILE_POSSIBLY_DELETED(errno))
ereport(LOG, (errmsg("could not remove cache file \"%s\": %m", initfilename)));
}
}

84
src/common/backend/utils/cache/relmapper.cpp vendored
View File

@ -53,6 +53,7 @@
#include "miscadmin.h"
#include "storage/smgr/fd.h"
#include "storage/lock/lwlock.h"
#include "storage/file/fio_device.h"
#include "utils/inval.h"
#include "utils/relmapper.h"
@ -486,6 +487,17 @@ void RelationMapFinishBootstrap(void)
{
Assert(IsBootstrapProcessingMode());
if (ENABLE_DSS) {
char map_file_name[MAXPGPATH];
int rc = snprintf_s(map_file_name, sizeof(map_file_name), sizeof(map_file_name) - 1, "%s/global/%s",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, RELMAPPER_FILENAME);
securec_check_ss_c(rc, "\0", "\0");
if (dss_exist_file(map_file_name)) {
return;
}
}
knl_u_relmap_context *relmap_cxt = GetRelMapCxt();
/* Shouldn't be anything "pending" ... */
Assert(relmap_cxt->active_shared_updates->num_mappings == 0);
@ -604,10 +616,18 @@ void load_relmap_file(bool shared, RelMapFile *map)
bool isNewMap;
if (shared) {
rc = snprintf_s(
map_file_name[0], sizeof(map_file_name[0]), sizeof(map_file_name[0]) - 1, "global/%s", RELMAPPER_FILENAME);
rc = snprintf_s(map_file_name[0],
sizeof(map_file_name[0]),
sizeof(map_file_name[0]) - 1,
"%s/%s",
GLOTBSDIR,
RELMAPPER_FILENAME);
securec_check_ss(rc, "\0", "\0");
rc = snprintf_s(map_file_name[1], sizeof(map_file_name[1]), sizeof(map_file_name[1]) - 1, "global/%s",
rc = snprintf_s(map_file_name[1],
sizeof(map_file_name[1]),
sizeof(map_file_name[1]) - 1,
"%s/%s",
GLOTBSDIR,
RELMAPPER_FILENAME_BAK);
securec_check_ss(rc, "\0", "\0");
} else {
@ -630,10 +650,9 @@ void load_relmap_file(bool shared, RelMapFile *map)
// check backup file
if (stat(map_file_name[1], &stat_buf) != 0) {
if (ENOENT != errno) {
if (!FILE_POSSIBLY_DELETED(errno)) {
ereport(LOG, (errmsg("can not stat file \"%s\", ignore backup file", map_file_name[1])));
}
else {
} else {
fix_backup = true;
// switch to exclusive lock to do backup map file recovery
LWLockRelease(RelationMappingLock);
@ -761,11 +780,21 @@ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, b
knl_u_relmap_context *relmap_cxt = GetRelMapCxt();
for (int i = 0; i < 2; i++) {
if (shared) {
rc = snprintf_s(map_file_name, sizeof(map_file_name), sizeof(map_file_name) - 1, "global/%s", fname[i]);
rc = snprintf_s(map_file_name,
sizeof(map_file_name),
sizeof(map_file_name) - 1,
"%s/%s",
GLOTBSDIR,
fname[i]);
securec_check_ss_c(rc, "\0", "\0");
real_map = relmap_cxt->shared_map;
} else {
rc = snprintf_s(map_file_name, sizeof(map_file_name), sizeof(map_file_name) - 1, "%s/%s", dbpath, fname[i]);
rc = snprintf_s(map_file_name,
sizeof(map_file_name),
sizeof(map_file_name) - 1,
"%s/%s",
dbpath,
fname[i]);
securec_check_ss_c(rc, "\0", "\0");
real_map = relmap_cxt->local_map;
}
@ -962,7 +991,12 @@ static void recover_relmap_file(bool shared, bool backupfile, RelMapFile* real_m
}
if (shared) {
rc = snprintf_s(map_file_name, sizeof(map_file_name), sizeof(map_file_name) - 1, "global/%s", file_name);
rc = snprintf_s(map_file_name,
sizeof(map_file_name),
sizeof(map_file_name) - 1,
"%s/%s",
GLOTBSDIR,
file_name);
securec_check_ss(rc, "\0", "\0");
} else {
rc = snprintf_s(map_file_name,
@ -1076,14 +1110,25 @@ void relmap_redo(XLogReaderState* record)
static int WriteOldVersionRelmap(RelMapFile* map, int fd)
{
errno_t rc;
char* mapCache = (char*)palloc0(RELMAP_SIZE_OLD);
char* mapCache_ori = NULL;
char* mapCache = NULL;
if (ENABLE_DSS) {
mapCache_ori = (char*)palloc0(RELMAP_SIZE_OLD + ALIGNOF_BUFFER);
mapCache = (char *)BUFFERALIGN(mapCache_ori);
} else {
mapCache = (char*)palloc0(RELMAP_SIZE_OLD);
}
rc = memcpy_s(mapCache, RELMAP_SIZE_OLD, map, MAPPING_LEN_OLDMAP_HEAD);
securec_check(rc, "\0", "\0");
rc = memcpy_s(
mapCache + MAPPING_LEN_OLDMAP_HEAD, RELMAP_SIZE_OLD - MAPPING_LEN_OLDMAP_HEAD, &(map->crc), MAPPING_LEN_TAIL);
securec_check(rc, "\0", "\0");
int writeBytes = write(fd, mapCache, RELMAP_SIZE_OLD);
pfree_ext(mapCache);
if (ENABLE_DSS) {
pfree_ext(mapCache_ori);
} else {
pfree_ext(mapCache);
}
return writeBytes;
}
@ -1176,10 +1221,25 @@ static int32 ReadRelMapFile(RelMapFile* map, int fd, bool isNewMap)
static int32 WriteRelMapFile(RelMapFile* map, int fd)
{
int32 writeBytes = 0;
char* unalignRelMap = NULL;
RelMapFile* relMap = NULL;
if (IS_NEW_RELMAP(map->magic)) {
writeBytes = write(fd, map, sizeof(RelMapFile));
if (ENABLE_DSS) {
unalignRelMap = (char*)palloc0(sizeof(RelMapFile) + ALIGNOF_BUFFER);
relMap = (RelMapFile*)BUFFERALIGN(unalignRelMap);
errno_t err = memcpy_s(relMap, sizeof(RelMapFile), map, sizeof(RelMapFile));
securec_check(err, "\0", "\0");
} else {
relMap = map;
}
writeBytes = write(fd, relMap, sizeof(RelMapFile));
} else {
writeBytes = WriteOldVersionRelmap(map, fd);
}
if (unalignRelMap != NULL) {
pfree(unalignRelMap);
}
return writeBytes;
}

3
src/common/backend/utils/error/be_module.cpp
View File

@ -130,6 +130,9 @@ const module_data module_map[] = {{MOD_ALL, "ALL"},
{MOD_DISASTER_READ, "DISASTER_READ"},
{MODE_REPSYNC, "REPSYNC"},
{MOD_SQLPATCH, "SQLPATCH"},
{MOD_DMS, "DMS"},
{MOD_DSS, "DSS_API"},
{MOD_GPI, "GPI"},
/* add your module name above */
{MOD_MAX, "BACKEND"}};

6
src/common/backend/utils/error/elog.cpp
View File

@ -309,7 +309,7 @@ bool errstart(int elevel, const char* filename, int lineno, const char* funcname
*/
for (i = 0; i <= t_thrd.log_cxt.errordata_stack_depth; i++)
elevel = Max(elevel, t_thrd.log_cxt.errordata[i].elevel);
if (elevel == FATAL && t_thrd.role == JOB_WORKER) {
if (elevel == FATAL && (t_thrd.role == JOB_WORKER || t_thrd.role == DMS_WORKER)) {
elevel = ERROR;
}
}
@ -851,11 +851,14 @@ int errcode_for_file_access(void)
/* File not found */
case ENOENT: /* No such file or directory */
case ERR_DSS_FILE_NOT_EXIST: /* No such file in dss */
case ERR_DSS_DIR_NOT_EXIST: /* No such directory in dss */
edata->sqlerrcode = ERRCODE_UNDEFINED_FILE;
break;
/* Duplicate file */
case EEXIST: /* File exists */
case ERR_DSS_DIR_CREATE_DUPLICATED: /* File or directory already existed in DSS */
edata->sqlerrcode = ERRCODE_DUPLICATE_FILE;
break;
@ -870,6 +873,7 @@ int errcode_for_file_access(void)
/* Insufficient resources */
case ENOSPC: /* No space left on device */
case ERR_DSS_NO_SPACE: /* No space left on dss */
edata->sqlerrcode = ERRCODE_DISK_FULL;
break;

1
src/common/backend/utils/fmgr/dfmgr.cpp
View File

@ -28,6 +28,7 @@
#include "utils/hsearch.h"
#include "utils/memutils.h"
#include "utils/syscall_lock.h"
#include "storage/file/fio_device.h"
/* Max size of error message of dlopen */
#define DLERROR_MSG_MAX_LEN 512

3
src/common/backend/utils/init/globals.cpp
View File

@ -164,6 +164,9 @@ const uint32 STANDBY_STMTHIST_VERSION_NUM = 92777;
bool useLocalXid = false;
#endif
/* allow to store tables in segment storage while initdb */
bool EnableInitDBSegment = false;
/*
* EarlyBindingTLSVariables
* Bind static variables to another static TLS variable's address.

113
src/common/backend/utils/init/miscinit.cpp
View File

@ -62,6 +62,7 @@
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "gs_policy/policy_common.h"
#include "storage/file/fio_device.h"
#ifdef ENABLE_MULTIPLE_NODES
#include "tsdb/compaction/compaction_entry.h"
@ -1775,6 +1776,11 @@ void ValidatePgVersion(const char* path)
const char* version_string = PG_VERSION;
errno_t rc;
// skip in dss mode
if (ENABLE_DSS) {
return;
}
my_major = strtol(version_string, &endptr, 10);
if (*endptr == '.')
@ -1984,3 +1990,110 @@ bool contain_backend_version(uint32 version_number) {
version_number < V5R2C00_START_VERSION_NUM) ||
(version_number >= V5R2C00_BACKEND_VERSION_NUM));
}
void ss_initdwsubdir(char *dssdir, int instance_id)
{
int rc;
/* file correspanding to double write directory */
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwOldPath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwPathPrefix, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw_", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwSinglePath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw_single", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwBuildPath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw.build", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwUpgradePath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/dw_upgrade", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwBatchUpgradeMetaPath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/dw_batch_upgrade_meta", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwBatchUpgradeFilePath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/dw_batch_upgrade_files", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwMetaPath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw_meta", dssdir, instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.dw_subdir_cxt.dwExtChunkPath, MAXPGPATH, MAXPGPATH - 1,
"%s/pg_doublewrite%d/pg_dw_ext_chunk", dssdir, instance_id);
securec_check_ss(rc, "", "");
g_instance.datadir_cxt.dw_subdir_cxt.dwStorageType = (uint8)DEV_TYPE_DSS;
}
/*
* Check whether dss connect is successful.
*/
void initDSSConf(void)
{
if (!ENABLE_DSS) {
return;
}
if (!dss_exist_dir(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name)) {
ereport(FATAL, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("Could not connect dssserver, vgname: \"%s\", socketpath: \"%s\"",
g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name,
g_instance.attr.attr_storage.dss_attr.ss_dss_conn_path),
errhint("Check vgname and socketpath and restart later.")));
} else {
errno_t rc = EOK;
char *dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name;
rc = snprintf_s(g_instance.datadir_cxt.baseDir, MAXPGPATH, MAXPGPATH - 1, "%s/base", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.globalDir, MAXPGPATH, MAXPGPATH - 1, "%s/global", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.locationDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_location", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.tblspcDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_tblspc", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.clogDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_clog", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.csnlogDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_csnlog", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.serialDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_serial", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.twophaseDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_twophase", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.multixactDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_multixact", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.xlogDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", dssdir,
g_instance.attr.attr_storage.dms_attr.instance_id);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.controlPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", dssdir);
securec_check_ss(rc, "", "");
rc = snprintf_s(g_instance.datadir_cxt.controlBakPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control.backup",
dssdir);
securec_check_ss(rc, "", "");
ss_initdwsubdir(dssdir, g_instance.attr.attr_storage.dms_attr.instance_id);
}
/* set xlog seg size to 1GB */
XLogSegmentSize = DSS_XLOG_SEG_SIZE;
}

9
src/common/backend/utils/init/postinit.cpp
View File

@ -72,6 +72,7 @@
#include "storage/procsignal.h"
#include "storage/sinvaladt.h"
#include "storage/smgr/smgr.h"
#include "storage/file/fio_device.h"
#include "tcop/tcopprot.h"
#include "threadpool/threadpool.h"
#include "utils/acl.h"
@ -609,7 +610,9 @@ void BaseInit(void)
InitSync();
smgrinit();
InitBufferPoolAccess();
undo::UndoLogInit();
if (!ENABLE_DSS) {
undo::UndoLogInit();
}
}
/* -------------------------------------
@ -2657,7 +2660,7 @@ void PostgresInitializer::SetDatabasePath()
securec_check_ss(rcs, "\0", "\0");
pgaudit_user_login(FALSE, (char*)m_username, m_details);
if (errno == ENOENT)
if (FILE_POSSIBLY_DELETED(errno))
ereport(FATAL,
(errcode(ERRCODE_UNDEFINED_DATABASE),
errmsg("database \"%s\" does not exist", m_dbname),
@ -2915,5 +2918,3 @@ void PostgresInitializer::InitBarrierCreator()
return;
}

62
src/common/backend/utils/misc/guc.cpp
View File

@ -188,8 +188,10 @@
#define WRITE_CONFIG_LOCK_LEN (1024 * 1024)
#ifdef EXEC_BACKEND
#define CONFIG_EXEC_PARAMS "global/config_exec_params"
#define CONFIG_EXEC_PARAMS_NEW "global/config_exec_params.new"
#define CONFIG_EXEC_PARAMS (g_instance.attr.attr_storage.dss_attr.ss_enable_dss ? \
((char*)"config_exec_params") : ((char*)"global/config_exec_params"))
#define CONFIG_EXEC_PARAMS_NEW (g_instance.attr.attr_storage.dss_attr.ss_enable_dss ? \
((char*)"config_exec_params.new") : ((char*)"global/config_exec_params.new"))
#endif
/* upper limit for GUC variables measured in kilobytes of memory */
@ -445,6 +447,8 @@ static void assign_syslog_facility(int newval, void* extra);
static void assign_syslog_ident(const char* newval, void* extra);
static void assign_session_replication_role(int newval, void* extra);
static bool check_client_min_messages(int* newval, void** extra, GucSource source);
static bool check_default_transaction_isolation(int* newval, void** extra, GucSource source);
static bool check_enable_stmt_track(bool* newval, void** extra, GucSource source);
static bool check_debug_assertions(bool* newval, void** extra, GucSource source);
#ifdef USE_BONJOUR
static bool check_bonjour(bool* newval, void** extra, GucSource source);
@ -999,6 +1003,8 @@ const char* const config_group_names[] = {
gettext_noop("Instruments Options"),
gettext_noop("Column Encryption"),
gettext_noop("Compress Options"),
/* SHARED_STORAGE_OPTIONS */
gettext_noop("Shared Storage Options"),
#ifdef PGXC
/* DATA_NODES */
gettext_noop("Datanodes and Connection Pooling"),
@ -1125,7 +1131,7 @@ static void InitConfigureNamesBool()
gettext_noop("Enable full/slow sql feature"), NULL},
&u_sess->attr.attr_common.enable_stmt_track,
true,
NULL,
check_enable_stmt_track,
NULL,
NULL},
{{"track_stmt_parameter",
@ -4056,7 +4062,7 @@ static void InitConfigureNamesEnum()
&u_sess->attr.attr_common.DefaultXactIsoLevel,
XACT_READ_COMMITTED,
isolation_level_options,
NULL,
check_default_transaction_isolation,
NULL,
NULL},
@ -5058,6 +5064,32 @@ static int guc_name_compare(const char* namea, const char* nameb)
return 0;
}
static void parseDmsInstanceCount()
{
if (!ENABLE_DMS) {
return;
}
char *dms_url = g_instance.attr.attr_storage.dms_attr.interconnect_url;
List *l = NULL;
char *url = pstrdup(dms_url);
if (!SplitIdentifierString(url, ',', &l)) {
pfree(url);
g_instance.attr.attr_storage.dms_attr.inst_count = 1;
return;
}
if (list_length(l) == 0 || list_length(l) > DMS_MAX_INSTANCE) {
pfree(url);
g_instance.attr.attr_storage.dms_attr.inst_count = 1;
return;
}
g_instance.attr.attr_storage.dms_attr.inst_count = list_length(l);
pfree(url);
return;
}
/*
* Initiaize Postmaster level GUC options during postmaster proc.
*
@ -5072,6 +5104,11 @@ void InitializePostmasterGUC()
g_instance.attr.attr_storage.enable_gtm_free = true;
#endif
g_instance.attr.attr_network.PoolerPort = g_instance.attr.attr_network.PostPortNumber + 1;
parseDmsInstanceCount();
#ifndef USE_ASSERT_CHECKING
/* in Release, this param is ON and undisclosed */
g_instance.attr.attr_storage.dms_attr.enable_reform = true;
#endif
}
/*
@ -11196,6 +11233,23 @@ static bool check_client_min_messages(int* newval, void** extra, GucSource sourc
return true;
}
static bool check_default_transaction_isolation(int *newval, void **extra, GucSource source)
{
if (ENABLE_DMS && *newval != XACT_READ_COMMITTED) {
ereport(ERROR, (errmsg("Only support read committed transaction isolation level while DMS and DSS enabled")));
return false;
}
return true;
}
static bool check_enable_stmt_track(bool *newval, void **extra, GucSource source)
{
if (ENABLE_DMS && !SS_MY_INST_IS_MASTER) {
*newval = false;
}
return true;
}
static bool check_debug_assertions(bool* newval, void** extra, GucSource source)
{
#ifndef USE_ASSERT_CHECKING

322
src/common/backend/utils/misc/guc/guc_storage.cpp
View File

@ -204,6 +204,12 @@ static bool check_and_assign_namespace_oids(List* elemlist);
static bool check_and_assign_general_oids(List* elemlist);
static int GetLengthAndCheckReplConn(const char* ConnInfoList);
static bool check_ss_interconnect_type(char **newval, void **extra, GucSource source);
static bool check_ss_rdma_work_config(char** newval, void** extra, GucSource source);
static bool check_ss_dss_vg_name(char** newval, void** extra, GucSource source);
static bool check_ss_dss_conn_path(char** newval, void** extra, GucSource source);
static bool check_ss_enable_ssl(bool* newval, void** extra, GucSource source);
#ifndef ENABLE_MULTIPLE_NODES
static void assign_dcf_election_timeout(int newval, void* extra);
static void assign_dcf_auto_elc_priority_en(int newval, void* extra);
@ -975,6 +981,83 @@ static void InitStorageConfigureNamesBool()
NULL,
NULL},
{{"ss_enable_dss",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Whether use dss"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dss_attr.ss_enable_dss,
false,
NULL,
NULL,
NULL},
{{"ss_enable_dms",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Whether use dms"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.enable_dms,
false,
NULL,
NULL,
NULL},
{{"ss_enable_catalog_centralized",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Whether dms catalog stored centralized or distributed"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.enable_catalog_centralized,
true,
NULL,
NULL,
NULL},
#ifdef USE_ASSERT_CHECKING
{{"ss_enable_reform",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Whether use dms reform"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.enable_reform,
true,
NULL,
NULL,
NULL},
#endif
{{"ss_enable_ssl",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Whether use dms ssl"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.enable_ssl,
true,
check_ss_enable_ssl,
NULL,
NULL},
{{"ss_enable_log_level",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Set dms and dss log level to LOG"),
NULL},
&g_instance.attr.attr_storage.dms_attr.enable_log_level,
false,
NULL,
NULL,
NULL},
#ifdef USE_ASSERT_CHECKING
{{"enable_hashbucket",
PGC_SUSET,
@ -3289,6 +3372,62 @@ static void InitStorageConfigureNamesInt()
NULL,
NULL,
NULL},
{{"ss_instance_id",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the instance id."),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.instance_id,
0,
0,
63,
NULL,
NULL,
NULL},
{{"ss_interconnect_channel_count",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets ss mes interconnect channel count"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.channel_count,
16,
1,
32,
NULL,
NULL,
NULL},
{{"ss_work_thread_count",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets ss mes work thread count"),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.work_thread_count,
32,
16,
128,
NULL,
NULL,
NULL},
{{"ss_recv_msg_pool_size",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the ss receive message pool size (KB)"),
NULL,
GUC_SUPERUSER_ONLY | GUC_UNIT_KB},
&g_instance.attr.attr_storage.dms_attr.recv_msg_pool_size,
16 * 1024,
1024,
1024 * 1024,
NULL,
NULL,
NULL},
/* End-of-list marker */
{{NULL,
(GucContext)0,
@ -4089,6 +4228,76 @@ static void InitStorageConfigureNamesString()
check_logical_decode_options_default,
assign_logical_decode_options_default,
NULL},
{{"ss_dss_vg_name",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the vg name of DSS node."),
NULL},
&g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name,
"",
check_ss_dss_vg_name,
NULL,
NULL},
{{"ss_dss_conn_path",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the socket file path of DSS node."),
NULL},
&g_instance.attr.attr_storage.dss_attr.ss_dss_conn_path,
"UDS:/tmp/.dss_unix_d_socket",
check_ss_dss_conn_path,
NULL,
NULL},
{{"ss_interconnect_url",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the url to connect to ss."),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.interconnect_url,
"0:127.0.0.1:1611",
NULL,
NULL,
NULL},
{{"ss_interconnect_type",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets the type of connect to ss, range: TCP, RDMA."),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.interconnect_type,
"TCP",
check_ss_interconnect_type,
NULL,
NULL},
{{"ss_rdma_work_config",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets config with digit number: A B."),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.rdma_work_config,
"",
check_ss_rdma_work_config,
NULL,
NULL},
{{"ss_ock_log_path",
PGC_POSTMASTER,
NODE_SINGLENODE,
SHARED_STORAGE_OPTIONS,
gettext_noop("Sets config with string file path."),
NULL,
GUC_SUPERUSER_ONLY},
&g_instance.attr.attr_storage.dms_attr.ock_log_path,
"",
NULL,
NULL,
NULL},
{{NULL,
(GucContext)0,
(GucNodeType)0,
@ -5380,6 +5589,119 @@ static int GetLengthAndCheckReplConn(const char* ConnInfoList)
return repl_len;
}
static bool check_ss_interconnect_type(char **newval, void **extra, GucSource source)
{
return (strcmp("TCP", *newval) == 0 || strcmp("RDMA", *newval) == 0);
}
static inline bool check_digit_text(char *str, uint32* len)
{
uint32 idx = 0;
if (str == NULL) {
*len = 0;
return true;
}
while (*str != '\0' && *str == ' ') {
idx++;
++str;
}
while (*str != '\0') {
if (*str == ' ') {
break;
}
if (*str >= '0' && *str <= '9') {
++str;
++idx;
} else {
*len = 0;
return false;
}
}
*len = idx;
return true;
}
static bool check_ss_rdma_work_config(char** newval, void** extra, GucSource source)
{
uint32 idx1 = 0;
uint32 idx2 = 0;
if (!check_digit_text(*newval, &idx1)) {
return false;
}
if (!check_digit_text(*newval + idx1, &idx2)) {
return false;
}
return true;
}
static bool check_ss_dss_vg_name(char** newval, void** extra, GucSource source)
{
char *ReplStr = NULL;
char *ptr = NULL;
if (newval == NULL || *newval == NULL || **newval == '\0') {
return true;
}
ReplStr = pstrdup(*newval);
if (*ReplStr == '+') {
ptr = ReplStr;
while (*ptr != '\0') {
if (*ptr == '/') {
break;
}
ptr++;
}
if (*ptr == '\0') {
pfree(ReplStr);
return true;
}
}
ereport(ERROR, (errmsg("DSS vg name must start with '+' and not comtain '\\'.")));
pfree(ReplStr);
return false;
}
static bool check_ss_dss_conn_path(char** newval, void** extra, GucSource source)
{
char *ReplStr = NULL;
int strlen = sizeof("UDS:") - 1;
if (newval == NULL || *newval == NULL || **newval == '\0') {
ereport(ERROR, (errmsg("DSS conn path can not be NULL.")));
return false;
}
canonicalize_path(*newval);
ReplStr = pstrdup(*newval);
if (strncmp(ReplStr, "UDS:", strlen) != 0) {
ereport(ERROR, (errmsg("DSS conn path format: \"UDS:socket_domain\"")));
return false;
}
if (!ENABLE_DSS) {
return true;
}
ReplStr = ReplStr + strlen;
if (is_absolute_path(ReplStr)) {
return true;
} else {
ereport(ERROR, (errmsg("DSS conn path must be absolute path.")));
}
return false;
}
static bool check_ss_enable_ssl(bool *newval, void **extra, GucSource source)
{
if (!*newval) {
ereport(WARNING, (errmsg("The SSL connection will be disabled during build, which brings security risks.")));
}
return true;
}
#ifndef ENABLE_MULTIPLE_NODES
static void assign_dcf_election_timeout(int newval, void* extra)

2
src/common/backend/utils/misc/pg_controldata.cpp
View File

@ -140,7 +140,7 @@ ControlFileData* GetControlfile(const char *dataDir, bool *crc_ok_p)
AssertArg(crc_ok_p);
controlFile = (ControlFileData*)palloc(sizeof(ControlFileData));
errno_t rc = snprintf_s(controlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/global/pg_control", dataDir);
errno_t rc = snprintf_s(controlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dataDir, XLOG_CONTROL_FILE);
securec_check_ss_c(rc, "\0", "\0");
fd = OpenControlFile(controlFilePath);

19
src/common/backend/utils/misc/postgresql_single.conf.sample
View File

@ -810,3 +810,22 @@ job_queue_processes = 10 # Number of concurrent jobs, optional: [0..1000]
#plsql_show_all_error=off
#enable_seqscan_fusion = off
#enable_cachedplan_mgr=on
#------------------------------------------------------------------------------
# SHARED STORAGE OPTIONS
#------------------------------------------------------------------------------
#ss_enable_dms = off
#ss_enable_dss = off
#ss_enable_reform = on
#ss_enable_ssl = on
#ss_enable_catalog_centralized = on
#ss_instance_id = 0
#ss_dss_vg_name = ''
#ss_dss_conn_path = ''
#ss_interconnect_channel_count = 16
#ss_work_thread_count = 32
#ss_recv_msg_pool_size = 16MB
#ss_interconnect_type = 'TCP'
#ss_interconnect_url = '0:127.0.0.1:1611'
#ss_rdma_work_config = ''
#ss_ock_log_path = ''

1
src/common/backend/utils/mmgr/memprot.cpp
View File

@ -956,6 +956,7 @@ void gs_memprot_reserved_backend(int avail_mem)
int reserved_thread_count = g_instance.attr.attr_network.ReservedBackends +
NUM_CMAGENT_PROCS + wal_thread_count +
NUM_DCF_CALLBACK_PROCS +
NUM_DMS_CALLBACK_PROCS +
g_instance.attr.attr_storage.max_wal_senders;
/* reserve 10MB per-thread for sysadmin user */
reserved_mem += reserved_thread_count * 10;

36
src/common/backend/utils/time/snapmgr.cpp
View File

@ -59,6 +59,10 @@
#include "utils/memutils.h"
#include "utils/snapmgr.h"
#include "utils/syscache.h"
#include "ddes/dms/ss_common_attr.h"
#include "ddes/dms/ss_transaction.h"
#include "storage/file/fio_device.h"
#ifdef PGXC
#include "pgxc/pgxc.h"
#endif
@ -82,19 +86,21 @@ typedef struct ActiveSnapshotElt {
static THR_LOCAL bool RegisterStreamSnapshot = false;
/* Define pathname of exported-snapshot files */
#define SNAPSHOT_EXPORT_DIR "pg_snapshots"
#define SNAPSHOT_EXPORT_DIR (g_instance.datadir_cxt.snapshotsDir)
/* Structure holding info about exported snapshot. */
typedef struct ExportedSnapshot {
char *snapfile;
Snapshot snapshot;
} ExportedSnapshot;
#define XactExportFilePath(path, xid, num, suffix) \
{ \
int rc = snprintf_s(path, \
sizeof(path), \
sizeof(path) - 1, \
SNAPSHOT_EXPORT_DIR "/%08X%08X-%d%s", \
"%s/%08X%08X-%d%s", \
(g_instance.datadir_cxt.snapshotsDir), \
(uint32)((xid) >> 32), \
(uint32)(xid), \
(num), \
@ -175,7 +181,7 @@ bool IsXidVisibleInGtmLiteLocalSnapshot(TransactionId xid, Snapshot snapshot,
return false;
}
static void RecheckXidFinish(TransactionId xid, CommitSeqNo csn)
void RecheckXidFinish(TransactionId xid, CommitSeqNo csn)
{
if (TransactionIdIsInProgress(xid)) {
ereport(defence_errlevel(), (errmsg("transaction id %lu is still running, "
@ -208,7 +214,11 @@ bool XidVisibleInSnapshot(TransactionId xid, Snapshot snapshot, TransactionIdSta
#endif
loop:
csn = TransactionIdGetCommitSeqNo(xid, false, true, false, snapshot);
if (SS_STANDBY_MODE) {
csn = SSTransactionIdGetCommitSeqNo(xid, false, true, false, snapshot, sync);
} else {
csn = TransactionIdGetCommitSeqNo(xid, false, true, false, snapshot);
}
#ifdef XIDVIS_DEBUG
ereport(DEBUG1,
@ -228,6 +238,10 @@ loop:
else
return false;
} else if (COMMITSEQNO_IS_COMMITTING(csn)) {
/* SS master node would've already sync-waited, so this should never happen */
if (SS_STANDBY_MODE) {
ereport(FATAL, (errmsg("SS xid %lu's csn %lu is still COMMITTING after Master txn waited.", xid, csn)));
}
if (looped) {
ereport(DEBUG1, (errmsg("transaction id %lu's csn %ld is changed to ABORT after lockwait.", xid, csn)));
/* recheck if transaction id is finished */
@ -346,9 +360,17 @@ bool CommittedXidVisibleInSnapshot(TransactionId xid, Snapshot snapshot, Buffer
}
loop:
csn = TransactionIdGetCommitSeqNo(xid, true, true, false, snapshot);
if (SS_STANDBY_MODE) {
csn = SSTransactionIdGetCommitSeqNo(xid, true, true, false, snapshot, NULL);
} else {
csn = TransactionIdGetCommitSeqNo(xid, true, true, false, snapshot);
}
if (COMMITSEQNO_IS_COMMITTING(csn)) {
/* SS master node would've already sync-waited, so this should never happen */
if (SS_STANDBY_MODE) {
ereport(FATAL, (errmsg("SS xid %lu's csn %lu is still COMMITTING after Master txn waited.", xid, csn)));
}
if (looped) {
ereport(WARNING,
(errmsg("committed transaction id %lu's csn %lu"
@ -1544,7 +1566,7 @@ void ImportSnapshot(const char* idstr)
ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid snapshot identifier: \"%s\"", idstr)));
/* OK, read the file */
int rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, SNAPSHOT_EXPORT_DIR "/%s", idstr);
int rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", SNAPSHOT_EXPORT_DIR, idstr);
securec_check_ss(rc, "", "");
f = AllocateFile(path, PG_BINARY_R);
@ -1668,7 +1690,7 @@ void DeleteAllExportedSnapshotFiles(void)
if (strcmp(s_de->d_name, ".") == 0 || strcmp(s_de->d_name, "..") == 0)
continue;
rc = snprintf_s(buf, MAXPGPATH, MAXPGPATH - 1, SNAPSHOT_EXPORT_DIR "/%s", s_de->d_name);
rc = snprintf_s(buf, MAXPGPATH, MAXPGPATH - 1, "%s/%s", SNAPSHOT_EXPORT_DIR, s_de->d_name);
securec_check_ss(rc, "", "");
/* Again, unlink failure is not worthy of FATAL */
if (unlink(buf))

11
src/common/interfaces/libpq/CMakeLists.txt
View File

@ -50,6 +50,8 @@ execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/backend/libpq/md5.cpp ${CMAKE_CURRENT_SOURCE_DIR}/md5.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/backend/libpq/sha2.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sha2.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/port/pgstrcasecmp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgstrcasecmp.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/file/fio_device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fio_device.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/fio_dss.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fio_dss.cpp
)
# libpq.a
@ -87,6 +89,9 @@ list(APPEND TGT_pq_SRC
${CMAKE_CURRENT_SOURCE_DIR}/frontend_parser/fe-wchar.cpp
${CMAKE_CURRENT_SOURCE_DIR}/frontend_parser/frontend_mbutils.cpp
${CMAKE_CURRENT_SOURCE_DIR}/frontend_parser/wstrncmp.cpp
${CMAKE_CURRENT_SOURCE_DIR}/fio_device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/fio_dss.cpp
)
set(TGT_pq_INC
@ -98,6 +103,7 @@ set(TGT_pq_INC
${LIBEDIT_INCLUDE_PATH}
${ZLIB_INCLUDE_PATH}
${PROJECT_SRC_DIR}/include/libpq
${PROJECT_SRC_DIR}/include/storage/file
if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON")
${JAVA_HOME}/include
${JAVA_HOME}/include/linux
@ -142,6 +148,8 @@ execute_process(
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/interfaces/libpq/fe-protocol2.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libpq_ce/fe-protocol2.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/interfaces/libpq/fe-protocol3.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libpq_ce/fe-protocol3.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/common/interfaces/libpq/fe-secure.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libpq_ce/fe-secure.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/file/fio_device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fio_device.cpp
COMMAND ln -fs ${PROJECT_SRC_DIR}/gausskernel/storage/dss/fio_dss.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fio_dss.cpp
)
SET(TGT_pq_ce_INC
@ -187,6 +195,9 @@ list(APPEND TGT_pq_ce_SRC
${CMAKE_CURRENT_SOURCE_DIR}/gs_syscall_lock.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gs_readdir.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cl_state.cpp
${CMAKE_CURRENT_SOURCE_DIR}/fio_device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/fio_dss.cpp
)
set(pq_ce_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DFRONTEND_PARSER -DUNSAFE_STAT_OK -DHAVE_CE -DWORDS_BIGENDIAN -DSO_MAJOR_VERSION=5)
set(pq_ce_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${LIB_SECURE_OPTIONS} ${CHECK_OPTIONS} -fstack-protector-all)

3
src/common/interfaces/libpq/Makefile
View File

@ -63,7 +63,8 @@ override CFLAGS := $(filter-out -fPIE, $(CFLAGS)) -fPIC -fstack-protector-all
# OBJS from this file.
OBJS= fe-auth.o fe-connect.o fe-exec.o fe-misc.o fe-print.o fe-lobj.o \
fe-protocol2.o fe-protocol3.o pqexpbuffer.o pqsignal.o fe-secure.o \
libpq-events.o
libpq-events.o \
$(top_builddir)/src/gausskernel/storage/dss/fio_dss.o $(top_builddir)/src/gausskernel/storage/file/fio_device.o
# libpgport C files we always use
OBJS += chklocale.o inet_net_ntop.o noblock.o pgstrcasecmp.o thread.o cipher.o path.o pgsleep.o
# libpgport C files that are needed if identified by configure

2
src/common/pl/plpgsql/src/pl_handler.cpp
View File

@ -209,6 +209,7 @@ void InsertGsSource(Oid objId, Oid nspid, const char* name, const char* type, bo
"EXCEPTION WHEN OTHERS THEN NULL; \n");
appendStringInfoString(&str, "end;");
List* rawParseList = raw_parser(str.data);
pfree_ext(str.data);
DoStmt* stmt = (DoStmt *)linitial(rawParseList);
int save_compile_status = getCompileStatus();
int save_compile_list_length = list_length(u_sess->plsql_cxt.compile_context_list);
@ -242,7 +243,6 @@ void InsertGsSource(Oid objId, Oid nspid, const char* name, const char* type, bo
if (temp != NULL) {
MemoryContextSwitchTo(temp);
}
pfree_ext(str.data);
}
static void PkgInsertGsSource(Oid pkgOid, bool isSpec, bool status)
{

Some files were not shown because too many files have changed in this diff Show More