p75086429
/
openGauss-server
forked from openGauss-Ecosystem/openGauss-server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!2211 提升external sort性能 

Merge pull request !2211 from Oreo/pjr_commit_sort3
This commit is contained in:
opengauss-bot 2022-10-18 09:18:59 +00:00 committed by Gitee
parent 709e1dadf5 964fac0c6f
commit 8a6e620831
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
9 changed files with 511 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/common/backend/utils/sort/logtape.cpp
View File

@ -144,6 +144,8 @@ typedef struct LogicalTape {
int max_size; /* highest useful, safe buffer_size */
int pos; /* next read/write position in buffer */
int nbytes; /* total # of valid bytes in buffer */
int read_buffer_size;
} LogicalTape;
/*
@ -979,3 +981,28 @@ long LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
return lts->nBlocksAllocated - lts->nHoleBlocks;
}
/*
* Set buffer size to use, when reading from given tape.
*/
void
LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum, size_t avail_mem)
{
LogicalTape *lt;
Assert(tapenum >= 0 && tapenum < lts->nTapes);
lt = &lts->tapes[tapenum];
/*
* The buffer size must be a multiple of BLCKSZ in size, so round the
* given value down to nearest BLCKSZ. Make sure we have at least one
* page. Also, don't go above MaxAllocSize, to avoid erroring out. A
* multi-gigabyte buffer is unlikely to be helpful, anyway.
*/
if (avail_mem < BLCKSZ)
avail_mem = BLCKSZ;
if (avail_mem > MaxAllocSize)
avail_mem = MaxAllocSize;
avail_mem -= avail_mem % BLCKSZ;
lt->read_buffer_size = avail_mem;
}

775
src/common/backend/utils/sort/tuplesort.cpp
View File

File diff suppressed because it is too large Load Diff

14
src/gausskernel/optimizer/commands/cluster.cpp
View File

@ -1892,11 +1892,9 @@ double CopyUHeapDataInternal(Relation oldHeap, Relation oldIndex, Relation newHe
for (;;) {
UHeapTuple utuple;
bool shouldfree = false;
CHECK_FOR_INTERRUPTS();
utuple = (UHeapTuple)tuplesort_getheaptuple(tuplesort, true, &shouldfree);
utuple = (UHeapTuple)tuplesort_getheaptuple(tuplesort, true);
if (utuple == NULL)
break;
@ -1904,9 +1902,6 @@ double CopyUHeapDataInternal(Relation oldHeap, Relation oldIndex, Relation newHe
ReformAndRewriteUTuple(utuple, oldTupDesc, newTupDesc, values, isnull, newHeap->rd_rel->relhasoids,
rwstate);
if (shouldfree)
UHeapFreeTuple(utuple);
}
tuplesort_end(tuplesort);
@ -2193,19 +2188,14 @@ double copy_heap_data_internal(Relation OldHeap, Relation OldIndex, Relation New
for (;;) {
HeapTuple tuple;
bool shouldfree = false;
CHECK_FOR_INTERRUPTS();
tuple = (HeapTuple)tuplesort_getheaptuple(tuplesort, true, &shouldfree);
tuple = (HeapTuple)tuplesort_getheaptuple(tuplesort, true);
if (tuple == NULL)
break;
reform_and_rewrite_tuple(
tuple, oldTupDesc, newTupDesc, values, isnull, NewHeap->rd_rel->relhasoids, rwstate);
if (shouldfree)
heap_freetuple(tuple);
}
tuplesort_end(tuplesort);

5
src/gausskernel/storage/access/hash/hashsort.cpp
View File

@ -113,13 +113,10 @@ void _h_spool(HSpool *hspool, ItemPointer self, Datum *values, const bool *isnul
void _h_indexbuild(HSpool *hspool, Relation heapRel)
{
IndexTuple itup;
bool should_free = false;
tuplesort_performsort(hspool->sortstate);
while ((itup = tuplesort_getindextuple(hspool->sortstate, true, &should_free)) != NULL) {
while ((itup = tuplesort_getindextuple(hspool->sortstate, true)) != NULL) {
_hash_doinsert(hspool->index, itup, heapRel);
if (should_free)
pfree(itup);
}
}

24
src/gausskernel/storage/access/nbtree/nbtsort.cpp
View File

@ -820,8 +820,6 @@ static void _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
bool merge = (btspool2 != NULL);
IndexTuple itup = NULL;
IndexTuple itup2 = NULL;
bool should_free = false;
bool should_free2 = false;
bool load1 = false;
TupleDesc tupdes = RelationGetDescr(wstate->index);
int keysz = IndexRelationGetNumberOfKeyAttributes(wstate->index);
@ -834,8 +832,8 @@ static void _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
*
* the preparation of merge
*/
itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true, &should_free2);
itup = tuplesort_getindextuple(btspool->sortstate, true);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
indexScanKey = _bt_mkscankey_nodata(wstate->index);
for (;;) {
@ -850,33 +848,21 @@ static void _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
if (load1) {
_bt_buildadd(wstate, state, itup);
if (should_free) {
pfree(itup);
itup = NULL;
}
itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free);
itup = tuplesort_getindextuple(btspool->sortstate, true);
} else {
_bt_buildadd(wstate, state, itup2);
if (should_free2) {
pfree(itup2);
itup2 = NULL;
}
itup2 = tuplesort_getindextuple(btspool2->sortstate, true, &should_free2);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
}
}
_bt_freeskey(indexScanKey);
} else {
/* merge is unnecessary */
while ((itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free)) != NULL) {
while ((itup = tuplesort_getindextuple(btspool->sortstate, true)) != NULL) {
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
_bt_buildadd(wstate, state, itup);
if (should_free) {
pfree(itup);
itup = NULL;
}
}
}

24
src/gausskernel/storage/access/ubtree/ubtsort.cpp
View File

@ -750,8 +750,6 @@ static void UBTreeLoad(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2
bool merge = (btspool2 != NULL);
IndexTuple itup = NULL;
IndexTuple itup2 = NULL;
bool should_free = false;
bool should_free2 = false;
bool load1 = false;
TupleDesc tupdes = RelationGetDescr(wstate->index);
int keysz = IndexRelationGetNumberOfKeyAttributes(wstate->index);
@ -763,8 +761,8 @@ static void UBTreeLoad(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2
*
* the preparation of merge
*/
itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true, &should_free2);
itup = tuplesort_getindextuple(btspool->sortstate, true);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
for (;;) {
if (itup == NULL && itup2 == NULL) {
@ -780,32 +778,20 @@ static void UBTreeLoad(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2
if (load1) {
UBTreeBuildAdd(wstate, state, itup, false);
if (should_free) {
pfree(itup);
itup = NULL;
}
itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free);
itup = tuplesort_getindextuple(btspool->sortstate, true);
} else {
UBTreeBuildAdd(wstate, state, itup2, false);
if (should_free2) {
pfree(itup2);
itup2 = NULL;
}
itup2 = tuplesort_getindextuple(btspool2->sortstate, true, &should_free2);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
}
}
} else {
/* merge is unnecessary */
while ((itup = tuplesort_getindextuple(btspool->sortstate, true, &should_free)) != NULL) {
while ((itup = tuplesort_getindextuple(btspool->sortstate, true)) != NULL) {
/* When we see first tuple, create first index page */
if (state == NULL)
state = UBTreePageState(wstate, 0);
UBTreeBuildAdd(wstate, state, itup, false);
if (should_free) {
pfree(itup);
itup = NULL;
}
}
}

2
src/include/utils/logtape.h
View File

@ -68,5 +68,7 @@ extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size
extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum, long blocknum, int offset);
extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
extern void LogicalTapeAssignReadBufferSize(LogicalTapeSet *lts, int tapenum,
size_t bufsize);
#endif /* LOGTAPE_H */

4
src/include/utils/tuplesort.h
View File

@ -152,8 +152,8 @@ extern void tuplesort_performsort(Tuplesortstate* state);
extern bool tuplesort_gettupleslot(Tuplesortstate* state, bool forward, TupleTableSlot* slot, Datum* abbrev);
extern bool tuplesort_gettupleslot_into_tuplestore(
Tuplesortstate* state, bool forward, TupleTableSlot* slot, Datum* abbrev, Tuplestorestate* tstate);
extern void* tuplesort_getheaptuple(Tuplesortstate* state, bool forward, bool* should_free);
extern IndexTuple tuplesort_getindextuple(Tuplesortstate* state, bool forward, bool* should_free);
extern void* tuplesort_getheaptuple(Tuplesortstate* state, bool forward);
extern IndexTuple tuplesort_getindextuple(Tuplesortstate* state, bool forward);
extern bool tuplesort_getdatum(Tuplesortstate* state, bool forward, Datum* val, bool* isNull);
extern void tuplesort_end(Tuplesortstate* state);

2
src/test/regress/pg_regress.cpp
View File

@ -5238,7 +5238,7 @@ static void check_global_variables()
}
}
#define BASE_PGXC_LIKE_MACRO_NUM 1399
#define BASE_PGXC_LIKE_MACRO_NUM 1398
static void check_pgxc_like_macros()
{
#ifdef BUILD_BY_CMAKE