From 8613865e11f754257313ca6c7b527ac4ab5f3139 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 28 Nov 2016 13:20:59 +0500 Subject: [PATCH 1/3] xvac --- gin_delete.sql | 1 + gin_init.sql | 14 ++ gin_shot.sql | 12 ++ gin_vacuum.sql | 3 + prevacuum.sh | 2 + src/backend/access/gin/ginvacuum.c | 257 +++++++++++++++-------------- status.sh | 2 + test.sh | 4 + 8 files changed, 171 insertions(+), 124 deletions(-) create mode 100644 gin_delete.sql create mode 100644 gin_init.sql create mode 100644 gin_shot.sql create mode 100644 gin_vacuum.sql create mode 100644 prevacuum.sh create mode 100644 status.sh create mode 100644 test.sh diff --git a/gin_delete.sql b/gin_delete.sql new file mode 100644 index 0000000000000..ef64212b67f92 --- /dev/null +++ b/gin_delete.sql @@ -0,0 +1 @@ +delete from box where uid in (select uid from box limit 200000); diff --git a/gin_init.sql b/gin_init.sql new file mode 100644 index 0000000000000..35d785ce202a0 --- /dev/null +++ b/gin_init.sql @@ -0,0 +1,14 @@ +CREATE TABLE BOX (uid bigint, + lids integer[] NOT NULL + CHECK (array_ndims(lids) = 1)); + +CREATE OR REPLACE FUNCTION ulids( + i_uid bigint, + i_lids integer[] +) RETURNS bigint[] AS $$ + SELECT array_agg((i_uid << 32) | lid) + FROM unnest(i_lids) lid; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE INDEX i_box_uid_lids + ON box USING gin (ulids(uid, lids)) WITH (FASTUPDATE=OFF); diff --git a/gin_shot.sql b/gin_shot.sql new file mode 100644 index 0000000000000..5eb7efda250e0 --- /dev/null +++ b/gin_shot.sql @@ -0,0 +1,12 @@ +--\setrandom uid 1 1500000 +\set uid 777 +--\setrandom lid 1 1000 +--\setrandom lid2 1 1000 +--\setrandom lid3 1 1000 +\set lid 1 +\set lid2 2 +\set lid3 3 +BEGIN; +insert into box values(:uid,'{:lid,:lid2,:lid3}'); +insert into box values(:uid,'{}'); +END; diff --git a/gin_vacuum.sql b/gin_vacuum.sql new file mode 100644 index 0000000000000..adcb35bd58941 --- /dev/null +++ b/gin_vacuum.sql @@ -0,0 +1,3 @@ +select pg_size_pretty(pg_relation_size('i_box_uid_lids')); +vacuum box; +select pg_size_pretty(pg_relation_size('i_box_uid_lids')); diff --git a/prevacuum.sh b/prevacuum.sh new file mode 100644 index 0000000000000..87f722c0c7607 --- /dev/null +++ b/prevacuum.sh @@ -0,0 +1,2 @@ +#!/bin/sh +../project/bin/psql postgresindex, MAIN_FORKNUM, blkno, - RBM_NORMAL, gvs->strategy); - page = BufferGetPage(buffer); - - /* - * We should be sure that we don't concurrent with inserts, insert process - * never release root page until end (but it can unlock it and lock - * again). New scan can't start but previously started ones work - * concurrently. - */ - if (isRoot) - LockBufferForCleanup(buffer); - else - LockBuffer(buffer, GIN_EXCLUSIVE); - - Assert(GinPageIsData(page)); - if (GinPageIsLeaf(page)) - { - oldCxt = MemoryContextSwitchTo(gvs->tmpCxt); - ginVacuumPostingTreeLeaf(gvs->index, buffer, gvs); - MemoryContextSwitchTo(oldCxt); - MemoryContextReset(gvs->tmpCxt); - - /* if root is a leaf page, we don't desire further processing */ - if (!isRoot && !hasVoidPage && GinDataLeafPageIsEmpty(page)) - hasVoidPage = TRUE; - } - else - { - OffsetNumber i; - bool isChildHasVoid = FALSE; - - for (i = FirstOffsetNumber; i <= GinPageGetOpaque(page)->maxoff; i++) - { - PostingItem *pitem = GinDataPageGetPostingItem(page, i); - - if (ginVacuumPostingTreeLeaves(gvs, PostingItemGetBlockNumber(pitem), FALSE, NULL)) - isChildHasVoid = TRUE; - } - - if (isChildHasVoid) - hasVoidPage = TRUE; - } +typedef struct DataPageDeleteStack +{ + struct DataPageDeleteStack *child; + struct DataPageDeleteStack *parent; - /* - * if we have root and there are empty pages in tree, then we don't - * release lock to go further processing and guarantee that tree is unused - */ - if (!(isRoot && hasVoidPage)) - { - UnlockReleaseBuffer(buffer); - } - else - { - Assert(rootBuffer); - *rootBuffer = buffer; - } + BlockNumber blkno; /* current block number */ + BlockNumber leftBlkno; /* rightest non-deleted page on left */ + bool isRoot; +} DataPageDeleteStack; - return hasVoidPage; -} /* * Delete a posting tree page. @@ -195,6 +137,8 @@ ginDeletePage(GinVacuumState *gvs, BlockNumber deleteBlkno, BlockNumber leftBlkn /* * Lock the pages in the same order as an insertion would, to avoid * deadlocks: left, then right, then parent. + * + * AB: WE HAVE CLEANUP LOCK, NO INSERTS HERE */ lBuffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, leftBlkno, RBM_NORMAL, gvs->strategy); @@ -204,10 +148,10 @@ ginDeletePage(GinVacuumState *gvs, BlockNumber deleteBlkno, BlockNumber leftBlkn RBM_NORMAL, gvs->strategy); LockBuffer(lBuffer, GIN_EXCLUSIVE); - LockBuffer(dBuffer, GIN_EXCLUSIVE); - if (!isParentRoot) /* parent is already locked by - * LockBufferForCleanup() */ - LockBuffer(pBuffer, GIN_EXCLUSIVE); + //LockBuffer(dBuffer, GIN_EXCLUSIVE); + //if (!isParentRoot) /* parent is already locked by + // * LockBufferForCleanup() */ + // LockBuffer(pBuffer, GIN_EXCLUSIVE); START_CRIT_SECTION(); @@ -271,26 +215,17 @@ ginDeletePage(GinVacuumState *gvs, BlockNumber deleteBlkno, BlockNumber leftBlkn PageSetLSN(BufferGetPage(lBuffer), recptr); } - if (!isParentRoot) - LockBuffer(pBuffer, GIN_UNLOCK); + //if (!isParentRoot) + // LockBuffer(pBuffer, GIN_UNLOCK); ReleaseBuffer(pBuffer); UnlockReleaseBuffer(lBuffer); - UnlockReleaseBuffer(dBuffer); + ReleaseBuffer(dBuffer);//UnlockReleaseBuffer(dBuffer); END_CRIT_SECTION(); gvs->result->pages_deleted++; } -typedef struct DataPageDeleteStack -{ - struct DataPageDeleteStack *child; - struct DataPageDeleteStack *parent; - - BlockNumber blkno; /* current block number */ - BlockNumber leftBlkno; /* rightest non-deleted page on left */ - bool isRoot; -} DataPageDeleteStack; /* * scans posting tree and deletes empty pages @@ -324,6 +259,10 @@ ginScanToDelete(GinVacuumState *gvs, BlockNumber blkno, bool isRoot, buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, blkno, RBM_NORMAL, gvs->strategy); + + if(!isRoot) + LockBuffer(buffer, GIN_EXCLUSIVE); + page = BufferGetPage(buffer); Assert(GinPageIsData(page)); @@ -358,6 +297,9 @@ ginScanToDelete(GinVacuumState *gvs, BlockNumber blkno, bool isRoot, } } + if(!isRoot) + LockBuffer(buffer, GIN_UNLOCK); + ReleaseBuffer(buffer); if (!meDelete) @@ -366,37 +308,106 @@ ginScanToDelete(GinVacuumState *gvs, BlockNumber blkno, bool isRoot, return meDelete; } -static void -ginVacuumPostingTree(GinVacuumState *gvs, BlockNumber rootBlkno) + +static bool +ginVacuumPostingTreeLeaves(GinVacuumState *gvs, BlockNumber blkno, bool isRoot) { - Buffer rootBuffer = InvalidBuffer; - DataPageDeleteStack root, - *ptr, - *tmp; + Buffer buffer; + Page page; + bool hasVoidPage = FALSE; + MemoryContext oldCxt; + + buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, blkno, + RBM_NORMAL, gvs->strategy); + page = BufferGetPage(buffer); - if (ginVacuumPostingTreeLeaves(gvs, rootBlkno, TRUE, &rootBuffer) == FALSE) + ginTraverseLock(buffer,false); + + Assert(GinPageIsData(page)); + + if (GinPageIsLeaf(page)) { - Assert(rootBuffer == InvalidBuffer); - return; + oldCxt = MemoryContextSwitchTo(gvs->tmpCxt); + ginVacuumPostingTreeLeaf(gvs->index, buffer, gvs); + MemoryContextSwitchTo(oldCxt); + MemoryContextReset(gvs->tmpCxt); + + /* if root is a leaf page, we don't desire further processing */ + if (GinDataLeafPageIsEmpty(page)) + hasVoidPage = TRUE; + + UnlockReleaseBuffer(buffer); + + return hasVoidPage; } + else + { + OffsetNumber i; + bool isChildHasVoid = FALSE; + bool isanyonealife = FALSE; + OffsetNumber maxoff = GinPageGetOpaque(page)->maxoff; + BlockNumber* children = palloc(sizeof(BlockNumber) * (maxoff + 1)); - memset(&root, 0, sizeof(DataPageDeleteStack)); - root.leftBlkno = InvalidBlockNumber; - root.isRoot = TRUE; + for (i = FirstOffsetNumber; i <= maxoff; i++) + { + PostingItem *pitem = GinDataPageGetPostingItem(page, i); - vacuum_delay_point(); + children[i] = PostingItemGetBlockNumber(pitem); + } - ginScanToDelete(gvs, rootBlkno, TRUE, &root, InvalidOffsetNumber); + UnlockReleaseBuffer(buffer); - ptr = root.child; - while (ptr) - { - tmp = ptr->child; - pfree(ptr); - ptr = tmp; + for (i = FirstOffsetNumber; i <= maxoff; i++) + { + if (ginVacuumPostingTreeLeaves(gvs, children[i], FALSE)) + isChildHasVoid = TRUE; + else + isanyonealife = TRUE; + } + + pfree(children); // seems it's good to clean up here, at least root caller did it + + vacuum_delay_point(); + + if(isChildHasVoid && !isanyonealife && !isRoot) // we have to kill this page + return TRUE; // let the parent do cleanup locked deletion + + if(isChildHasVoid) + { + DataPageDeleteStack root, + *ptr, + *tmp; + + buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, blkno, + RBM_NORMAL, gvs->strategy); + LockBufferForCleanup(buffer); + + memset(&root, 0, sizeof(DataPageDeleteStack)); + root.leftBlkno = InvalidBlockNumber; + root.isRoot = TRUE; + + ginScanToDelete(gvs, blkno, TRUE, &root, InvalidOffsetNumber); + + ptr = root.child; + + while (ptr) + { + tmp = ptr->child; + pfree(ptr); + ptr = tmp; + } + + UnlockReleaseBuffer(buffer); + } + + return FALSE; } +} - UnlockReleaseBuffer(rootBuffer); +static void +ginVacuumPostingTree(GinVacuumState *gvs, BlockNumber rootBlkno) +{ + ginVacuumPostingTreeLeaves(gvs, rootBlkno, TRUE); } /* @@ -513,13 +524,10 @@ ginVacuumEntryPage(GinVacuumState *gvs, Buffer buffer, BlockNumber *roots, uint3 return (tmppage == origpage) ? NULL : tmppage; } -Datum -ginbulkdelete(PG_FUNCTION_ARGS) +IndexBulkDeleteResult * +ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, + IndexBulkDeleteCallback callback, void *callback_state) { - IndexVacuumInfo *info = (IndexVacuumInfo *) PG_GETARG_POINTER(0); - IndexBulkDeleteResult *stats = (IndexBulkDeleteResult *) PG_GETARG_POINTER(1); - IndexBulkDeleteCallback callback = (IndexBulkDeleteCallback) PG_GETARG_POINTER(2); - void *callback_state = (void *) PG_GETARG_POINTER(3); Relation index = info->index; BlockNumber blkno = GIN_ROOT_BLKNO; GinVacuumState gvs; @@ -529,9 +537,7 @@ ginbulkdelete(PG_FUNCTION_ARGS) gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext, "Gin vacuum temporary context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_SIZES); gvs.index = index; gvs.callback = callback; gvs.callback_state = callback_state; @@ -543,8 +549,12 @@ ginbulkdelete(PG_FUNCTION_ARGS) { /* Yes, so initialize stats to zeroes */ stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); - /* and cleanup any pending inserts */ - ginInsertCleanup(&gvs.ginstate, true, stats); + + /* + * and cleanup any pending inserts + */ + ginInsertCleanup(&gvs.ginstate, !IsAutoVacuumWorkerProcess(), + false, stats); } /* we'll re-count the tuples each time */ @@ -634,14 +644,12 @@ ginbulkdelete(PG_FUNCTION_ARGS) MemoryContextDelete(gvs.tmpCxt); - PG_RETURN_POINTER(gvs.result); + return gvs.result; } -Datum -ginvacuumcleanup(PG_FUNCTION_ARGS) +IndexBulkDeleteResult * +ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) { - IndexVacuumInfo *info = (IndexVacuumInfo *) PG_GETARG_POINTER(0); - IndexBulkDeleteResult *stats = (IndexBulkDeleteResult *) PG_GETARG_POINTER(1); Relation index = info->index; bool needLock; BlockNumber npages, @@ -659,9 +667,9 @@ ginvacuumcleanup(PG_FUNCTION_ARGS) if (IsAutoVacuumWorkerProcess()) { initGinState(&ginstate, index); - ginInsertCleanup(&ginstate, true, stats); + ginInsertCleanup(&ginstate, false, true, stats); } - PG_RETURN_POINTER(stats); + return stats; } /* @@ -672,7 +680,8 @@ ginvacuumcleanup(PG_FUNCTION_ARGS) { stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); initGinState(&ginstate, index); - ginInsertCleanup(&ginstate, true, stats); + ginInsertCleanup(&ginstate, !IsAutoVacuumWorkerProcess(), + false, stats); } memset(&idxStat, 0, sizeof(idxStat)); @@ -746,5 +755,5 @@ ginvacuumcleanup(PG_FUNCTION_ARGS) if (needLock) UnlockRelationForExtension(index, ExclusiveLock); - PG_RETURN_POINTER(stats); + return stats; } diff --git a/status.sh b/status.sh new file mode 100644 index 0000000000000..6db94c8854147 --- /dev/null +++ b/status.sh @@ -0,0 +1,2 @@ +#!/bin/sh +../project/bin/psql postgres Date: Mon, 28 Nov 2016 14:24:33 +0500 Subject: [PATCH 2/3] fixes --- src/backend/access/gin/ginbtree.c | 2 +- src/backend/access/gin/ginvacuum.c | 39 +++++++++++++++--------------- src/include/access/gin_private.h | 2 ++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/backend/access/gin/ginbtree.c b/src/backend/access/gin/ginbtree.c index 9b57b774e59db..0137985cb7802 100644 --- a/src/backend/access/gin/ginbtree.c +++ b/src/backend/access/gin/ginbtree.c @@ -30,7 +30,7 @@ static void ginFinishSplit(GinBtree btree, GinBtreeStack *stack, /* * Lock buffer by needed method for search. */ -static int +int ginTraverseLock(Buffer buffer, bool searchMode) { Page page; diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index 11e2c327ac0c4..23ca00d16382c 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -523,11 +523,13 @@ ginVacuumEntryPage(GinVacuumState *gvs, Buffer buffer, BlockNumber *roots, uint3 return (tmppage == origpage) ? NULL : tmppage; } - -IndexBulkDeleteResult * -ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, - IndexBulkDeleteCallback callback, void *callback_state) +Datum +ginbulkdelete(PG_FUNCTION_ARGS) { + IndexVacuumInfo *info = (IndexVacuumInfo *) PG_GETARG_POINTER(0); + IndexBulkDeleteResult *stats = (IndexBulkDeleteResult *) PG_GETARG_POINTER(1); + IndexBulkDeleteCallback callback = (IndexBulkDeleteCallback) PG_GETARG_POINTER(2); + void *callback_state = (void *) PG_GETARG_POINTER(3); Relation index = info->index; BlockNumber blkno = GIN_ROOT_BLKNO; GinVacuumState gvs; @@ -537,7 +539,9 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext, "Gin vacuum temporary context", - ALLOCSET_DEFAULT_SIZES); + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); gvs.index = index; gvs.callback = callback; gvs.callback_state = callback_state; @@ -549,12 +553,8 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, { /* Yes, so initialize stats to zeroes */ stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); - - /* - * and cleanup any pending inserts - */ - ginInsertCleanup(&gvs.ginstate, !IsAutoVacuumWorkerProcess(), - false, stats); + /* and cleanup any pending inserts */ + ginInsertCleanup(&gvs.ginstate, true, stats); } /* we'll re-count the tuples each time */ @@ -644,12 +644,14 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, MemoryContextDelete(gvs.tmpCxt); - return gvs.result; + PG_RETURN_POINTER(gvs.result); } -IndexBulkDeleteResult * -ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) +Datum +ginvacuumcleanup(PG_FUNCTION_ARGS) { + IndexVacuumInfo *info = (IndexVacuumInfo *) PG_GETARG_POINTER(0); + IndexBulkDeleteResult *stats = (IndexBulkDeleteResult *) PG_GETARG_POINTER(1); Relation index = info->index; bool needLock; BlockNumber npages, @@ -667,9 +669,9 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) if (IsAutoVacuumWorkerProcess()) { initGinState(&ginstate, index); - ginInsertCleanup(&ginstate, false, true, stats); + ginInsertCleanup(&ginstate, true, stats); } - return stats; + PG_RETURN_POINTER(stats); } /* @@ -680,8 +682,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) { stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); initGinState(&ginstate, index); - ginInsertCleanup(&ginstate, !IsAutoVacuumWorkerProcess(), - false, stats); + ginInsertCleanup(&ginstate, true, stats); } memset(&idxStat, 0, sizeof(idxStat)); @@ -755,5 +756,5 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) if (needLock) UnlockRelationForExtension(index, ExclusiveLock); - return stats; + PG_RETURN_POINTER(stats); } diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h index fce9e221607ec..ffa0222d8645e 100644 --- a/src/include/access/gin_private.h +++ b/src/include/access/gin_private.h @@ -973,4 +973,6 @@ ginCompareItemPointers(ItemPointer a, ItemPointer b) #define ginCompareItemPointers(a, b) ItemPointerCompare(a, b) #endif /* PG_USE_INLINE */ +extern int ginTraverseLock(Buffer buffer, bool searchMode); + #endif /* GIN_PRIVATE_H */ From 0832ef832362cd9ff0b235323939a3ead1ee8cdb Mon Sep 17 00:00:00 2001 From: x4m Date: Mon, 28 Nov 2016 21:10:37 +0500 Subject: [PATCH 3/3] something --- gin_shot.sql | 16 ++++++++-------- prevacuum.sh | 0 status.sh | 0 test.sh | 0 4 files changed, 8 insertions(+), 8 deletions(-) mode change 100644 => 100755 prevacuum.sh mode change 100644 => 100755 status.sh mode change 100644 => 100755 test.sh diff --git a/gin_shot.sql b/gin_shot.sql index 5eb7efda250e0..53977891734cb 100644 --- a/gin_shot.sql +++ b/gin_shot.sql @@ -1,11 +1,11 @@ ---\setrandom uid 1 1500000 -\set uid 777 ---\setrandom lid 1 1000 ---\setrandom lid2 1 1000 ---\setrandom lid3 1 1000 -\set lid 1 -\set lid2 2 -\set lid3 3 +\setrandom uid 1 1500000 +--\set uid 777 +\setrandom lid 1 1000 +\setrandom lid2 1 1000 +\setrandom lid3 1 1000 +--\set lid 1 +--\set lid2 2 +--\set lid3 3 BEGIN; insert into box values(:uid,'{:lid,:lid2,:lid3}'); insert into box values(:uid,'{}'); diff --git a/prevacuum.sh b/prevacuum.sh old mode 100644 new mode 100755 diff --git a/status.sh b/status.sh old mode 100644 new mode 100755 diff --git a/test.sh b/test.sh old mode 100644 new mode 100755