From c0e34f02fa29726287eb64dd91cb40dc6cb01b36 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Thu, 9 Nov 2017 21:03:58 +0300 Subject: [PATCH 1/4] GiST VACUUM rebase Some forgoten markers More forgoten markers more rebase some cleanup --- src/backend/access/gist/gist.c | 6 + src/backend/access/gist/gistbuild.c | 5 - src/backend/access/gist/gistutil.c | 4 +- src/backend/access/gist/gistvacuum.c | 784 ++++++++++++++++++++++++++- src/backend/access/gist/gistxlog.c | 88 ++- src/include/access/gist_private.h | 26 +- src/include/access/gistxlog.h | 24 +- 7 files changed, 918 insertions(+), 19 deletions(-) diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index aec174cd00609..5c3b00f52fc19 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -693,6 +693,12 @@ gistdoinsert(Relation r, IndexTuple itup, Size freespace, GISTSTATE *giststate) GISTInsertStack *item; OffsetNumber downlinkoffnum; + if(GistPageIsDeleted(stack->page)) { + UnlockReleaseBuffer(stack->buffer); + xlocked = false; + state.stack = stack = stack->parent; + continue; + } downlinkoffnum = gistchoose(state.r, stack->page, itup, giststate); iid = PageGetItemId(stack->page, downlinkoffnum); idxtuple = (IndexTuple) PageGetItem(stack->page, iid); diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index b4cb36486908b..39a71e0fb37b4 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -1126,11 +1126,6 @@ gistGetMaxLevel(Relation index) * but will be added there the first time we visit them. */ -typedef struct -{ - BlockNumber childblkno; /* hash key */ - BlockNumber parentblkno; -} ParentMapEntry; static void gistInitParentMap(GISTBuildState *buildstate) diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index 26d89f79ae6c5..5943180b12bdf 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -24,6 +24,7 @@ #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/syscache.h" +#include "utils/snapmgr.h" /* @@ -801,13 +802,14 @@ gistNewBuffer(Relation r) if (ConditionalLockBuffer(buffer)) { Page page = BufferGetPage(buffer); + PageHeader p = (PageHeader) page; if (PageIsNew(page)) return buffer; /* OK to use, if never initialized */ gistcheckpage(r, buffer); - if (GistPageIsDeleted(page)) + if (GistPageIsDeleted(page) && TransactionIdPrecedes(p->pd_prune_xid, RecentGlobalDataXmin)) return buffer; /* OK to use */ LockBuffer(buffer, GIST_UNLOCK); diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 77d9d12f0b7a1..67c4f2440c349 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -20,6 +20,8 @@ #include "miscadmin.h" #include "storage/indexfsm.h" #include "storage/lmgr.h" +#include "utils/snapmgr.h" +#include "access/xact.h" /* @@ -106,6 +108,31 @@ typedef struct GistBDItem struct GistBDItem *next; } GistBDItem; +typedef struct GistBDSItem +{ + BlockNumber blkno; + bool isParent; + struct GistBDSItem *next; +} GistBDSItem; + +typedef enum +{ + NOT_NEED_TO_PROCESS, /* without action */ + PROCESSED, /* action is done */ + NEED_TO_PROCESS, + NEED_TO_DELETE /* */ +} GistBlockInfoFlag; + +typedef struct GistBlockInfo { + BlockNumber blkno; + BlockNumber parent; + BlockNumber leftblock; /* block that has rightlink on blkno */ + GistBlockInfoFlag flag; + //bool toDelete; /* is need delete this block? */ + //bool isDeleted; /* this block was processed */ + bool hasRightLink; +} GistBlockInfo; + static void pushStackIfSplited(Page page, GistBDItem *stack) { @@ -125,7 +152,150 @@ pushStackIfSplited(Page page, GistBDItem *stack) stack->next = ptr; } } +static void +gistFillBlockInfo(HTAB * map, BlockNumber blkno) +{ + GistBlockInfo *entry; + bool found; + + entry = (GistBlockInfo *) hash_search(map, + (const void *) &blkno, + HASH_ENTER, + &found); + if(!found) { + entry->parent = InvalidBlockNumber; + entry->leftblock = InvalidBlockNumber; + entry->hasRightLink = false; + entry->flag = NOT_NEED_TO_PROCESS; + //entry->toDelete = false; + //entry->isDeleted = false; + } +} + +static void +gistMemorizeParentTab(HTAB * map, BlockNumber child, BlockNumber parent) +{ + GistBlockInfo *entry; + bool found; + + entry = (GistBlockInfo *) hash_search(map, + (const void *) &child, + HASH_ENTER, + &found); + if(!found) { + entry->parent = InvalidBlockNumber; + entry->leftblock = InvalidBlockNumber; + entry->hasRightLink = false; + entry->flag = NOT_NEED_TO_PROCESS; + } + entry->parent = parent; +} +static BlockNumber +gistGetParentTab(HTAB * map, BlockNumber child) +{ + GistBlockInfo *entry; + bool found; + + /* Find node buffer in hash table */ + entry = (GistBlockInfo *) hash_search(map, + (const void *) &child, + HASH_FIND, + &found); + if (!found) + elog(ERROR, "could not find parent of block %d in lookup table", child); + + return entry->parent; +} + +static BlockNumber +gistGetLeftLink(HTAB * map, BlockNumber right) +{ + GistBlockInfo *entry; + bool found; + entry = (GistBlockInfo *) hash_search(map, + (const void *) &right, + HASH_FIND, + &found); + if (!found) + return InvalidBlockNumber; + if(entry->hasRightLink == false) { + return InvalidBlockNumber; + } + return entry->leftblock; +} +static void +gistMemorizeLeftLink(HTAB * map, BlockNumber right, BlockNumber left, bool hasRightLink) +{ + GistBlockInfo *entry; + bool found; + entry = (GistBlockInfo *) hash_search(map, + (const void *) &right, + HASH_ENTER, + &found); + if (!found) { + entry->leftblock = InvalidBlockNumber; + entry->parent = InvalidBlockNumber; + entry->hasRightLink = false; + entry->flag = NOT_NEED_TO_PROCESS; + } + + if(hasRightLink) { + entry->leftblock = left; + entry->hasRightLink = hasRightLink; + } else { + if(!found) { + entry->leftblock = left; + entry->hasRightLink = hasRightLink; + } + } + +} + +static bool +gistGetDeleteLink(HTAB* map, BlockNumber blkno) { + GistBlockInfo *entry; + bool found; + + /* Find node buffer in hash table */ + entry = (GistBlockInfo *) hash_search(map, + (const void *) &blkno, + HASH_FIND, + &found); + + if (!found) + return false; + + return entry->flag == NEED_TO_DELETE; +} +static bool +gistIsProcessed(HTAB* map, BlockNumber blkno) { + GistBlockInfo *entry; + bool found; + + /* Find node buffer in hash table */ + entry = (GistBlockInfo *) hash_search(map, + (const void *) &blkno, + HASH_FIND, + &found); + return entry ? entry->flag == PROCESSED: false; +} +static void +gistMemorizeLinkToDelete(HTAB* map, BlockNumber blkno, GistBlockInfoFlag flag) { + GistBlockInfo *entry; + bool found; + entry = (GistBlockInfo *) hash_search(map, + (const void *) &blkno, + HASH_ENTER, + &found); + if (!found) { + entry->parent = InvalidBlockNumber; + entry->leftblock = InvalidBlockNumber; + entry->hasRightLink = false; + entry->flag = NOT_NEED_TO_PROCESS; + } + entry->flag = flag; +} /* * Bulk deletion of all index entries pointing to a set of heap tuples and @@ -135,9 +305,8 @@ pushStackIfSplited(Page page, GistBDItem *stack) * * Result: a palloc'd struct containing statistical info for VACUUM displays. */ -IndexBulkDeleteResult * -gistbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, - IndexBulkDeleteCallback callback, void *callback_state) +static IndexBulkDeleteResult * +gistbulkdeletelogical(IndexVacuumInfo * info, IndexBulkDeleteResult * stats, IndexBulkDeleteCallback callback, void* callback_state) { Relation rel = info->index; GistBDItem *stack, @@ -273,3 +442,612 @@ gistbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, return stats; } + +static void +gistvacuumcheckrightlink(Relation rel, IndexVacuumInfo * info, + HTAB* infomap, BlockNumber blkno, Page page) +{ + /* + * + * if there is right link on this page but not rightlink from this page. remove rightlink from left page. + * if there is right link on this page and there is a right link . right link of left page must be rightlink to rightlink of this page. + */ + + BlockNumber leftblkno; + GISTPageOpaque childopaque; + + leftblkno = gistGetLeftLink(infomap, blkno); + if (leftblkno != InvalidBlockNumber) { + /* + * there is a right link to child page + */ + BlockNumber newRight = InvalidBuffer; + GISTPageOpaque leftOpaque; + Page left; + Buffer leftbuffer; + leftbuffer = ReadBufferExtended(rel, MAIN_FORKNUM, leftblkno, + RBM_NORMAL, info->strategy); + left = (Page) BufferGetPage(leftbuffer); + + LockBuffer(leftbuffer, GIST_EXCLUSIVE); + childopaque = GistPageGetOpaque(page); + leftOpaque = GistPageGetOpaque(left); + + while (leftOpaque->rightlink != InvalidBlockNumber + && leftOpaque->rightlink != blkno) { + leftblkno = leftOpaque->rightlink; + UnlockReleaseBuffer(leftbuffer); + leftbuffer = ReadBufferExtended(rel, MAIN_FORKNUM, leftblkno, + RBM_NORMAL, info->strategy); + left = (Page) BufferGetPage(leftbuffer); + + LockBuffer(leftbuffer, GIST_EXCLUSIVE); + leftOpaque = GistPageGetOpaque(left); + + } + if (leftOpaque->rightlink == InvalidBlockNumber) { + /* + * error!! we dont find leftpage. + */ + + UnlockReleaseBuffer(leftbuffer); + return; + } + if (childopaque->rightlink != InvalidBlockNumber) { + newRight = childopaque->rightlink; + } + leftOpaque->rightlink = newRight; + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + recptr = gistXLogRightLinkChange(rel->rd_node, leftbuffer, newRight); + PageSetLSN(left, recptr); + } else + PageSetLSN(left, gistGetFakeLSN(rel)); + + UnlockReleaseBuffer(leftbuffer); + } +} +static void +gistvacuumrepairpage(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * stats, + IndexBulkDeleteCallback callback, void* callback_state, + + HTAB* infomap, BlockNumber blkno) +{ + Buffer buffer; + Page page; + PageHeader header; + OffsetNumber maxoff, i; + IndexTuple idxtuple; + ItemId iid; + OffsetNumber todelete[MaxOffsetNumber]; + int ntodelete = 0; + bool isNew; + + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, + RBM_NORMAL, info->strategy); + LockBuffer(buffer, GIST_EXCLUSIVE); + + gistcheckpage(rel, buffer); + page = (Page) BufferGetPage(buffer); + /* + * if page is inner do nothing. + */ + if(GistPageIsLeaf(page)) { + maxoff = PageGetMaxOffsetNumber(page); + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + iid = PageGetItemId(page, i); + idxtuple = (IndexTuple) PageGetItem(page, iid); + + if (callback(&(idxtuple->t_tid), callback_state)) { + todelete[ntodelete] = i - ntodelete; + ntodelete++; + } + } + isNew = PageIsNew(page) || PageIsEmpty(page); + if (ntodelete || isNew) { + START_CRIT_SECTION(); + + MarkBufferDirty(buffer); + + for (i = 0; i < ntodelete; i++) + PageIndexTupleDelete(page, todelete[i]); + GistMarkTuplesDeleted(page); + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + + recptr = gistXLogUpdate(buffer, todelete, + ntodelete, + NULL, 0, InvalidBuffer); + PageSetLSN(page, recptr); + } else + PageSetLSN(page, gistGetFakeLSN(rel)); + END_CRIT_SECTION(); + + /* + * ok. links has been deleted. and this in wal! now we can set deleted and repair rightlinks + */ + + gistvacuumcheckrightlink(rel, info, infomap, blkno, page); + + /* + * ok. rightlinks has been repaired. + */ + header = (PageHeader) page; + + header->pd_prune_xid = GetCurrentTransactionId(); + + GistPageSetDeleted(page); + stats->pages_deleted++; + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + + recptr = gistXLogSetDeleted(rel->rd_node, buffer, header->pd_prune_xid); + PageSetLSN(page, recptr); + } else + PageSetLSN(page, gistGetFakeLSN(rel)); + } + } + + UnlockReleaseBuffer(buffer); +} +static void +gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * stats, + IndexBulkDeleteCallback callback, void* callback_state, + BlockNumber npages, HTAB* infomap, + GistBDSItem* rescanstack, GistBDSItem* tail) +{ + BlockNumber blkno = GIST_ROOT_BLKNO; + for (; blkno < npages; blkno++) { + Buffer buffer; + Page page; + OffsetNumber i, maxoff; + IndexTuple idxtuple; + ItemId iid; + OffsetNumber todelete[MaxOffsetNumber]; + int ntodelete = 0; + GISTPageOpaque opaque; + BlockNumber child; + GistBDSItem *item; + bool isNew; + + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, + info->strategy); + LockBuffer(buffer, GIST_SHARE); + gistcheckpage(rel, buffer); + page = (Page) BufferGetPage(buffer); + + isNew = PageIsNew(page) || PageIsEmpty(page); + opaque = GistPageGetOpaque(page); + + gistFillBlockInfo(infomap, blkno); + + gistMemorizeLeftLink(infomap, blkno, InvalidBlockNumber, false); + + if(opaque->rightlink != InvalidBlockNumber) { + gistMemorizeLeftLink(infomap, opaque->rightlink, blkno, true); + } + if (GistPageIsLeaf(page)) { + + LockBuffer(buffer, GIST_UNLOCK); + LockBuffer(buffer, GIST_EXCLUSIVE); + + maxoff = PageGetMaxOffsetNumber(page); + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + iid = PageGetItemId(page, i); + idxtuple = (IndexTuple) PageGetItem(page, iid); + + if (callback(&(idxtuple->t_tid), callback_state)) { + todelete[ntodelete] = i - ntodelete; + ntodelete++; + stats->tuples_removed += 1; + } else + stats->num_index_tuples += 1; + } + } else { + /* + * first scan + */ + + maxoff = PageGetMaxOffsetNumber(page); + if (blkno != GIST_ROOT_BLKNO + /* && (GistFollowRight(page) || lastNSN < GistPageGetNSN(page)) */ + && opaque->rightlink != InvalidBlockNumber) { + /* + * build left link map. add to rescan later. + */ + item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); + + item->isParent = false; + item->blkno = opaque->rightlink; + item->next = NULL; + + tail->next = item; + tail = item; + + } + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + iid = PageGetItemId(page, i); + idxtuple = (IndexTuple) PageGetItem(page, iid); + child = ItemPointerGetBlockNumber(&(idxtuple->t_tid)); + + gistMemorizeParentTab(infomap, child, blkno); + + if (GistTupleIsInvalid(idxtuple)) + ereport(LOG, + (errmsg("index \"%s\" contains an inner tuple marked as invalid", RelationGetRelationName(rel)), errdetail("This is caused by an incomplete page split at crash recovery before upgrading to PostgreSQL 9.1."), errhint("Please REINDEX it."))); + } + + } + if (ntodelete || isNew) { + if ((maxoff == ntodelete) || isNew) { + + item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); + item->isParent = true; + item->blkno = blkno; + item->next = NULL; + + tail->next = item; + tail = item; + + + gistMemorizeLinkToDelete(infomap, blkno, NEED_TO_DELETE); + } else { + START_CRIT_SECTION(); + + MarkBufferDirty(buffer); + + for (i = 0; i < ntodelete; i++) + PageIndexTupleDelete(page, todelete[i]); + GistMarkTuplesDeleted(page); + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + + recptr = gistXLogUpdate(buffer, todelete, + ntodelete, + NULL, 0, InvalidBuffer); + PageSetLSN(page, recptr); + } else + PageSetLSN(page, gistGetFakeLSN(rel)); + + END_CRIT_SECTION(); + } + } + + UnlockReleaseBuffer(buffer); + vacuum_delay_point(); + } +} +static void +gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * stats, + IndexBulkDeleteCallback callback, void* callback_state, + HTAB* infomap, + GistBDSItem* rescanstack, GistBDSItem* tail) +{ + GistBDSItem * ptr; + while (rescanstack != NULL) { + Buffer buffer; + Page page; + OffsetNumber i, maxoff; + IndexTuple idxtuple; + ItemId iid; + OffsetNumber todelete[MaxOffsetNumber]; + int ntodelete = 0; + GISTPageOpaque opaque; + BlockNumber blkno, child; + Buffer childBuffer; + GistBDSItem *item; + bool isNew; + bool isProcessed; + + BlockNumber setdeletedblkno[MaxOffsetNumber]; + + blkno = rescanstack->blkno; + if (gistGetParentTab(infomap, blkno) == InvalidBlockNumber && blkno != GIST_ROOT_BLKNO) { + /* + * strange pages. it's maybe(pages without parent but is not root). + * for example when last vacuum shut down and we can delete link to this page but dont set deleted + * repair that pages. + * how repaire: remove data if exists. rightlink repair. set-deleted + */ + gistvacuumrepairpage(rel, info, stats, callback, callback_state, infomap, blkno); + + ptr = rescanstack->next; + pfree(rescanstack); + rescanstack = ptr; + + vacuum_delay_point(); + continue; + } + if (rescanstack->isParent == true) { + blkno = gistGetParentTab(infomap, blkno); + } + + isProcessed = gistIsProcessed(infomap, blkno); + + if(isProcessed == true || blkno == InvalidBlockNumber) { + + ptr = rescanstack->next; + pfree(rescanstack); + rescanstack = ptr; + + vacuum_delay_point(); + continue; + } + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, + RBM_NORMAL, info->strategy); + LockBuffer(buffer, GIST_SHARE); + + gistcheckpage(rel, buffer); + page = (Page) BufferGetPage(buffer); + + opaque = GistPageGetOpaque(page); + + if (blkno != GIST_ROOT_BLKNO + && opaque->rightlink != InvalidBlockNumber) { + item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); + + item->isParent = false; + item->blkno = opaque->rightlink; + item->next = rescanstack->next; + + rescanstack->next = item; + } + + if (GistPageIsLeaf(page)) { + /* usual procedure with leafs pages*/ + LockBuffer(buffer, GIST_UNLOCK); + LockBuffer(buffer, GIST_EXCLUSIVE); + + maxoff = PageGetMaxOffsetNumber(page); + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + iid = PageGetItemId(page, i); + idxtuple = (IndexTuple) PageGetItem(page, iid); + + if (callback(&(idxtuple->t_tid), callback_state)) { + todelete[ntodelete] = i - ntodelete; + ntodelete++; + } + } + } else { + LockBuffer(buffer, GIST_UNLOCK); + LockBuffer(buffer, GIST_EXCLUSIVE); + maxoff = PageGetMaxOffsetNumber(page); + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + bool delete; + iid = PageGetItemId(page, i); + idxtuple = (IndexTuple) PageGetItem(page, iid); + + child = ItemPointerGetBlockNumber(&(idxtuple->t_tid)); + + delete = gistGetDeleteLink(infomap, child); + /* + * leaf is needed to delete???? + */ + if (delete) { + IndexTuple idxtuplechild; + ItemId iidchild; + OffsetNumber todeletechild[MaxOffsetNumber]; + int ntodeletechild = 0; + OffsetNumber j, maxoffchild; + Page childpage; + bool childIsNew; + + childBuffer = ReadBufferExtended(rel, MAIN_FORKNUM, child, + RBM_NORMAL, info->strategy); + + LockBuffer(childBuffer, GIST_EXCLUSIVE); + + childpage = (Page) BufferGetPage(childBuffer); + childIsNew = PageIsNew(childpage) || PageIsEmpty(childpage); + + if (GistPageIsLeaf(childpage)) { + maxoffchild = PageGetMaxOffsetNumber(childpage); + for (j = FirstOffsetNumber; j <= maxoffchild; j = + OffsetNumberNext(j)) { + iidchild = PageGetItemId(childpage, j); + idxtuplechild = (IndexTuple) PageGetItem(childpage, + iidchild); + + if (callback(&(idxtuplechild->t_tid), + callback_state)) { + todeletechild[ntodeletechild] = j + - ntodeletechild; + ntodeletechild++; + } + } + if (ntodeletechild || childIsNew) { + START_CRIT_SECTION(); + + MarkBufferDirty(childBuffer); + + for (j = 0; j < ntodeletechild; j++) + PageIndexTupleDelete(childpage, + todeletechild[j]); + GistMarkTuplesDeleted(childpage); + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + + recptr = gistXLogUpdate( + childBuffer, todeletechild, + ntodeletechild, + NULL, 0, InvalidBuffer); + PageSetLSN(childpage, recptr); + } else + PageSetLSN(childpage, gistGetFakeLSN(rel)); + + END_CRIT_SECTION(); + + if ((ntodeletechild == maxoffchild) || childIsNew) { + /* + * + * if there is right link on this page but not rightlink from this page. remove rightlink from left page. + * if there is right link on this page and there is a right link . right link of left page must be rightlink to rightlink of this page. + */ + todelete[ntodelete] = i - ntodelete; + setdeletedblkno[ntodelete] = child; + ntodelete++; + } + } + } + UnlockReleaseBuffer(childBuffer); + } + } + } + isNew = PageIsNew(page) || PageIsEmpty(page); + if (ntodelete || isNew) { + if(GistPageIsLeaf(page)) { + item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); + + item->isParent = false; + item->blkno = gistGetParentTab(infomap, blkno); + item->next = rescanstack->next; + rescanstack->next = item; + } else { + /* + * delete links to pages + */ + if(ntodelete && (ntodelete == maxoff) ) { + /* save 1 link on inner page */ + ntodelete--; + } + START_CRIT_SECTION(); + + MarkBufferDirty(buffer); + + for (i = 0; i < ntodelete; i++) + PageIndexTupleDelete(page, todelete[i]); + GistMarkTuplesDeleted(page); + + if (RelationNeedsWAL(rel)) { + XLogRecPtr recptr; + + recptr = gistXLogUpdate(buffer, todelete, + ntodelete, + NULL, 0, InvalidBuffer); + PageSetLSN(page, recptr); + } else + PageSetLSN(page, gistGetFakeLSN(rel)); + END_CRIT_SECTION(); + + /* + * ok. links has been deleted. and this in wal! now we can set deleted and repair rightlinks + */ + for (i = 0; i < ntodelete; i++) { + Buffer childBuffertodelete; + Page childpagetodelete; + PageHeader p; + gistMemorizeLinkToDelete(infomap, setdeletedblkno[i], PROCESSED); + + childBuffertodelete = ReadBufferExtended(rel, MAIN_FORKNUM, setdeletedblkno[i], + RBM_NORMAL, info->strategy); + + LockBuffer(childBuffertodelete, GIST_EXCLUSIVE); + + childpagetodelete = (Page) BufferGetPage(childBuffertodelete); + + p = (PageHeader) childpagetodelete; + + p->pd_prune_xid = GetCurrentTransactionId(); + + gistvacuumcheckrightlink(rel, info, infomap, + setdeletedblkno[i], childpagetodelete); + GistPageSetDeleted(childpagetodelete); + MarkBufferDirty(childBuffertodelete); + UnlockReleaseBuffer(childBuffertodelete); + stats->pages_deleted++; + } + } + } + gistMemorizeLinkToDelete(infomap, blkno, PROCESSED); + UnlockReleaseBuffer(buffer); + + ptr = rescanstack->next; + pfree(rescanstack); + rescanstack = ptr; + + vacuum_delay_point(); + } +} + +IndexBulkDeleteResult *gistbulkdelete(IndexVacuumInfo *info, + IndexBulkDeleteResult *stats, + IndexBulkDeleteCallback callback, + void *callback_state) +{ + Relation rel = info->index; + GistBDSItem *rescanstack = NULL, + *tail = NULL; + + int memoryneeded = 0; + + BlockNumber npages; + + bool needLock; + HTAB *infomap; + HASHCTL hashCtl; + + + if (stats == NULL) + stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); + stats->estimated_count = false; + stats->num_index_tuples = 0; + + hashCtl.keysize = sizeof(BlockNumber); + hashCtl.entrysize = sizeof(GistBlockInfo); + hashCtl.hcxt = CurrentMemoryContext; + + needLock = !RELATION_IS_LOCAL(rel); + + if (needLock) + LockRelationForExtension(rel, ExclusiveLock); + npages = RelationGetNumberOfBlocks(rel); + if (needLock) + UnlockRelationForExtension(rel, ExclusiveLock); + + /* + * estimate memory limit + * if map more than maintance_mem_work use old version of vacuum + */ + + memoryneeded = npages * (sizeof(GistBlockInfo)); + if(memoryneeded > maintenance_work_mem * 1024) { + return gistbulkdeletelogical(info, stats, callback, callback_state); + } + + + infomap = hash_create("gistvacuum info map", + npages, + &hashCtl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT ); + + rescanstack = (GistBDSItem *) palloc(sizeof(GistBDSItem)); + + rescanstack->isParent = false; + rescanstack->blkno = GIST_ROOT_BLKNO; + rescanstack->next = NULL; + tail = rescanstack; + + /* + * this part of the vacuum use scan in physical order. Also this function fill hashmap `infomap` + * that stores information about parent, rightlinks and etc. Pages is needed to rescan will be pushed to tail of rescanstack. + * this function don't set flag gist_deleted. + */ + gistphysicalvacuum(rel, info, stats, callback, callback_state, npages, infomap, rescanstack, tail); + /* + * this part of the vacuum is not in physical order. It scans only pages from rescanstack. + * we get page if this page is leaf we use usual procedure, but if pages is inner that we scan + * it and delete links to childrens(but firstly recheck children and if all is ok). + * if any pages is empty or new after processing set flag gist_delete , store prune_xid number + * and etc. if all links from pages are deleted push parent of page to rescan stack to processing. + * special case is when all tuples are deleted from index. in this case root block will be setted in leaf. + */ + gistrescanvacuum(rel, info, stats, callback, callback_state, infomap, rescanstack, tail); + + hash_destroy(infomap); + return stats; +} diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index 7fd91ce640a29..7b82c02e1508b 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -60,6 +60,50 @@ gistRedoClearFollowRight(XLogReaderState *record, uint8 block_id) UnlockReleaseBuffer(buffer); } +static void +gistRedoRightlinkChange(XLogReaderState *record) { + XLogRecPtr lsn = record->EndRecPtr; + gistxlogPageRightlinkChange *xldata = (gistxlogPageRightlinkChange *) XLogRecGetData(record); + Buffer buffer; + Page page; + BlockNumber newright; + GISTPageOpaque opaque; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + newright = xldata->newRightLink; + page = BufferGetPage(buffer); + opaque = GistPageGetOpaque(page); + opaque->rightlink = newright; + /*if(newright == InvalidBlockNumber) { + GistClearFollowRight(page); + }*/ + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } +} + +static void +gistRedoPageSetDeleted(XLogReaderState *record) { + XLogRecPtr lsn = record->EndRecPtr; + gistxlogPageDelete *xldata = (gistxlogPageDelete *) XLogRecGetData(record); + Buffer buffer; + Page page; + PageHeader header; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + page = (Page) BufferGetPage(buffer); + header = (PageHeader) page; + + header->pd_prune_xid = xldata->id; + GistPageSetDeleted(page); + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + + } +} /* * redo any page update (except page split) */ @@ -112,8 +156,8 @@ gistRedoPageUpdateRecord(XLogReaderState *record) data += sizeof(OffsetNumber) * xldata->ntodelete; PageIndexMultiDelete(page, todelete, xldata->ntodelete); - if (GistPageIsLeaf(page)) - GistMarkTuplesDeleted(page); + + GistMarkTuplesDeleted(page); } /* Add new tuples if any */ @@ -324,6 +368,12 @@ gist_redo(XLogReaderState *record) case XLOG_GIST_CREATE_INDEX: gistRedoCreateIndex(record); break; + case XLOG_GIST_PAGE_DELETE: + gistRedoPageSetDeleted(record); + break; + case XLOG_GIST_RIGHTLINK_CHANGE: + gistRedoRightlinkChange(record); + break; default: elog(PANIC, "gist_redo: unknown op code %u", info); } @@ -442,6 +492,40 @@ gistXLogSplit(bool page_is_leaf, return recptr; } + +XLogRecPtr +gistXLogSetDeleted(RelFileNode node, Buffer buffer, TransactionId xid) { + gistxlogPageDelete xlrec; + XLogRecPtr recptr; + + xlrec.id = xid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(gistxlogPageDelete)); + + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + /* new tuples */ + recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_PAGE_DELETE); + return recptr; +} + +XLogRecPtr +gistXLogRightLinkChange(RelFileNode node, Buffer buffer, + BlockNumber newRightLink) { + gistxlogPageRightlinkChange xlrec; + XLogRecPtr recptr; + + xlrec.newRightLink = newRightLink; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(gistxlogPageRightlinkChange)); + + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + /* new tuples */ + recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_RIGHTLINK_CHANGE); + return recptr; +} + /* * Write XLOG record describing a page update. The update can include any * number of deletions and/or insertions of tuples on a single index page. diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index bfef2df420601..8cc5909dc1107 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -16,6 +16,7 @@ #include "access/amapi.h" #include "access/gist.h" +#include "access/gistxlog.h" #include "access/itup.h" #include "fmgr.h" #include "lib/pairingheap.h" @@ -51,6 +52,11 @@ typedef struct char tupledata[FLEXIBLE_ARRAY_MEMBER]; } GISTNodeBufferPage; +typedef struct +{ + BlockNumber childblkno; /* hash key */ + BlockNumber parentblkno; +} ParentMapEntry; #define BUFFER_PAGE_DATA_OFFSET MAXALIGN(offsetof(GISTNodeBufferPage, tupledata)) /* Returns free space in node buffer page */ #define PAGE_FREE_SPACE(nbp) (nbp->freespace) @@ -176,13 +182,6 @@ typedef struct GISTScanOpaqueData typedef GISTScanOpaqueData *GISTScanOpaque; -/* despite the name, gistxlogPage is not part of any xlog record */ -typedef struct gistxlogPage -{ - BlockNumber blkno; - int num; /* number of index tuples following */ -} gistxlogPage; - /* SplitedPageLayout - gistSplit function result */ typedef struct SplitedPageLayout { @@ -409,6 +408,19 @@ extern bool gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, extern SplitedPageLayout *gistSplit(Relation r, Page page, IndexTuple *itup, int len, GISTSTATE *giststate); +/* gistxlog.c */ +extern void gist_redo(XLogReaderState *record); +extern void gist_desc(StringInfo buf, XLogReaderState *record); +extern const char *gist_identify(uint8 info); +extern void gist_xlog_startup(void); +extern void gist_xlog_cleanup(void); + +extern XLogRecPtr gistXLogSetDeleted(RelFileNode node, Buffer buffer, + TransactionId xid); + +extern XLogRecPtr gistXLogRightLinkChange(RelFileNode node, Buffer buffer, + BlockNumber newRightLink) ; + extern XLogRecPtr gistXLogUpdate(Buffer buffer, OffsetNumber *todelete, int ntodelete, IndexTuple *itup, int ntup, diff --git a/src/include/access/gistxlog.h b/src/include/access/gistxlog.h index 3b126eca2ac30..d91c8b7e5443e 100644 --- a/src/include/access/gistxlog.h +++ b/src/include/access/gistxlog.h @@ -17,12 +17,15 @@ #include "access/xlogreader.h" #include "lib/stringinfo.h" +/* XLog stuff */ + #define XLOG_GIST_PAGE_UPDATE 0x00 /* #define XLOG_GIST_NEW_ROOT 0x20 */ /* not used anymore */ #define XLOG_GIST_PAGE_SPLIT 0x30 /* #define XLOG_GIST_INSERT_COMPLETE 0x40 */ /* not used anymore */ #define XLOG_GIST_CREATE_INDEX 0x50 - /* #define XLOG_GIST_PAGE_DELETE 0x60 */ /* not used anymore */ +#define XLOG_GIST_PAGE_DELETE 0x60 +#define XLOG_GIST_RIGHTLINK_CHANGE 0x70 /* * Backup Blk 0: updated page. @@ -59,6 +62,25 @@ typedef struct gistxlogPageSplit */ } gistxlogPageSplit; +typedef struct gistxlogPageDelete +{ + TransactionId id; +} gistxlogPageDelete; + +typedef struct gistxlogPageRightlinkChange +{ + BlockNumber newRightLink; + +} gistxlogPageRightlinkChange; + + +/* despite the name, gistxlogPage is not part of any xlog record */ +typedef struct gistxlogPage +{ + BlockNumber blkno; + int num; /* number of index tuples following */ +} gistxlogPage; + extern void gist_redo(XLogReaderState *record); extern void gist_desc(StringInfo buf, XLogReaderState *record); extern const char *gist_identify(uint8 info); From 75063c71648c65e347046869891eb3a4051a1669 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 18 Dec 2017 12:46:46 +0500 Subject: [PATCH 2/4] Fix for small tables and tests --- src/backend/access/gist/gistvacuum.c | 8 +++++--- src/test/regress/expected/gist.out | 4 +--- src/test/regress/sql/gist.sql | 4 +--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 67c4f2440c349..fccb2b9a91588 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -746,7 +746,7 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s BlockNumber setdeletedblkno[MaxOffsetNumber]; blkno = rescanstack->blkno; - if (gistGetParentTab(infomap, blkno) == InvalidBlockNumber && blkno != GIST_ROOT_BLKNO) { + if (blkno != GIST_ROOT_BLKNO && gistGetParentTab(infomap, blkno) == InvalidBlockNumber) { /* * strange pages. it's maybe(pages without parent but is not root). * for example when last vacuum shut down and we can delete link to this page but dont set deleted @@ -900,11 +900,13 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s } isNew = PageIsNew(page) || PageIsEmpty(page); if (ntodelete || isNew) { - if(GistPageIsLeaf(page)) { + if (GistPageIsLeaf(page)) { item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); item->isParent = false; - item->blkno = gistGetParentTab(infomap, blkno); + if (blkno != GIST_ROOT_BLKNO) { + item->blkno = gistGetParentTab(infomap, blkno); + } item->next = rescanstack->next; rescanstack->next = item; } else { diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out index f5a2993aaf22a..5b92f08c74717 100644 --- a/src/test/regress/expected/gist.out +++ b/src/test/regress/expected/gist.out @@ -27,9 +27,7 @@ insert into gist_point_tbl (id, p) select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g; -- To test vacuum, delete some entries from all over the index. delete from gist_point_tbl where id % 2 = 1; --- And also delete some concentration of values. (GiST doesn't currently --- attempt to delete pages even when they become empty, but if it did, this --- would exercise it) +-- And also delete some concentration of values. delete from gist_point_tbl where id < 10000; vacuum analyze gist_point_tbl; -- rebuild the index with a different fillfactor diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql index bae722fe13ced..e66396e851bbd 100644 --- a/src/test/regress/sql/gist.sql +++ b/src/test/regress/sql/gist.sql @@ -28,9 +28,7 @@ select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g; -- To test vacuum, delete some entries from all over the index. delete from gist_point_tbl where id % 2 = 1; --- And also delete some concentration of values. (GiST doesn't currently --- attempt to delete pages even when they become empty, but if it did, this --- would exercise it) +-- And also delete some concentration of values. delete from gist_point_tbl where id < 10000; vacuum analyze gist_point_tbl; From 05193868d97f189de0cc2c994cf224bd6eef8deb Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 18 Dec 2017 12:47:02 +0500 Subject: [PATCH 3/4] Local test --- test.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 test.sh diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000000..e6d89478d8d6e --- /dev/null +++ b/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e +pkill -9 postgres || true +#make -j 16 && make install + +rm -rf ~/DemoDb +cd ~/project/bin/ +./initdb ~/DemoDb +./pg_ctl -D ~/DemoDb start +./psql postgres -c "create extension cube;" + + +for i in $(seq 1 12); do +size=$((100 * 2**$i)) +./psql postgres -c "create table x as select cube(random()) c from generate_series(1,$size) y; create index on x using gist(c);" +./psql postgres -c "delete from x;" +./psql postgres -c "VACUUM x;" +./psql postgres -c "VACUUM x;" +./psql postgres -c "drop table x;" +./psql postgres -c "create table x as select cube(random()) c from generate_series(1,$size) y; create index on x using gist(c);" +./psql postgres -c "delete from x where (c~>1)>0.1;" +./psql postgres -c "VACUUM x;" +./psql postgres -c "insert into x select cube(random()) c from generate_series(1,$size) y;" +./psql postgres -c "VACUUM x;" +./psql postgres -c "delete from x where (c~>1)>0.1;" +./psql postgres -c "select pg_size_pretty(pg_relation_size('x_c_idx'));" +./psql postgres -c "VACUUM FULL x;" +./psql postgres -c "select pg_size_pretty(pg_relation_size('x_c_idx'));" +./psql postgres -c "drop table x;" +done + +./pg_ctl -D ~/DemoDb stop \ No newline at end of file From 47e0335833f9e91948e9cda7e66bbcb907d93d4b Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 18 Dec 2017 13:08:19 +0500 Subject: [PATCH 4/4] fix block endings --- src/backend/access/gist/gist.c | 3 +- src/backend/access/gist/gistvacuum.c | 218 +++++++++++++++++---------- src/backend/access/gist/gistxlog.c | 12 +- 3 files changed, 150 insertions(+), 83 deletions(-) diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 5c3b00f52fc19..08f88a2d90d92 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -693,7 +693,8 @@ gistdoinsert(Relation r, IndexTuple itup, Size freespace, GISTSTATE *giststate) GISTInsertStack *item; OffsetNumber downlinkoffnum; - if(GistPageIsDeleted(stack->page)) { + if(GistPageIsDeleted(stack->page)) + { UnlockReleaseBuffer(stack->buffer); xlocked = false; state.stack = stack = stack->parent; diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index fccb2b9a91588..67a020970e7d8 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -120,16 +120,13 @@ typedef enum NOT_NEED_TO_PROCESS, /* without action */ PROCESSED, /* action is done */ NEED_TO_PROCESS, - NEED_TO_DELETE /* */ + NEED_TO_DELETE } GistBlockInfoFlag; typedef struct GistBlockInfo { - BlockNumber blkno; BlockNumber parent; BlockNumber leftblock; /* block that has rightlink on blkno */ GistBlockInfoFlag flag; - //bool toDelete; /* is need delete this block? */ - //bool isDeleted; /* this block was processed */ bool hasRightLink; } GistBlockInfo; @@ -167,8 +164,6 @@ gistFillBlockInfo(HTAB * map, BlockNumber blkno) entry->leftblock = InvalidBlockNumber; entry->hasRightLink = false; entry->flag = NOT_NEED_TO_PROCESS; - //entry->toDelete = false; - //entry->isDeleted = false; } } @@ -218,7 +213,8 @@ gistGetLeftLink(HTAB * map, BlockNumber right) &found); if (!found) return InvalidBlockNumber; - if(entry->hasRightLink == false) { + if(entry->hasRightLink == false) + { return InvalidBlockNumber; } return entry->leftblock; @@ -232,18 +228,23 @@ gistMemorizeLeftLink(HTAB * map, BlockNumber right, BlockNumber left, bool hasRi (const void *) &right, HASH_ENTER, &found); - if (!found) { + if (!found) + { entry->leftblock = InvalidBlockNumber; entry->parent = InvalidBlockNumber; entry->hasRightLink = false; entry->flag = NOT_NEED_TO_PROCESS; } - if(hasRightLink) { + if(hasRightLink) + { entry->leftblock = left; entry->hasRightLink = hasRightLink; - } else { - if(!found) { + } + else + { + if(!found) + { entry->leftblock = left; entry->hasRightLink = hasRightLink; } @@ -252,7 +253,8 @@ gistMemorizeLeftLink(HTAB * map, BlockNumber right, BlockNumber left, bool hasRi } static bool -gistGetDeleteLink(HTAB* map, BlockNumber blkno) { +gistGetDeleteLink(HTAB* map, BlockNumber blkno) +{ GistBlockInfo *entry; bool found; @@ -268,7 +270,8 @@ gistGetDeleteLink(HTAB* map, BlockNumber blkno) { return entry->flag == NEED_TO_DELETE; } static bool -gistIsProcessed(HTAB* map, BlockNumber blkno) { +gistIsProcessed(HTAB* map, BlockNumber blkno) +{ GistBlockInfo *entry; bool found; @@ -281,14 +284,16 @@ gistIsProcessed(HTAB* map, BlockNumber blkno) { return entry ? entry->flag == PROCESSED: false; } static void -gistMemorizeLinkToDelete(HTAB* map, BlockNumber blkno, GistBlockInfoFlag flag) { +gistMemorizeLinkToDelete(HTAB* map, BlockNumber blkno, GistBlockInfoFlag flag) +{ GistBlockInfo *entry; bool found; entry = (GistBlockInfo *) hash_search(map, (const void *) &blkno, HASH_ENTER, &found); - if (!found) { + if (!found) + { entry->parent = InvalidBlockNumber; entry->leftblock = InvalidBlockNumber; entry->hasRightLink = false; @@ -457,7 +462,8 @@ gistvacuumcheckrightlink(Relation rel, IndexVacuumInfo * info, GISTPageOpaque childopaque; leftblkno = gistGetLeftLink(infomap, blkno); - if (leftblkno != InvalidBlockNumber) { + if (leftblkno != InvalidBlockNumber) + { /* * there is a right link to child page */ @@ -474,7 +480,8 @@ gistvacuumcheckrightlink(Relation rel, IndexVacuumInfo * info, leftOpaque = GistPageGetOpaque(left); while (leftOpaque->rightlink != InvalidBlockNumber - && leftOpaque->rightlink != blkno) { + && leftOpaque->rightlink != blkno) + { leftblkno = leftOpaque->rightlink; UnlockReleaseBuffer(leftbuffer); leftbuffer = ReadBufferExtended(rel, MAIN_FORKNUM, leftblkno, @@ -485,7 +492,8 @@ gistvacuumcheckrightlink(Relation rel, IndexVacuumInfo * info, leftOpaque = GistPageGetOpaque(left); } - if (leftOpaque->rightlink == InvalidBlockNumber) { + if (leftOpaque->rightlink == InvalidBlockNumber) + { /* * error!! we dont find leftpage. */ @@ -493,26 +501,29 @@ gistvacuumcheckrightlink(Relation rel, IndexVacuumInfo * info, UnlockReleaseBuffer(leftbuffer); return; } - if (childopaque->rightlink != InvalidBlockNumber) { + if (childopaque->rightlink != InvalidBlockNumber) + { newRight = childopaque->rightlink; } leftOpaque->rightlink = newRight; - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogRightLinkChange(rel->rd_node, leftbuffer, newRight); PageSetLSN(left, recptr); - } else + } + else + { PageSetLSN(left, gistGetFakeLSN(rel)); + } UnlockReleaseBuffer(leftbuffer); } } static void gistvacuumrepairpage(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * stats, - IndexBulkDeleteCallback callback, void* callback_state, - - HTAB* infomap, BlockNumber blkno) + IndexBulkDeleteCallback callback, void* callback_state, HTAB* infomap, BlockNumber blkno) { Buffer buffer; Page page; @@ -533,19 +544,23 @@ gistvacuumrepairpage(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult /* * if page is inner do nothing. */ - if(GistPageIsLeaf(page)) { + if(GistPageIsLeaf(page)) + { maxoff = PageGetMaxOffsetNumber(page); - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { iid = PageGetItemId(page, i); idxtuple = (IndexTuple) PageGetItem(page, iid); - if (callback(&(idxtuple->t_tid), callback_state)) { + if (callback(&(idxtuple->t_tid), callback_state)) + { todelete[ntodelete] = i - ntodelete; ntodelete++; } } isNew = PageIsNew(page) || PageIsEmpty(page); - if (ntodelete || isNew) { + if (ntodelete || isNew) + { START_CRIT_SECTION(); MarkBufferDirty(buffer); @@ -554,15 +569,19 @@ gistvacuumrepairpage(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult PageIndexTupleDelete(page, todelete[i]); GistMarkTuplesDeleted(page); - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogUpdate(buffer, todelete, ntodelete, NULL, 0, InvalidBuffer); PageSetLSN(page, recptr); - } else + } + else + { PageSetLSN(page, gistGetFakeLSN(rel)); + } END_CRIT_SECTION(); /* @@ -581,7 +600,8 @@ gistvacuumrepairpage(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult GistPageSetDeleted(page); stats->pages_deleted++; - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogSetDeleted(rel->rd_node, buffer, header->pd_prune_xid); @@ -600,7 +620,8 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * GistBDSItem* rescanstack, GistBDSItem* tail) { BlockNumber blkno = GIST_ROOT_BLKNO; - for (; blkno < npages; blkno++) { + for (; blkno < npages; blkno++) + { Buffer buffer; Page page; OffsetNumber i, maxoff; @@ -626,27 +647,36 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * gistMemorizeLeftLink(infomap, blkno, InvalidBlockNumber, false); - if(opaque->rightlink != InvalidBlockNumber) { + if(opaque->rightlink != InvalidBlockNumber) + { gistMemorizeLeftLink(infomap, opaque->rightlink, blkno, true); } - if (GistPageIsLeaf(page)) { + if (GistPageIsLeaf(page)) + { LockBuffer(buffer, GIST_UNLOCK); LockBuffer(buffer, GIST_EXCLUSIVE); maxoff = PageGetMaxOffsetNumber(page); - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { iid = PageGetItemId(page, i); idxtuple = (IndexTuple) PageGetItem(page, iid); - if (callback(&(idxtuple->t_tid), callback_state)) { + if (callback(&(idxtuple->t_tid), callback_state)) + { todelete[ntodelete] = i - ntodelete; ntodelete++; stats->tuples_removed += 1; - } else + } + else + { stats->num_index_tuples += 1; + } } - } else { + } + else + { /* * first scan */ @@ -654,7 +684,8 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * maxoff = PageGetMaxOffsetNumber(page); if (blkno != GIST_ROOT_BLKNO /* && (GistFollowRight(page) || lastNSN < GistPageGetNSN(page)) */ - && opaque->rightlink != InvalidBlockNumber) { + && opaque->rightlink != InvalidBlockNumber) + { /* * build left link map. add to rescan later. */ @@ -668,7 +699,8 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * tail = item; } - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { iid = PageGetItemId(page, i); idxtuple = (IndexTuple) PageGetItem(page, iid); child = ItemPointerGetBlockNumber(&(idxtuple->t_tid)); @@ -681,8 +713,10 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * } } - if (ntodelete || isNew) { - if ((maxoff == ntodelete) || isNew) { + if (ntodelete || isNew) + { + if ((maxoff == ntodelete) || isNew) + { item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); item->isParent = true; @@ -694,7 +728,9 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * gistMemorizeLinkToDelete(infomap, blkno, NEED_TO_DELETE); - } else { + } + else + { START_CRIT_SECTION(); MarkBufferDirty(buffer); @@ -703,15 +739,19 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * PageIndexTupleDelete(page, todelete[i]); GistMarkTuplesDeleted(page); - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogUpdate(buffer, todelete, ntodelete, NULL, 0, InvalidBuffer); PageSetLSN(page, recptr); - } else + } + else + { PageSetLSN(page, gistGetFakeLSN(rel)); + } END_CRIT_SECTION(); } @@ -723,12 +763,12 @@ gistphysicalvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * } static void gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * stats, - IndexBulkDeleteCallback callback, void* callback_state, - HTAB* infomap, + IndexBulkDeleteCallback callback, void* callback_state, HTAB* infomap, GistBDSItem* rescanstack, GistBDSItem* tail) { GistBDSItem * ptr; - while (rescanstack != NULL) { + while (rescanstack != NULL) + { Buffer buffer; Page page; OffsetNumber i, maxoff; @@ -746,7 +786,8 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s BlockNumber setdeletedblkno[MaxOffsetNumber]; blkno = rescanstack->blkno; - if (blkno != GIST_ROOT_BLKNO && gistGetParentTab(infomap, blkno) == InvalidBlockNumber) { + if (blkno != GIST_ROOT_BLKNO && gistGetParentTab(infomap, blkno) == InvalidBlockNumber) + { /* * strange pages. it's maybe(pages without parent but is not root). * for example when last vacuum shut down and we can delete link to this page but dont set deleted @@ -762,14 +803,15 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s vacuum_delay_point(); continue; } - if (rescanstack->isParent == true) { + if (rescanstack->isParent == true) + { blkno = gistGetParentTab(infomap, blkno); } isProcessed = gistIsProcessed(infomap, blkno); - if(isProcessed == true || blkno == InvalidBlockNumber) { - + if(isProcessed == true || blkno == InvalidBlockNumber) + { ptr = rescanstack->next; pfree(rescanstack); rescanstack = ptr; @@ -787,7 +829,8 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s opaque = GistPageGetOpaque(page); if (blkno != GIST_ROOT_BLKNO - && opaque->rightlink != InvalidBlockNumber) { + && opaque->rightlink != InvalidBlockNumber) + { item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); item->isParent = false; @@ -797,26 +840,32 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s rescanstack->next = item; } - if (GistPageIsLeaf(page)) { + if (GistPageIsLeaf(page)) + { /* usual procedure with leafs pages*/ LockBuffer(buffer, GIST_UNLOCK); LockBuffer(buffer, GIST_EXCLUSIVE); maxoff = PageGetMaxOffsetNumber(page); - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { iid = PageGetItemId(page, i); idxtuple = (IndexTuple) PageGetItem(page, iid); - if (callback(&(idxtuple->t_tid), callback_state)) { + if (callback(&(idxtuple->t_tid), callback_state)) + { todelete[ntodelete] = i - ntodelete; ntodelete++; } } - } else { + } + else + { LockBuffer(buffer, GIST_UNLOCK); LockBuffer(buffer, GIST_EXCLUSIVE); maxoff = PageGetMaxOffsetNumber(page); - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) { + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { bool delete; iid = PageGetItemId(page, i); idxtuple = (IndexTuple) PageGetItem(page, iid); @@ -827,7 +876,8 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s /* * leaf is needed to delete???? */ - if (delete) { + if (delete) + { IndexTuple idxtuplechild; ItemId iidchild; OffsetNumber todeletechild[MaxOffsetNumber]; @@ -844,22 +894,24 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s childpage = (Page) BufferGetPage(childBuffer); childIsNew = PageIsNew(childpage) || PageIsEmpty(childpage); - if (GistPageIsLeaf(childpage)) { + if (GistPageIsLeaf(childpage)) + { maxoffchild = PageGetMaxOffsetNumber(childpage); - for (j = FirstOffsetNumber; j <= maxoffchild; j = - OffsetNumberNext(j)) { + for (j = FirstOffsetNumber; j <= maxoffchild; j = OffsetNumberNext(j)) + { iidchild = PageGetItemId(childpage, j); idxtuplechild = (IndexTuple) PageGetItem(childpage, iidchild); - if (callback(&(idxtuplechild->t_tid), - callback_state)) { + if (callback(&(idxtuplechild->t_tid), callback_state)) + { todeletechild[ntodeletechild] = j - ntodeletechild; ntodeletechild++; } } - if (ntodeletechild || childIsNew) { + if (ntodeletechild || childIsNew) + { START_CRIT_SECTION(); MarkBufferDirty(childBuffer); @@ -869,7 +921,8 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s todeletechild[j]); GistMarkTuplesDeleted(childpage); - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogUpdate( @@ -882,7 +935,8 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s END_CRIT_SECTION(); - if ((ntodeletechild == maxoffchild) || childIsNew) { + if ((ntodeletechild == maxoffchild) || childIsNew) + { /* * * if there is right link on this page but not rightlink from this page. remove rightlink from left page. @@ -899,21 +953,27 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s } } isNew = PageIsNew(page) || PageIsEmpty(page); - if (ntodelete || isNew) { - if (GistPageIsLeaf(page)) { + if (ntodelete || isNew) + { + if (GistPageIsLeaf(page)) + { item = (GistBDSItem *) palloc(sizeof(GistBDSItem)); item->isParent = false; - if (blkno != GIST_ROOT_BLKNO) { + if (blkno != GIST_ROOT_BLKNO) + { item->blkno = gistGetParentTab(infomap, blkno); } item->next = rescanstack->next; rescanstack->next = item; - } else { + } + else + { /* * delete links to pages */ - if(ntodelete && (ntodelete == maxoff) ) { + if(ntodelete && (ntodelete == maxoff)) + { /* save 1 link on inner page */ ntodelete--; } @@ -925,21 +985,26 @@ gistrescanvacuum(Relation rel, IndexVacuumInfo * info, IndexBulkDeleteResult * s PageIndexTupleDelete(page, todelete[i]); GistMarkTuplesDeleted(page); - if (RelationNeedsWAL(rel)) { + if (RelationNeedsWAL(rel)) + { XLogRecPtr recptr; recptr = gistXLogUpdate(buffer, todelete, ntodelete, NULL, 0, InvalidBuffer); PageSetLSN(page, recptr); - } else + } + else + { PageSetLSN(page, gistGetFakeLSN(rel)); + } END_CRIT_SECTION(); /* * ok. links has been deleted. and this in wal! now we can set deleted and repair rightlinks */ - for (i = 0; i < ntodelete; i++) { + for (i = 0; i < ntodelete; i++) + { Buffer childBuffertodelete; Page childpagetodelete; PageHeader p; @@ -1017,7 +1082,8 @@ IndexBulkDeleteResult *gistbulkdelete(IndexVacuumInfo *info, */ memoryneeded = npages * (sizeof(GistBlockInfo)); - if(memoryneeded > maintenance_work_mem * 1024) { + if(memoryneeded > maintenance_work_mem * 1024) + { return gistbulkdeletelogical(info, stats, callback, callback_state); } diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index 7b82c02e1508b..624a694579778 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -61,7 +61,8 @@ gistRedoClearFollowRight(XLogReaderState *record, uint8 block_id) } static void -gistRedoRightlinkChange(XLogReaderState *record) { +gistRedoRightlinkChange(XLogReaderState *record) +{ XLogRecPtr lsn = record->EndRecPtr; gistxlogPageRightlinkChange *xldata = (gistxlogPageRightlinkChange *) XLogRecGetData(record); Buffer buffer; @@ -75,16 +76,14 @@ gistRedoRightlinkChange(XLogReaderState *record) { page = BufferGetPage(buffer); opaque = GistPageGetOpaque(page); opaque->rightlink = newright; - /*if(newright == InvalidBlockNumber) { - GistClearFollowRight(page); - }*/ PageSetLSN(page, lsn); MarkBufferDirty(buffer); } } static void -gistRedoPageSetDeleted(XLogReaderState *record) { +gistRedoPageSetDeleted(XLogReaderState *record) +{ XLogRecPtr lsn = record->EndRecPtr; gistxlogPageDelete *xldata = (gistxlogPageDelete *) XLogRecGetData(record); Buffer buffer; @@ -511,7 +510,8 @@ gistXLogSetDeleted(RelFileNode node, Buffer buffer, TransactionId xid) { XLogRecPtr gistXLogRightLinkChange(RelFileNode node, Buffer buffer, - BlockNumber newRightLink) { + BlockNumber newRightLink) +{ gistxlogPageRightlinkChange xlrec; XLogRecPtr recptr;