From 13e42280e1a00715fb6c97f38aed4d111a8e35e6 Mon Sep 17 00:00:00 2001 From: Anton Chikunov Date: Sun, 21 Apr 2019 21:25:48 +0500 Subject: [PATCH] Generic WAL support. Script for replicas --- Dockerfile | 14 +++------ create_replica.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++ gist.c | 75 ++++++++++++++++++++++++----------------------- gistvacuum.c | 17 ++++++++--- 4 files changed, 128 insertions(+), 51 deletions(-) create mode 100755 create_replica.sh diff --git a/Dockerfile b/Dockerfile index 0e1db2b..e1035df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN localedef -i en_US -c -f UTF-8 en_US.UTF-8 RUN groupadd -r postgres --gid=999 \ && useradd -m -r -g postgres --uid=999 postgres WORKDIR /home/postgres -RUN su postgres -c "git clone https://github.com/postgres/postgres.git --depth=1 --branch=REL_11_STABLE" +RUN su postgres -c "git clone https://github.com/postgres/postgres.git --depth=1 --branch=master" WORKDIR /home/postgres/postgres COPY . ./contrib/ags RUN su postgres -c "./configure \ @@ -45,12 +45,6 @@ RUN make install WORKDIR /home/postgres/postgres/contrib/ags RUN make install RUN chown -R postgres:postgres /usr/local/pgsql -RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./test.sh" -# ENTRYPOINT [ "./test.sh" ] - -# RUN make install -# RUN make -C contrib install - -# RUN cd project && make - -# CMD ["/bin/bash", "~/project/text.sh"] \ No newline at end of file +# RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./test.sh" +ENV PGINSTALL /usr/local/pgsql +RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./create_replica.sh" diff --git a/create_replica.sh b/create_replica.sh new file mode 100755 index 0000000..8fdbf09 --- /dev/null +++ b/create_replica.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Taken from https://github.com/afiskon/pgscripts/blob/master/install.sh + +set -e + +if [[ -z $PGINSTALL ]]; then + echo "ERROR: \$PGINSTALL environment variable is empty" + exit 1 +fi + +M=$PGINSTALL +U=`whoami` + +pkill -9 postgres || true + +# rm -rf $M || true +# mkdir $M + +make install + +$M/bin/initdb -D $M/data-master + +# probably not cheap! +echo "wal_consistency_checking = 'all'" >> $M/data-master/postgresql.conf +echo "max_prepared_transactions = 100" >> $M/data-master/postgresql.conf +echo "wal_level = logical" >> $M/data-master/postgresql.conf +echo "wal_keep_segments = 128" >> $M/data-master/postgresql.conf +# keep max_connections large enough or `make installcheck-world` may fail during `prep` test +echo "max_connections = 100" >> $M/data-master/postgresql.conf +echo "wal_log_hints = on" >> $M/data-master/postgresql.conf +echo "max_wal_senders = 8" >> $M/data-master/postgresql.conf +echo "wal_keep_segments = 64" >> $M/data-master/postgresql.conf +echo "listen_addresses = '*'" >> $M/data-master/postgresql.conf +echo "hot_standby = on" >> $M/data-master/postgresql.conf +echo "log_statement = all" >> $M/data-master/postgresql.conf +echo "max_locks_per_transaction = 256" >> $M/data-master/postgresql.conf +#echo "shared_buffers = 1GB" >> $M/data-master/postgresql.conf +#echo "fsync = off" >> $M/data-master/postgresql.conf +#echo "autovacuum = off" >> $M/data-master/postgresql.conf + +echo "host replication $U 127.0.0.1/24 trust" >> $M/data-master/pg_hba.conf +echo "host all $U 127.0.0.1/24 trust" >> $M/data-master/pg_hba.conf +echo "host all all 10.128.0.0/16 trust" >> $M/data-master/pg_hba.conf + +# CREATE ROLE scram_role LOGIN PASSWORD ('pass' USING 'scram'); +# CREATE DATABASE scram_role; +# GRANT ALL privileges ON DATABASE scram_role TO scram_role; +# psql -U scram_role + +echo '' > $M/data-master/logfile + +echo "=== STARTING MASTER ===" + +$M/bin/pg_ctl -w -D $M/data-master -l $M/data-master/logfile start +# $M/bin/createdb $U +$M/bin/psql -c "create table test(k int primary key, v text);" + +echo "=== RUNNING PG_BASEBACKUP ===" + +$M/bin/pg_basebackup -P -R -X stream -c fast -h 127.0.0.1 -U $U -D $M/data-slave +echo "port = 5433" >> $M/data-slave/postgresql.conf + +echo "=== STARTING SLAVE ===" + +$M/bin/pg_ctl -w -D $M/data-slave -l $M/data-slave/logfile start + +$M/bin/psql postgres -c "create extension cube;" +$M/bin/psql postgres -c "create extension ags;" + +$M/bin/psql postgres -c "create table x as select cube(random()) c from generate_series(1,10000) y;" +$M/bin/psql postgres -c "create index on x using ags(c);" +$M/bin/psql postgres -c "set enable_bitmapscan to off;explain analyze select * from x where c <@ cube(0,0.1);" +$M/bin/psql postgres -c "delete from x where (c~>1)>0.1;" diff --git a/gist.c b/gist.c index 60301b0..0602392 100644 --- a/gist.c +++ b/gist.c @@ -16,6 +16,7 @@ #include "gist_private.h" #include "gistscan.h" +#include "access/generic_xlog.h" #include "catalog/pg_collation.h" #include "miscadmin.h" #include "storage/lmgr.h" @@ -30,14 +31,14 @@ /* non-export function prototypes */ static void gistfixsplit(GISTInsertState *state, GISTSTATE *giststate); static bool gistinserttuple(GISTInsertState *state, GISTInsertStack *stack, - GISTSTATE *giststate, IndexTuple tuple, OffsetNumber oldoffnum); + GISTSTATE *giststate, IndexTuple tuple, OffsetNumber oldoffnum); static bool gistinserttuples(GISTInsertState *state, GISTInsertStack *stack, - GISTSTATE *giststate, - IndexTuple *tuples, int ntup, OffsetNumber oldoffnum, - Buffer leftchild, Buffer rightchild, - bool unlockbuf, bool unlockleftchild, int ndeltup, OffsetNumber skipoffnum); + GISTSTATE *giststate, + IndexTuple *tuples, int ntup, OffsetNumber oldoffnum, + Buffer leftchild, Buffer rightchild, + bool unlockbuf, bool unlockleftchild, int ndeltup, OffsetNumber skipoffnum); static void gistfinishsplit(GISTInsertState *state, GISTInsertStack *stack, - GISTSTATE *giststate, List *splitinfo, bool releasebuf); + GISTSTATE *giststate, List *splitinfo, bool releasebuf); static void gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel); @@ -59,7 +60,7 @@ PG_FUNCTION_INFO_V1(agshandler); * and callbacks. */ Datum -agshandler(PG_FUNCTION_ARGS) + agshandler(PG_FUNCTION_ARGS) { IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); @@ -220,17 +221,18 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, Buffer buffer, IndexTuple *itup, int ntup, OffsetNumber oldoffnum, BlockNumber *newblkno, - Buffer leftchildbuf, - List **splitinfo, - bool markfollowright, - Relation heapRel, - bool is_build, - int ndeltup, + Buffer leftchildbuf, + List **splitinfo, + bool markfollowright, + Relation heapRel, + bool is_build, + int ndeltup, OffsetNumber skipoffnum) { BlockNumber blkno = BufferGetBlockNumber(buffer); Page page = BufferGetPage(buffer); bool is_leaf = (GistPageIsLeaf(page)) ? true : false; + GenericXLogState *state; XLogRecPtr recptr; int i; bool is_split; @@ -525,9 +527,16 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, else { if (RelationNeedsWAL(rel)) - recptr = gistXLogSplit(is_leaf, - dist, oldrlink, oldnsn, leftchildbuf, - markfollowright); + { + state = GenericXLogStart(rel); + for (ptr = dist; ptr; ptr = ptr->next) + { + GenericXLogRegisterBuffer(state, ptr->buffer, GENERIC_XLOG_FULL_IMAGE); + } + + recptr = GenericXLogFinish(state); + } + else recptr = gistGetFakeLSN(rel); } @@ -604,19 +613,10 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate, { if (RelationNeedsWAL(rel)) { - OffsetNumber ndeloffs = 0, - deloffs[BLCKSZ /sizeof(ItemIdData)]; - - if (OffsetNumberIsValid(oldoffnum)) - { - for (i = 0; i < ndeltup; i++) - deloffs[i] = oldoffnum + i; - ndeloffs = ndeltup; - } - - recptr = gistXLogUpdate(buffer, - deloffs, ndeloffs, itup, ntup, - leftchildbuf, skipoffnum); + state = GenericXLogStart(rel); + GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE); + GenericXLogRegisterBuffer(state, leftchildbuf, GENERIC_XLOG_FULL_IMAGE); + recptr = GenericXLogFinish(state); } else recptr = gistGetFakeLSN(rel); @@ -1414,7 +1414,7 @@ inline void gistcheckskippage(Page page) { OffsetNumber i, - maxoff; + maxoff; int skiplast = 0; bool wasskip = false; Assert(false); @@ -1485,7 +1485,7 @@ gisttestskipgroup(GISTInsertState *state, GISTInsertStack *stack, } } gistinserttuples(state, stack, giststate, itvec, totalsize, skipoffnum, - InvalidBuffer, InvalidBuffer, false, false, skipsize + 1, InvalidOffsetNumber); + InvalidBuffer, InvalidBuffer, false, false, skipsize + 1, InvalidOffsetNumber); } } @@ -1647,7 +1647,7 @@ gistSplit(Relation r, if (!gistfitpage(lvectup, v.splitVector.spl_nleft)) { SplitedPageLayout *resptr, - *subres; + *subres; resptr = subres = gistSplit(r, page, lvectup, v.splitVector.spl_nleft, giststate); @@ -1677,7 +1677,7 @@ gistSplitBySkipgroup(Relation r, GISTSTATE *giststate) { IndexTuple *lvectup, - *rvectup, + *rvectup, *skiptuples; OffsetNumber *skipoffsets; GistSplitVector v; @@ -1760,7 +1760,7 @@ gistSplitBySkipgroup(Relation r, if (!gistfitpage(lvectup, v.splitVector.spl_nleft)) { SplitedPageLayout *resptr, - *subres; + *subres; resptr = subres = gistSplitBySkipgroup(r, page, lvectup, v.splitVector.spl_nleft, giststate); if (subres == NULL) @@ -1972,10 +1972,11 @@ gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel) if (RelationNeedsWAL(rel)) { XLogRecPtr recptr; + GenericXLogState *state; - recptr = gistXLogDelete(buffer, - deletable, ndeletable, - heapRel->rd_node); + state = GenericXLogStart(rel); + GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE); + recptr = GenericXLogFinish(state); PageSetLSN(page, recptr); } diff --git a/gistvacuum.c b/gistvacuum.c index 59d4bff..8f684ea 100644 --- a/gistvacuum.c +++ b/gistvacuum.c @@ -17,6 +17,7 @@ #include "access/genam.h" #include "gist_private.h" #include "access/transam.h" +#include "access/generic_xlog.h" #include "commands/vacuum.h" #include "lib/integerset.h" #include "miscadmin.h" @@ -367,10 +368,12 @@ gistvacuumpage(GistVacState *vstate, BlockNumber blkno, BlockNumber orig_blkno) if (RelationNeedsWAL(rel)) { XLogRecPtr recptr; + GenericXLogState *state; + + state = GenericXLogStart(rel); + GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE); + recptr = GenericXLogFinish(state); - recptr = gistXLogDelete(buffer, - todelete, ntodelete, - rel->rd_node); PageSetLSN(page, recptr); } else @@ -663,7 +666,13 @@ gistdeletepage(IndexVacuumInfo *info, GistBulkDeleteResult *stats, PageIndexTupleDelete(parentPage, downlink); if (RelationNeedsWAL(info->index)) - recptr = gistXLogPageDelete(leafBuffer, txid, parentBuffer, downlink); + { + GenericXLogState *state; + + state = GenericXLogStart(info->index); + GenericXLogRegisterBuffer(state, leafBuffer, GENERIC_XLOG_FULL_IMAGE); + recptr = GenericXLogFinish(state); + } else recptr = gistGetFakeLSN(info->index); PageSetLSN(parentPage, recptr);