From 41855701c7033ff358de8b56822ce607dd3303c9 Mon Sep 17 00:00:00 2001
From: Victor Spirin <v.spirin@postgrespro.ru>
Date: Wed, 8 Jun 2022 00:38:51 +0300
Subject: [PATCH 1/4] [PBCKP-153] Added waldir option for location for the
write-ahead log directory (-X, --waldir=WALDIR)
---
src/dir.c | 30 +++++++++++++++++++++++++++++-
src/help.c | 6 ++++++
src/merge.c | 2 +-
src/pg_probackup.c | 17 +++++++++++++++++
src/pg_probackup.h | 5 ++++-
src/restore.c | 2 +-
6 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/src/dir.c b/src/dir.c
index 4ebe0939b..ac794cee4 100644
--- a/src/dir.c
+++ b/src/dir.c
@@ -1036,13 +1036,20 @@ opt_externaldir_map(ConfigOption *opt, const char *arg)
*/
void
create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir,
- bool extract_tablespaces, bool incremental, fio_location location)
+ bool extract_tablespaces, bool incremental, fio_location location,
+ const char* waldir_path)
{
int i;
parray *links = NULL;
mode_t pg_tablespace_mode = DIR_PERMISSION;
char to_path[MAXPGPATH];
+ if (waldir_path && !dir_is_empty(waldir_path, location))
+ {
+ elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path);
+ }
+
+
/* get tablespace map */
if (extract_tablespaces)
{
@@ -1107,6 +1114,27 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
/* skip external directory content */
if (dir->external_dir_num != 0)
continue;
+ /* Create WAL directory and symlink if waldir_path is setting */
+ if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) {
+ /* get full path to PG_XLOG_DIR */
+
+ join_path_components(to_path, data_dir, PG_XLOG_DIR);
+
+ elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"",
+ waldir_path, to_path);
+
+ /* create tablespace directory from waldir_path*/
+ fio_mkdir(waldir_path, pg_tablespace_mode, location);
+
+ /* create link to linked_path */
+ if (fio_symlink(waldir_path, to_path, incremental, location) < 0)
+ elog(ERROR, "Could not create symbolic link \"%s\": %s",
+ to_path, strerror(errno));
+
+ continue;
+
+
+ }
/* tablespace_map exists */
if (links)
diff --git a/src/help.c b/src/help.c
index b22fa912e..85894759e 100644
--- a/src/help.c
+++ b/src/help.c
@@ -169,6 +169,7 @@ help_pg_probackup(void)
printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n"));
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--skip-external-dirs] [--no-sync]\n"));
+ printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
printf(_(" [--db-include | --db-exclude]\n"));
printf(_(" [--remote-proto] [--remote-host]\n"));
@@ -435,6 +436,7 @@ help_restore(void)
printf(_(" [-T OLDDIR=NEWDIR]\n"));
printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n"));
printf(_(" [--skip-external-dirs]\n"));
+ printf(_(" [-X WALDIR | --waldir=WALDIR]\n"));
printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n"));
printf(_(" [--db-include dbname | --db-exclude dbname]\n"));
printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n"));
@@ -472,6 +474,10 @@ help_restore(void)
printf(_(" relocate the external directory from OLDDIR to NEWDIR\n"));
printf(_(" --skip-external-dirs do not restore all external directories\n"));
+
+ printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n"));
+
+
printf(_("\n Incremental restore options:\n"));
printf(_(" -I, --incremental-mode=none|checksum|lsn\n"));
printf(_(" reuse valid pages available in PGDATA if they have not changed\n"));
diff --git a/src/merge.c b/src/merge.c
index ff39c2510..1ce92bb42 100644
--- a/src/merge.c
+++ b/src/merge.c
@@ -614,7 +614,7 @@ merge_chain(InstanceState *instanceState,
/* Create directories */
create_data_directories(dest_backup->files, full_database_dir,
- dest_backup->root_dir, false, false, FIO_BACKUP_HOST);
+ dest_backup->root_dir, false, false, FIO_BACKUP_HOST, NULL);
/* External directories stuff */
if (dest_backup->external_dir_str)
diff --git a/src/pg_probackup.c b/src/pg_probackup.c
index b9b3af0b9..2c8100b83 100644
--- a/src/pg_probackup.c
+++ b/src/pg_probackup.c
@@ -122,6 +122,7 @@ static parray *datname_include_list = NULL;
/* arrays for --exclude-path's */
static parray *exclude_absolute_paths_list = NULL;
static parray *exclude_relative_paths_list = NULL;
+static char* waldir_path = NULL;
/* checkdb options */
bool need_amcheck = false;
@@ -238,6 +239,7 @@ static ConfigOption cmd_options[] =
{ 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT },
{ 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT },
{ 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT },
+ { 's', 'X', "waldir", &waldir_path, SOURCE_CMD_STRICT },
/* checkdb options */
{ 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT },
{ 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT },
@@ -754,6 +756,21 @@ main(int argc, char *argv[])
restore_params->partial_restore_type = INCLUDE;
restore_params->partial_db_list = datname_include_list;
}
+
+ if (waldir_path)
+ {
+ /* clean up xlog directory name, check it's absolute */
+ canonicalize_path(waldir_path);
+ if (!is_absolute_path(waldir_path))
+ {
+ elog(ERROR, "WAL directory location must be an absolute path");
+ }
+ if (strlen(waldir_path) > MAXPGPATH)
+ elog(ERROR, "Value specified to --waldir is too long");
+
+ }
+ restore_params->waldir = waldir_path;
+
}
/*
diff --git a/src/pg_probackup.h b/src/pg_probackup.h
index 2c4c61036..13650be8b 100644
--- a/src/pg_probackup.h
+++ b/src/pg_probackup.h
@@ -566,6 +566,8 @@ typedef struct pgRestoreParams
/* options for partial restore */
PartialRestoreType partial_restore_type;
parray *partial_db_list;
+
+ char* waldir;
} pgRestoreParams;
/* Options needed for set-backup command */
@@ -1022,7 +1024,8 @@ extern void create_data_directories(parray *dest_files,
const char *backup_dir,
bool extract_tablespaces,
bool incremental,
- fio_location location);
+ fio_location location,
+ const char *waldir_path);
extern void read_tablespace_map(parray *links, const char *backup_dir);
extern void opt_tablespace_map(ConfigOption *opt, const char *arg);
diff --git a/src/restore.c b/src/restore.c
index d8d808a4e..fbf0c0398 100644
--- a/src/restore.c
+++ b/src/restore.c
@@ -801,7 +801,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
create_data_directories(dest_files, instance_config.pgdata,
dest_backup->root_dir, backup_has_tblspc,
params->incremental_mode != INCR_NONE,
- FIO_DB_HOST);
+ FIO_DB_HOST, params->waldir);
/*
* Restore dest_backup external directories.
From 48a2c835d1c12353e23e08b901beaf39695773f9 Mon Sep 17 00:00:00 2001
From: Victor Spirin <v.spirin@postgrespro.ru>
Date: Wed, 8 Jun 2022 17:40:49 +0300
Subject: [PATCH 2/4] [PBCKP-153] Added a test for the waldir option for the
restore command
---
tests/restore.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
diff --git a/tests/restore.py b/tests/restore.py
index bbdadeb23..668cff4fd 100644
--- a/tests/restore.py
+++ b/tests/restore.py
@@ -3916,3 +3916,59 @@ def test_restore_issue_313(self):
# Clean after yourself
self.del_test_dir(module_name, fname)
+
+ # @unittest.skip("skip")
+ def test_restore_with_waldir(self):
+ """recovery using tablespace-mapping option and page backup"""
+ fname = self.id().split('.')[3]
+ node = self.make_simple_node(
+ base_dir=os.path.join(module_name, fname, 'node'),
+ initdb_params=['--data-checksums'])
+
+ backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+ self.init_pb(backup_dir)
+ self.add_instance(backup_dir, 'node', node)
+ self.set_archiving(backup_dir, 'node', node)
+ node.slow_start()
+
+
+ with node.connect("postgres") as con:
+ con.execute(
+ "CREATE TABLE tbl AS SELECT * "
+ "FROM generate_series(0,3) AS integer")
+ con.commit()
+
+ # Full backup
+ backup_id = self.backup_node(backup_dir, 'node', node)
+
+ node.stop()
+ node.cleanup()
+
+ # Create waldir
+ waldir_path = os.path.join(node.base_dir, "waldir")
+ os.makedirs(waldir_path)
+
+ # Test recovery from latest
+ self.assertIn(
+ "INFO: Restore of backup {0} completed.".format(backup_id),
+ self.restore_node(
+ backup_dir, 'node', node,
+ options=[
+ "-X", "%s" % (waldir_path)]),
+ '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+ repr(self.output), self.cmd))
+ node.slow_start()
+
+ count = node.execute("postgres", "SELECT count(*) FROM tbl")
+ self.assertEqual(count[0][0], 4)
+
+ # check pg_wal is symlink
+ if node.major_version >= 10:
+ wal_path=os.path.join(node.data_dir, "pg_wal")
+ else:
+ wal_path=os.path.join(node.data_dir, "pg_xlog")
+
+ self.assertEqual(os.path.islink(wal_path), True)
+
+ # Clean after yourself
+ self.del_test_dir(module_name, fname)
From e11ca786b1a466aff773ebec5fae0b88692d140d Mon Sep 17 00:00:00 2001
From: Victor Spirin <v.spirin@postgrespro.ru>
Date: Thu, 16 Jun 2022 12:02:27 +0300
Subject: [PATCH 3/4] [PBCKP-153] Changed expected/option_help.out and
option_help_ru.out files for the tests.option.OptionTest.test_help_1 and
help_6
---
tests/expected/option_help.out | 1 +
tests/expected/option_help_ru.out | 2 ++
2 files changed, 3 insertions(+)
diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out
index 8a1de1f67..659164250 100644
--- a/tests/expected/option_help.out
+++ b/tests/expected/option_help.out
@@ -86,6 +86,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
[-T OLDDIR=NEWDIR] [--progress]
[--external-mapping=OLDDIR=NEWDIR]
[--skip-external-dirs] [--no-sync]
+ [-X WALDIR | --waldir=WALDIR]
[-I | --incremental-mode=none|checksum|lsn]
[--db-include | --db-exclude]
[--remote-proto] [--remote-host]
diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out
index ee8da9a1c..2e90eb297 100644
--- a/tests/expected/option_help_ru.out
+++ b/tests/expected/option_help_ru.out
@@ -86,6 +86,7 @@ pg_probackup - утилита для управления резервным к
[-T OLDDIR=NEWDIR] [--progress]
[--external-mapping=OLDDIR=NEWDIR]
[--skip-external-dirs] [--no-sync]
+ [-X WALDIR | --waldir=WALDIR]
[-I | --incremental-mode=none|checksum|lsn]
[--db-include | --db-exclude]
[--remote-proto] [--remote-host]
@@ -178,6 +179,7 @@ pg_probackup - утилита для управления резервным к
[--remote-proto] [--remote-host]
[--remote-port] [--remote-path] [--remote-user]
[--ssh-options]
+ [--dry-run]
[--help]
Подробнее читайте на сайте <https://github.com/postgrespro/pg_probackup>.
From 61cd6209772c8ac8ec34a80444a074f66650a4bf Mon Sep 17 00:00:00 2001
From: Victor Spirin <v.spirin@postgrespro.ru>
Date: Fri, 24 Jun 2022 11:36:56 +0300
Subject: [PATCH 4/4] [PBCKP-153] global variable waldir_path renamed to
gl_waldir_path
---
src/pg_probackup.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/pg_probackup.c b/src/pg_probackup.c
index 2c8100b83..193cd9c39 100644
--- a/src/pg_probackup.c
+++ b/src/pg_probackup.c
@@ -122,7 +122,7 @@ static parray *datname_include_list = NULL;
/* arrays for --exclude-path's */
static parray *exclude_absolute_paths_list = NULL;
static parray *exclude_relative_paths_list = NULL;
-static char* waldir_path = NULL;
+static char* gl_waldir_path = NULL;
/* checkdb options */
bool need_amcheck = false;
@@ -239,7 +239,7 @@ static ConfigOption cmd_options[] =
{ 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT },
{ 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT },
{ 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT },
- { 's', 'X', "waldir", &waldir_path, SOURCE_CMD_STRICT },
+ { 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT },
/* checkdb options */
{ 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT },
{ 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT },
@@ -757,19 +757,19 @@ main(int argc, char *argv[])
restore_params->partial_db_list = datname_include_list;
}
- if (waldir_path)
+ if (gl_waldir_path)
{
/* clean up xlog directory name, check it's absolute */
- canonicalize_path(waldir_path);
- if (!is_absolute_path(waldir_path))
+ canonicalize_path(gl_waldir_path);
+ if (!is_absolute_path(gl_waldir_path))
{
elog(ERROR, "WAL directory location must be an absolute path");
}
- if (strlen(waldir_path) > MAXPGPATH)
+ if (strlen(gl_waldir_path) > MAXPGPATH)
elog(ERROR, "Value specified to --waldir is too long");
}
- restore_params->waldir = waldir_path;
+ restore_params->waldir = gl_waldir_path;
}