Skip to content

Commit e199c4a

Browse files
committed
Use CRC-32C again. Calculate crc for pg_control differently using ControlFileData struct.
Bump version to 2.0.25.
1 parent 990d67a commit e199c4a

File tree

9 files changed

+98
-36
lines changed

9 files changed

+98
-36
lines changed

src/backup.c

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
307307
to_path, strerror(errno_tmp));
308308
}
309309

310-
INIT_FILE_CRC32(false, file->crc);
310+
INIT_FILE_CRC32(true, file->crc);
311311

312312
/* read from stream and write to backup file */
313313
while (1)
@@ -333,14 +333,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
333333
{
334334
write_buffer_size = Min(row_length, sizeof(buf));
335335
memcpy(buf, copybuf, write_buffer_size);
336-
COMP_FILE_CRC32(false, file->crc, buf, write_buffer_size);
336+
COMP_FILE_CRC32(true, file->crc, buf, write_buffer_size);
337337

338338
/* TODO calc checksum*/
339339
if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
340340
{
341341
errno_tmp = errno;
342342
/* oops */
343-
FIN_FILE_CRC32(false, file->crc);
343+
FIN_FILE_CRC32(true, file->crc);
344344
fclose(out);
345345
PQfinish(conn);
346346
elog(ERROR, "cannot write to \"%s\": %s", to_path,
@@ -364,7 +364,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
364364
}
365365

366366
file->write_size = (int64) file->read_size;
367-
FIN_FILE_CRC32(false, file->crc);
367+
FIN_FILE_CRC32(true, file->crc);
368368

369369
fclose(out);
370370
}
@@ -763,7 +763,8 @@ do_backup_instance(void)
763763
{
764764
char pg_control_path[MAXPGPATH];
765765

766-
snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", pgdata, "global/pg_control");
766+
snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s",
767+
pgdata,"global/pg_control");
767768

768769
for (i = 0; i < parray_num(backup_files_list); i++)
769770
{
@@ -2262,9 +2263,12 @@ backup_files(void *arg)
22622263
continue;
22632264
}
22642265
}
2266+
else if (strcmp(file->name, "pg_control") == 0)
2267+
copy_pgcontrol_file(arguments->from_root, arguments->to_root,
2268+
file);
22652269
else
22662270
{
2267-
bool skip = false;
2271+
bool skip = false;
22682272

22692273
/* If non-data file has not changed since last backup... */
22702274
if (prev_file && file->exists_in_prev &&

src/data.c

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
487487
blknum, header.compressed_size, write_buffer_size); */
488488

489489
/* Update CRC */
490-
COMP_FILE_CRC32(false, *crc, write_buffer, write_buffer_size);
490+
COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size);
491491

492492
/* write data page */
493493
if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
@@ -547,13 +547,13 @@ backup_data_file(backup_files_arg* arguments,
547547
/* reset size summary */
548548
file->read_size = 0;
549549
file->write_size = 0;
550-
INIT_FILE_CRC32(false, file->crc);
550+
INIT_FILE_CRC32(true, file->crc);
551551

552552
/* open backup mode file for read */
553553
in = fopen(file->path, PG_BINARY_R);
554554
if (in == NULL)
555555
{
556-
FIN_FILE_CRC32(false, file->crc);
556+
FIN_FILE_CRC32(true, file->crc);
557557

558558
/*
559559
* If file is not found, this is not en error.
@@ -658,7 +658,7 @@ backup_data_file(backup_files_arg* arguments,
658658
to_path, strerror(errno));
659659
fclose(in);
660660

661-
FIN_FILE_CRC32(false, file->crc);
661+
FIN_FILE_CRC32(true, file->crc);
662662

663663
/*
664664
* If we have pagemap then file in the backup can't be a zero size.
@@ -927,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
927927
struct stat st;
928928
pg_crc32 crc;
929929

930-
INIT_FILE_CRC32(false, crc);
930+
INIT_FILE_CRC32(true, crc);
931931

932932
/* reset size summary */
933933
file->read_size = 0;
@@ -937,7 +937,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
937937
in = fopen(file->path, PG_BINARY_R);
938938
if (in == NULL)
939939
{
940-
FIN_FILE_CRC32(false, crc);
940+
FIN_FILE_CRC32(true, crc);
941941
file->crc = crc;
942942

943943
/* maybe deleted, it's not error */
@@ -986,7 +986,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
986986
strerror(errno_tmp));
987987
}
988988
/* update CRC */
989-
COMP_FILE_CRC32(false, crc, buf, read_len);
989+
COMP_FILE_CRC32(true, crc, buf, read_len);
990990

991991
file->read_size += read_len;
992992
}
@@ -1013,14 +1013,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
10131013
strerror(errno_tmp));
10141014
}
10151015
/* update CRC */
1016-
COMP_FILE_CRC32(false, crc, buf, read_len);
1016+
COMP_FILE_CRC32(true, crc, buf, read_len);
10171017

10181018
file->read_size += read_len;
10191019
}
10201020

10211021
file->write_size = (int64) file->read_size;
10221022
/* finish CRC calculation and store into pgFile */
1023-
FIN_FILE_CRC32(false, crc);
1023+
FIN_FILE_CRC32(true, crc);
10241024
file->crc = crc;
10251025

10261026
/* update file permission */
@@ -1423,7 +1423,7 @@ calc_file_checksum(pgFile *file)
14231423
{
14241424
Assert(S_ISREG(file->mode));
14251425

1426-
file->crc = pgFileGetCRC(file->path, false, false, &file->read_size);
1426+
file->crc = pgFileGetCRC(file->path, true, false, &file->read_size);
14271427
file->write_size = file->read_size;
14281428
}
14291429

@@ -1546,14 +1546,14 @@ validate_one_page(Page page, pgFile *file,
15461546

15471547
/* Valiate pages of datafile in backup one by one */
15481548
bool
1549-
check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
1550-
uint32 checksum_version, uint32 backup_version)
1549+
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
1550+
uint32 backup_version)
15511551
{
15521552
size_t read_len = 0;
15531553
bool is_valid = true;
15541554
FILE *in;
15551555
pg_crc32 crc;
1556-
bool use_crc32c = (backup_version <= 20021);
1556+
bool use_crc32c = backup_version <= 20021 || backup_version >= 20025;
15571557

15581558
elog(VERBOSE, "validate relation blocks for file %s", file->name);
15591559

src/merge.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,9 +524,11 @@ merge_files(void *arg)
524524
* do that.
525525
*/
526526
file->write_size = pgFileSize(to_path_tmp);
527-
file->crc = pgFileGetCRC(to_path_tmp, false, true, NULL);
527+
file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL);
528528
}
529529
}
530+
else if (strcmp(file->name, "pg_control") == 0)
531+
copy_pgcontrol_file(argument->from_root, argument->to_root, file);
530532
else
531533
copy_file(argument->from_root, argument->to_root, file);
532534

src/pg_probackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include "utils/thread.h"
1919
#include <time.h>
2020

21-
const char *PROGRAM_VERSION = "2.0.24";
21+
const char *PROGRAM_VERSION = "2.0.25";
2222
const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup";
2323
const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues";
2424

src/pg_probackup.h

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ do { \
9191
FIN_TRADITIONAL_CRC32(crc); \
9292
} while (0)
9393

94+
9495
/* Information about single file (or dir) in backup */
9596
typedef struct pgFile
9697
{
@@ -555,8 +556,7 @@ extern void get_wal_file(const char *from_path, const char *to_path);
555556

556557
extern void calc_file_checksum(pgFile *file);
557558

558-
extern bool check_file_pages(pgFile* file,
559-
XLogRecPtr stop_lsn,
559+
extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
560560
uint32 checksum_version, uint32 backup_version);
561561
/* parsexlog.c */
562562
extern void extractPageMap(const char *archivedir,
@@ -583,11 +583,15 @@ extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn,
583583
/* in util.c */
584584
extern TimeLineID get_current_timeline(bool safe);
585585
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
586-
extern uint64 get_system_identifier(char *pgdata);
586+
extern uint64 get_system_identifier(const char *pgdata_path);
587587
extern uint64 get_remote_system_identifier(PGconn *conn);
588588
extern uint32 get_data_checksum_version(bool safe);
589+
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
589590
extern uint32 get_xlog_seg_size(char *pgdata_path);
590-
extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn);
591+
extern void set_min_recovery_point(pgFile *file, const char *backup_path,
592+
XLogRecPtr stop_backup_lsn);
593+
extern void copy_pgcontrol_file(const char *from_root, const char *to_root,
594+
pgFile *file);
591595

592596
extern void sanityChecks(void);
593597
extern void time2iso(char *buf, size_t len, time_t time);

src/restore.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,8 @@ restore_files(void *arg)
628628
false,
629629
parse_program_version(arguments->backup->program_version));
630630
}
631+
else if (strcmp(file->name, "pg_control") == 0)
632+
copy_pgcontrol_file(from_root, pgdata, file);
631633
else
632634
copy_file(from_root, pgdata, file);
633635

src/util.c

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ get_checkpoint_location(PGconn *conn)
206206
}
207207

208208
uint64
209-
get_system_identifier(char *pgdata_path)
209+
get_system_identifier(const char *pgdata_path)
210210
{
211211
ControlFileData ControlFile;
212212
char *buffer;
@@ -293,7 +293,27 @@ get_data_checksum_version(bool safe)
293293
return ControlFile.data_checksum_version;
294294
}
295295

296-
/* MinRecoveryPoint 'as-is' is not to be trusted */
296+
pg_crc32c
297+
get_pgcontrol_checksum(const char *pgdata_path)
298+
{
299+
ControlFileData ControlFile;
300+
char *buffer;
301+
size_t size;
302+
303+
/* First fetch file... */
304+
buffer = slurpFile(pgdata_path, "global/pg_control", &size, false);
305+
if (buffer == NULL)
306+
return 0;
307+
digestControlFile(&ControlFile, buffer, size);
308+
pg_free(buffer);
309+
310+
return ControlFile.crc;
311+
}
312+
313+
/*
314+
* Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint
315+
* 'as-is' is not to be trusted.
316+
*/
297317
void
298318
set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn)
299319
{
@@ -321,24 +341,44 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba
321341

322342
/* Update checksum in pg_control header */
323343
INIT_CRC32C(ControlFile.crc);
324-
COMP_CRC32C(ControlFile.crc,
325-
(char *) &ControlFile,
344+
COMP_CRC32C(ControlFile.crc, (char *) &ControlFile,
326345
offsetof(ControlFileData, crc));
327346
FIN_CRC32C(ControlFile.crc);
328347

329-
/* paranoia */
330-
checkControlFile(&ControlFile);
331-
332348
/* overwrite pg_control */
333349
snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE);
334350
writeControlFile(&ControlFile, fullpath);
335351

336352
/* Update pg_control checksum in backup_list */
337-
file->crc = pgFileGetCRC(fullpath, false, true, NULL);
353+
file->crc = ControlFile.crc;
338354

339355
pg_free(buffer);
340356
}
341357

358+
/*
359+
* Copy pg_control file to backup. We do not apply compression to this file.
360+
*/
361+
void
362+
copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file)
363+
{
364+
ControlFileData ControlFile;
365+
char *buffer;
366+
size_t size;
367+
char to_path[MAXPGPATH];
368+
369+
buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false);
370+
371+
digestControlFile(&ControlFile, buffer, size);
372+
373+
file->crc = ControlFile.crc;
374+
file->read_size = size;
375+
file->write_size = size;
376+
377+
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
378+
writeControlFile(&ControlFile, to_path);
379+
380+
pg_free(buffer);
381+
}
342382

343383
/*
344384
* Convert time_t value to ISO-8601 format string. Always set timezone offset.

src/validate.c

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ static bool corrupted_backup_found = false;
2222

2323
typedef struct
2424
{
25+
const char *base_path;
2526
parray *files;
2627
bool corrupted;
2728
XLogRecPtr stop_lsn;
@@ -101,6 +102,7 @@ pgBackupValidate(pgBackup *backup)
101102
{
102103
validate_files_arg *arg = &(threads_args[i]);
103104

105+
arg->base_path = base_path;
104106
arg->files = files;
105107
arg->corrupted = false;
106108
arg->stop_lsn = backup->stop_lsn;
@@ -223,9 +225,17 @@ pgBackupValidateFiles(void *arg)
223225
* CRC-32C algorithm.
224226
* To avoid this problem we need to use different algorithm, CRC-32 in
225227
* this case.
228+
*
229+
* Starting from 2.0.25 we calculate crc of pg_control differently.
226230
*/
227-
crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021,
228-
true, NULL);
231+
if (arguments->backup_version >= 20025 &&
232+
strcmp(file->name, "pg_control") == 0)
233+
crc = get_pgcontrol_checksum(arguments->base_path);
234+
else
235+
crc = pgFileGetCRC(file->path,
236+
arguments->backup_version <= 20021 ||
237+
arguments->backup_version >= 20025,
238+
true, NULL);
229239
if (crc != file->crc)
230240
{
231241
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",

tests/expected/option_version.out

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pg_probackup 2.0.24
1+
pg_probackup 2.0.25

0 commit comments

Comments
 (0)