diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6f352fd5be4756a62dd3f4ec39583b2ee2192a40..30d877b6fdb9776e3dd8f499ea43ccbe33ed7bab 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -3456,19 +3456,36 @@ PreallocXlogFiles(XLogRecPtr endptr)
}
/*
- * Get the log/seg of the latest removed or recycled WAL segment.
- * Returns 0/0 if no WAL segments have been removed since startup.
+ * Throws an error if the given log segment has already been removed or
+ * recycled. The caller should only pass a segment that it knows to have
+ * existed while the server has been running, as this function always
+ * succeeds if no WAL segments have been removed since startup.
+ * 'tli' is only used in the error message.
*/
void
-XLogGetLastRemoved(uint32 *log, uint32 *seg)
+CheckXLogRemoved(uint32 log, uint32 seg, TimeLineID tli)
{
/* use volatile pointer to prevent code rearrangement */
volatile XLogCtlData *xlogctl = XLogCtl;
+ uint32 lastRemovedLog,
+ lastRemovedSeg;
SpinLockAcquire(&xlogctl->info_lck);
- *log = xlogctl->lastRemovedLog;
- *seg = xlogctl->lastRemovedSeg;
+ lastRemovedLog = xlogctl->lastRemovedLog;
+ lastRemovedSeg = xlogctl->lastRemovedSeg;
SpinLockRelease(&xlogctl->info_lck);
+
+ if (log < lastRemovedLog ||
+ (log == lastRemovedLog && seg <= lastRemovedSeg))
+ {
+ char filename[MAXFNAMELEN];
+
+ XLogFileName(filename, tli, log, seg);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("requested WAL segment %s has already been removed",
+ filename)));
+ }
}
/*
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index bc95215457ff9182fdc653e6228aaf203a8a0780..6011630029568fbc37760c2cf711230591d27921 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -55,11 +55,10 @@ static void base_backup_cleanup(int code, Datum arg);
static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir);
static void parse_basebackup_options(List *options, basebackup_options *opt);
static void SendXlogRecPtrResult(XLogRecPtr ptr);
+static int compareWalFileNames(const void *a, const void *b);
/*
* Size of each block sent into the tar stream for larger files.
- *
- * XLogSegSize *MUST* be evenly dividable by this
*/
#define TAR_SEND_SIZE 32768
@@ -221,68 +220,208 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
* We've left the last tar file "open", so we can now append the
* required WAL files to it.
*/
+ char pathbuf[MAXPGPATH];
uint32 logid,
logseg;
+ uint32 startlogid,
+ startlogseg;
uint32 endlogid,
endlogseg;
struct stat statbuf;
+ List *historyFileList = NIL;
+ List *walFileList = NIL;
+ char **walFiles;
+ int nWalFiles;
+ char firstoff[MAXFNAMELEN];
+ char lastoff[MAXFNAMELEN];
+ DIR *dir;
+ struct dirent *de;
+ int i;
+ ListCell *lc;
+ TimeLineID tli;
- MemSet(&statbuf, 0, sizeof(statbuf));
- statbuf.st_mode = S_IRUSR | S_IWUSR;
-#ifndef WIN32
- statbuf.st_uid = geteuid();
- statbuf.st_gid = getegid();
-#endif
- statbuf.st_size = XLogSegSize;
- statbuf.st_mtime = time(NULL);
-
- XLByteToSeg(startptr, logid, logseg);
+ /*
+ * I'd rather not worry about timelines here, so scan pg_xlog and
+ * include all WAL files in the range between 'startptr' and 'endptr',
+ * regardless of the timeline the file is stamped with. If there are
+ * some spurious WAL files belonging to timelines that don't belong
+ * in this server's history, they will be included too. Normally there
+ * shouldn't be such files, but if there are, there's little harm in
+ * including them.
+ */
+ XLByteToSeg(startptr, startlogid, startlogseg);
+ XLogFileName(firstoff, ThisTimeLineID, startlogid, startlogseg);
XLByteToPrevSeg(endptr, endlogid, endlogseg);
+ XLogFileName(lastoff, ThisTimeLineID, endlogid, endlogseg);
- while (true)
+ dir = AllocateDir("pg_xlog");
+ if (!dir)
+ ereport(ERROR,
+ (errmsg("could not open directory \"%s\": %m", "pg_xlog")));
+ while ((de = ReadDir(dir, "pg_xlog")) != NULL)
{
- /* Send another xlog segment */
- char fn[MAXPGPATH];
- int i;
+ /* Does it look like a WAL segment, and is it in the range? */
+ if (strlen(de->d_name) == 24 &&
+ strspn(de->d_name, "0123456789ABCDEF") == 24 &&
+ strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
+ strcmp(de->d_name + 8, lastoff + 8) <= 0)
+ {
+ walFileList = lappend(walFileList, pstrdup(de->d_name));
+ }
+ /* Does it look like a timeline history file? */
+ else if (strlen(de->d_name) == 8 + strlen(".history") &&
+ strspn(de->d_name, "0123456789ABCDEF") == 8 &&
+ strcmp(de->d_name + 8, ".history") == 0)
+ {
+ historyFileList = lappend(historyFileList, pstrdup(de->d_name));
+ }
+ }
+ FreeDir(dir);
- XLogFilePath(fn, ThisTimeLineID, logid, logseg);
- _tarWriteHeader(fn, NULL, &statbuf);
+ /*
+ * Before we go any further, check that none of the WAL segments we
+ * need were removed.
+ */
+ CheckXLogRemoved(startlogid, startlogseg, ThisTimeLineID);
+
+ /*
+ * Put the WAL filenames into an array, and sort. We send the files
+ * in order from oldest to newest, to reduce the chance that a file
+ * is recycled before we get a chance to send it over.
+ */
+ nWalFiles = list_length(walFileList);
+ walFiles = palloc(nWalFiles * sizeof(char *));
+ i = 0;
+ foreach(lc, walFileList)
+ {
+ walFiles[i++] = lfirst(lc);
+ }
+ qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
- /* Send the actual WAL file contents, block-by-block */
- for (i = 0; i < XLogSegSize / TAR_SEND_SIZE; i++)
+ /*
+ * Sanity check: the first and last segment should cover startptr and
+ * endptr, with no gaps in between.
+ */
+ XLogFromFileName(walFiles[0], &tli, &logid, &logseg);
+ if (logid != startlogid || logseg != startlogseg)
+ {
+ char startfname[MAXFNAMELEN];
+ XLogFileName(startfname, ThisTimeLineID, startlogid, startlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", startfname)));
+ }
+ for (i = 0; i < nWalFiles; i++)
+ {
+ int currlogid = logid,
+ currlogseg = logseg;
+ int nextlogid = logid,
+ nextlogseg = logseg;
+ NextLogSeg(nextlogid, nextlogseg);
+
+ XLogFromFileName(walFiles[i], &tli, &logid, &logseg);
+ if (!((nextlogid == logid && nextlogseg == logseg) ||
+ (currlogid == logid && currlogseg == logseg)))
{
- char buf[TAR_SEND_SIZE];
- XLogRecPtr ptr;
+ char nextfname[MAXFNAMELEN];
+ XLogFileName(nextfname, ThisTimeLineID, nextlogid, nextlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", nextfname)));
+ }
+ }
+ if (logid != endlogid || logseg != endlogseg)
+ {
+ char endfname[MAXFNAMELEN];
+ XLogFileName(endfname, ThisTimeLineID, endlogid, endlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", endfname)));
+ }
+
+ /* Ok, we have everything we need. Send the WAL files. */
+ for (i = 0; i < nWalFiles; i++)
+ {
+ FILE *fp;
+ char buf[TAR_SEND_SIZE];
+ size_t cnt;
+ pgoff_t len = 0;
- ptr.xlogid = logid;
- ptr.xrecoff = logseg * XLogSegSize + TAR_SEND_SIZE * i;
+ snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
+ XLogFromFileName(walFiles[i], &tli, &logid, &logseg);
+ fp = AllocateFile(pathbuf, "rb");
+ if (fp == NULL)
+ {
/*
- * Some old compilers, e.g. gcc 2.95.3/x86, think that passing
- * a struct in the same function as a longjump might clobber a
- * variable. bjm 2011-02-04
- * http://lists.apple.com/archives/xcode-users/2003/Dec//msg000
- * 51.html
+ * Most likely reason for this is that the file was already
+ * removed by a checkpoint, so check for that to get a better
+ * error message.
*/
- XLogRead(buf, ptr, TAR_SEND_SIZE);
- if (pq_putmessage('d', buf, TAR_SEND_SIZE))
+ CheckXLogRemoved(logid, logseg, tli);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m", pathbuf)));
+ }
+
+ if (fstat(fileno(fp), &statbuf) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ pathbuf)));
+ if (statbuf.st_size != XLogSegSize)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
+ }
+
+ _tarWriteHeader(pathbuf, NULL, &statbuf);
+
+ while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ /* Send the chunk as a CopyData message */
+ if (pq_putmessage('d', buf, cnt))
ereport(ERROR,
(errmsg("base backup could not send data, aborting backup")));
+
+ len += cnt;
+ if (len == XLogSegSize)
+ break;
}
- /*
- * Files are always fixed size, and always end on a 512 byte
- * boundary, so padding is never necessary.
- */
+ if (len != XLogSegSize)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
+ }
+ /* XLogSegSize is a multiple of 512, so no need for padding */
+ FreeFile(fp);
+ }
+
+ /*
+ * Send timeline history files too. Only the latest timeline history
+ * file is required for recovery, and even that only if there happens
+ * to be a timeline switch in the first WAL segment that contains the
+ * checkpoint record, or if we're taking a base backup from a standby
+ * server and the target timeline changes while the backup is taken.
+ * But they are small and highly useful for debugging purposes, so
+ * better include them all, always.
+ */
+ foreach(lc, historyFileList)
+ {
+ char *fname = lfirst(lc);
+ snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
- /* Advance to the next WAL file */
- NextLogSeg(logid, logseg);
+ if (lstat(pathbuf, &statbuf) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", pathbuf)));
- /* Have we reached our stop position yet? */
- if (logid > endlogid ||
- (logid == endlogid && logseg > endlogseg))
- break;
+ sendFile(pathbuf, pathbuf, &statbuf, false);
}
/* Send CopyDone message for the last tar file */
@@ -291,6 +430,19 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
SendXlogRecPtrResult(endptr);
}
+/*
+ * qsort comparison function, to compare log/seg portion of WAL segment
+ * filenames, ignoring the timeline portion.
+ */
+static int
+compareWalFileNames(const void *a, const void *b)
+{
+ char *fna = *((char **) a);
+ char *fnb = *((char **) b);
+
+ return strcmp(fna + 8, fnb + 8);
+}
+
/*
* Parse the base backup options passed down by the parser
*/
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 6c274497072f0de8cc3874766d2047600c054faa..5c9314695fe0878d0568549beb6ec891d837a245 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -977,8 +977,6 @@ XLogRead(char *buf, XLogRecPtr startptr, Size count)
char *p;
XLogRecPtr recptr;
Size nbytes;
- uint32 lastRemovedLog;
- uint32 lastRemovedSeg;
uint32 log;
uint32 seg;
@@ -1073,19 +1071,8 @@ retry:
* read() succeeds in that case, but the data we tried to read might
* already have been overwritten with new WAL records.
*/
- XLogGetLastRemoved(&lastRemovedLog, &lastRemovedSeg);
XLByteToSeg(startptr, log, seg);
- if (log < lastRemovedLog ||
- (log == lastRemovedLog && seg <= lastRemovedSeg))
- {
- char filename[MAXFNAMELEN];
-
- XLogFileName(filename, ThisTimeLineID, log, seg);
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("requested WAL segment %s has already been removed",
- filename)));
- }
+ CheckXLogRemoved(log, seg, ThisTimeLineID);
/*
* During recovery, the currently-open WAL file might be replaced with the
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index ecd3f0f420db9b2341b42bf10a7a5e74d6903973..c21e43ae146bdc8a43d33dd83f162cc800a9614f 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -275,7 +275,7 @@ extern int XLogFileInit(uint32 log, uint32 seg,
extern int XLogFileOpen(uint32 log, uint32 seg);
-extern void XLogGetLastRemoved(uint32 *log, uint32 *seg);
+extern void CheckXLogRemoved(uint32 log, uint32 seg, TimeLineID tli);
extern void XLogSetAsyncXactLSN(XLogRecPtr record);
extern Buffer RestoreBackupBlock(XLogRecPtr lsn, XLogRecord *record,