diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c
index 000dac5662a2ca70df93b2aca86b6e137567c347..f2a7697178e150395542ddc23b723dc4f5d6db7f 100644
--- a/contrib/pg_standby/pg_standby.c
+++ b/contrib/pg_standby/pg_standby.c
@@ -1,5 +1,5 @@
/*
- * $PostgreSQL: pgsql/contrib/pg_standby/pg_standby.c,v 1.21 2009/03/26 22:29:13 tgl Exp $
+ * $PostgreSQL: pgsql/contrib/pg_standby/pg_standby.c,v 1.22 2009/05/14 20:31:09 heikki Exp $
*
*
* pg_standby.c
@@ -26,6 +26,7 @@
#include <ctype.h>
#include <dirent.h>
#include <sys/stat.h>
+#include <fcntl.h>
#include <signal.h>
#ifdef WIN32
@@ -52,7 +53,6 @@ int maxwaittime = 0; /* how long are we prepared to wait for? */
int keepfiles = 0; /* number of WAL files to keep, 0 keep all */
int maxretries = 3; /* number of retries on restore command */
bool debug = false; /* are we debugging? */
-bool triggered = false; /* have we been triggered? */
bool need_cleanup = false; /* do we need to remove files from
* archive? */
@@ -69,6 +69,30 @@ char restoreCommand[MAXPGPATH]; /* run this to restore */
char exclusiveCleanupFileName[MAXPGPATH]; /* the file we need to
* get from archive */
+/*
+ * Two types of failover are supported (smart and fast failover).
+ *
+ * The content of the trigger file determines the type of failover. If the
+ * trigger file contains the word "smart" (or the file is empty), smart
+ * failover is chosen: pg_standby acts as cp or ln command itself, on
+ * successful completion all the available WAL records will be applied
+ * resulting in zero data loss. But, it might take a long time to finish
+ * recovery if there's a lot of unapplied WAL.
+ *
+ * On the other hand, if the trigger file contains the word "fast", the
+ * recovery is finished immediately even if unapplied WAL files remain. Any
+ * transactions in the unapplied WAL files are lost.
+ *
+ * An empty trigger file performs smart failover. SIGUSR or SIGINT triggers
+ * fast failover. A timeout causes fast failover (smart failover would have
+ * the same effect, since if the timeout is reached there is no unapplied WAL).
+ */
+#define NoFailover 0
+#define SmartFailover 1
+#define FastFailover 2
+
+static int Failover = NoFailover;
+
#define RESTORE_COMMAND_COPY 0
#define RESTORE_COMMAND_LINK 1
int restoreCommandType;
@@ -108,7 +132,6 @@ struct stat stat_buf;
*
* As an example, and probably the common case, we use either
* cp/ln commands on *nix, or copy/move command on Windows.
- *
*/
static void
CustomizableInitialize(void)
@@ -352,12 +375,16 @@ SetWALFileNameForCleanup(void)
/*
* CheckForExternalTrigger()
*
- * Is there a trigger file?
+ * Is there a trigger file? Sets global 'Failover' variable to indicate
+ * what kind of a trigger file it was. A "fast" trigger file is turned
+ * into a "smart" file as a side-effect.
*/
-static bool
+static void
CheckForExternalTrigger(void)
{
- int rc;
+ char buf[32];
+ int fd;
+ int len;
/*
* Look for a trigger file, if that option has been selected
@@ -365,28 +392,79 @@ CheckForExternalTrigger(void)
* We use stat() here because triggerPath is always a file rather than
* potentially being in an archive
*/
- if (triggerPath && stat(triggerPath, &stat_buf) == 0)
+ if (!triggerPath || stat(triggerPath, &stat_buf) != 0)
+ return;
+
+ /*
+ * An empty trigger file performs smart failover. There's a little race
+ * condition here: if the writer of the trigger file has just created
+ * the file, but not yet written anything to it, we'll treat that as
+ * smart shutdown even if the other process was just about to write "fast"
+ * to it. But that's fine: we'll restore one more WAL file, and when we're
+ * invoked next time, we'll see the word "fast" and fail over immediately.
+ */
+ if (stat_buf.st_size == 0)
{
- fprintf(stderr, "trigger file found\n");
+ Failover = SmartFailover;
+ fprintf(stderr, "trigger file found: smart failover\n");
+ fflush(stderr);
+ return;
+ }
+
+ if ((fd = open(triggerPath, O_RDWR, 0)) < 0)
+ {
+ fprintf(stderr, "WARNING: could not open \"%s\": %s\n",
+ triggerPath, strerror(errno));
+ fflush(stderr);
+ return;
+ }
+
+ if ((len = read(fd, buf, sizeof(buf))) < 0)
+ {
+ fprintf(stderr, "WARNING: could not read \"%s\": %s\n",
+ triggerPath, strerror(errno));
+ fflush(stderr);
+ close(fd);
+ return;
+ }
+ buf[len] = '\0';
+
+ if (strncmp(buf, "smart", 5) == 0)
+ {
+ Failover = SmartFailover;
+ fprintf(stderr, "trigger file found: smart failover\n");
+ fflush(stderr);
+ close(fd);
+ return;
+ }
+
+ if (strncmp(buf, "fast", 4) == 0)
+ {
+ Failover = FastFailover;
+
+ fprintf(stderr, "trigger file found: fast failover\n");
fflush(stderr);
/*
- * If trigger file found, we *must* delete it. Here's why: When
- * recovery completes, we will be asked again for the same file from
- * the archive using pg_standby so must remove trigger file so we can
- * reload file again and come up correctly.
+ * Turn it into a "smart" trigger by truncating the file. Otherwise
+ * if the server asks us again to restore a segment that was restored
+ * restored already, we would return "not found" and upset the server.
*/
- rc = unlink(triggerPath);
- if (rc != 0)
+ if (ftruncate(fd, 0) < 0)
{
- fprintf(stderr, "\n ERROR: could not remove \"%s\": %s", triggerPath, strerror(errno));
+ fprintf(stderr, "WARNING: could not read \"%s\": %s\n",
+ triggerPath, strerror(errno));
fflush(stderr);
- exit(rc);
}
- return true;
- }
+ close(fd);
- return false;
+ return;
+ }
+ close(fd);
+
+ fprintf(stderr, "WARNING: invalid content in \"%s\"\n", triggerPath);
+ fflush(stderr);
+ return;
}
/*
@@ -402,7 +480,7 @@ RestoreWALFileForRecovery(void)
if (debug)
{
- fprintf(stderr, "\nrunning restore :");
+ fprintf(stderr, "running restore :");
fflush(stderr);
}
@@ -413,7 +491,7 @@ RestoreWALFileForRecovery(void)
{
if (debug)
{
- fprintf(stderr, " OK");
+ fprintf(stderr, " OK\n");
fflush(stderr);
}
return true;
@@ -425,7 +503,7 @@ RestoreWALFileForRecovery(void)
* Allow caller to add additional info
*/
if (debug)
- fprintf(stderr, "not restored : ");
+ fprintf(stderr, "not restored\n");
return false;
}
@@ -552,8 +630,6 @@ main(int argc, char **argv)
break;
case 't': /* Trigger file */
triggerPath = optarg;
- if (CheckForExternalTrigger())
- exit(1); /* Normal exit, with non-zero */
break;
case 'w': /* Max wait time */
maxwaittime = atoi(optarg);
@@ -633,20 +709,20 @@ main(int argc, char **argv)
if (debug)
{
- fprintf(stderr, "\nTrigger file : %s", triggerPath ? triggerPath : "<not set>");
- fprintf(stderr, "\nWaiting for WAL file : %s", nextWALFileName);
- fprintf(stderr, "\nWAL file path : %s", WALFilePath);
- fprintf(stderr, "\nRestoring to... : %s", xlogFilePath);
- fprintf(stderr, "\nSleep interval : %d second%s",
+ fprintf(stderr, "Trigger file : %s\n", triggerPath ? triggerPath : "<not set>");
+ fprintf(stderr, "Waiting for WAL file : %s\n", nextWALFileName);
+ fprintf(stderr, "WAL file path : %s\n", WALFilePath);
+ fprintf(stderr, "Restoring to : %s\n", xlogFilePath);
+ fprintf(stderr, "Sleep interval : %d second%s\n",
sleeptime, (sleeptime > 1 ? "s" : " "));
- fprintf(stderr, "\nMax wait interval : %d %s",
+ fprintf(stderr, "Max wait interval : %d %s\n",
maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
- fprintf(stderr, "\nCommand for restore : %s", restoreCommand);
- fprintf(stderr, "\nKeep archive history : ");
+ fprintf(stderr, "Command for restore : %s\n", restoreCommand);
+ fprintf(stderr, "Keep archive history : ");
if (need_cleanup)
- fprintf(stderr, "%s and later", exclusiveCleanupFileName);
+ fprintf(stderr, "%s and later\n", exclusiveCleanupFileName);
else
- fprintf(stderr, "No cleanup required");
+ fprintf(stderr, "No cleanup required\n");
fflush(stderr);
}
@@ -676,56 +752,74 @@ main(int argc, char **argv)
/*
* Main wait loop
*/
- while (!CustomizableNextWALFileReady() && !triggered)
+ for (;;)
{
- if (sleeptime <= 60)
- pg_usleep(sleeptime * 1000000L);
-
+ /* Check for trigger file or signal first */
+ CheckForExternalTrigger();
if (signaled)
{
- triggered = true;
+ Failover = FastFailover;
if (debug)
{
- fprintf(stderr, "\nsignaled to exit\n");
+ fprintf(stderr, "signaled to exit: fast failover\n");
fflush(stderr);
}
}
- else
+
+ /*
+ * Check for fast failover immediately, before checking if the
+ * requested WAL file is available
+ */
+ if (Failover == FastFailover)
+ exit(1);
+
+ if (CustomizableNextWALFileReady())
{
+ /*
+ * Once we have restored this file successfully we can remove some
+ * prior WAL files. If this restore fails we musn't remove any file
+ * because some of them will be requested again immediately after
+ * the failed restore, or when we restart recovery.
+ */
+ if (RestoreWALFileForRecovery())
+ {
+ if (need_cleanup)
+ CustomizableCleanupPriorWALFiles();
- if (debug)
+ exit(0);
+ }
+ else
{
- fprintf(stderr, "\nWAL file not present yet.");
- if (triggerPath)
- fprintf(stderr, " Checking for trigger file...");
- fflush(stderr);
+ /* Something went wrong in copying the file */
+ exit(1);
}
+ }
+
+ /* Check for smart failover if the next WAL file was not available */
+ if (Failover == SmartFailover)
+ exit(1);
- waittime += sleeptime;
+ if (sleeptime <= 60)
+ pg_usleep(sleeptime * 1000000L);
- if (!triggered && (CheckForExternalTrigger() || (waittime >= maxwaittime && maxwaittime > 0)))
+ waittime += sleeptime;
+ if (waittime >= maxwaittime && maxwaittime > 0)
+ {
+ Failover = FastFailover;
+ if (debug)
{
- triggered = true;
- if (debug && waittime >= maxwaittime && maxwaittime > 0)
- fprintf(stderr, "\nTimed out after %d seconds\n", waittime);
+ fprintf(stderr, "Timed out after %d seconds: fast failover\n",
+ waittime);
+ fflush(stderr);
}
}
+ if (debug)
+ {
+ fprintf(stderr, "WAL file not present yet.");
+ if (triggerPath)
+ fprintf(stderr, " Checking for trigger file...");
+ fprintf(stderr, "\n");
+ fflush(stderr);
+ }
}
-
- /*
- * Action on exit
- */
- if (triggered)
- exit(1); /* Normal exit, with non-zero */
-
- /*
- * Once we have restored this file successfully we can remove some prior
- * WAL files. If this restore fails we musn't remove any file because some
- * of them will be requested again immediately after the failed restore,
- * or when we restart recovery.
- */
- if (RestoreWALFileForRecovery() && need_cleanup)
- CustomizableCleanupPriorWALFiles();
-
- return 0;
}
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 016050664fb1bedd29c863f998c600a04db46a77..b018e2fdb4856afef836463cda65a37122ce7cdb 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.125 2009/04/27 16:27:35 momjian Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.126 2009/05/14 20:31:09 heikki Exp $ -->
<chapter id="backup">
<title>Backup and Restore</title>
@@ -1126,6 +1126,29 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</listitem>
</varlistentry>
+ <varlistentry id="recovery-end-command" xreflabel="recovery_end_command">
+ <term><varname>recovery_end_command</varname> (<type>string</type>)</term>
+ <listitem>
+ <para>
+ This parameter specifies a shell command that will be executed once only
+ at the end of recovery. This parameter is optional. The purpose of the
+ recovery_end_command is to provide a mechanism for cleanup following
+ replication or recovery.
+ Any <literal>%r</> is replaced by the name of the file
+ containing the last valid restart point. That is the earliest file that
+ must be kept to allow a restore to be restartable, so this information
+ can be used to truncate the archive to just the minimum required to
+ support restart of the current restore. <literal>%r</> would only be
+ used in a warm-standby configuration (see <xref linkend="warm-standby">).
+ Write <literal>%%</> to embed an actual <literal>%</> character
+ in the command.
+ If the command returns a non-zero exit status then a WARNING log
+ message will be written, unless signalled in which case we return
+ a FATAL error.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="recovery-target-time" xreflabel="recovery_target_time">
<term><varname>recovery_target_time</varname>
(<type>timestamp</type>)
diff --git a/doc/src/sgml/pgstandby.sgml b/doc/src/sgml/pgstandby.sgml
index 6b381c69ac8217a61b8ae8f478299e52a3d5cf91..81e53b6a63fcf38687491f5fab2e951bab707153 100644
--- a/doc/src/sgml/pgstandby.sgml
+++ b/doc/src/sgml/pgstandby.sgml
@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/pgstandby.sgml,v 2.7 2009/02/27 09:30:21 petere Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/pgstandby.sgml,v 2.8 2009/05/14 20:31:09 heikki Exp $ -->
<sect1 id="pgstandby">
<title>pg_standby</title>
@@ -92,6 +92,37 @@ pg_standby <optional> <replaceable>option</> ... </optional> <replaceable>archiv
is specified,
the <replaceable>archivelocation</> directory must be writable too.
</para>
+ <para>
+ There are two ways to fail over a <quote>warm standby</> database server.
+ You control the type of failover with the contents of the trigger file:
+
+ <variablelist>
+ <varlistentry>
+ <term>Smart Failover</term>
+ <listitem>
+ <para>
+ In smart failover, the server is brought up after applying all
+ WAL files available in the archive. This results in zero data loss,
+ even if the standby server has fallen behind, but if there is a lot
+ unapplied WAL the recovery can take a long time. To trigger a smart
+ failover, create a trigger file containing the word <literal>smart</>,
+ or just leave it empty.
+ </para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
+ <term>Fast Failover</term>
+ <listitem>
+ <para>
+ In fast failover, the server is brought up immediately. Any WAL files
+ in the archive that have not yet been applied will be ignored, and
+ all transactions in those files are lost. To trigger a fast failover,
+ write the word <literal>fast</> into the trigger file.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+ </para>
<table>
<title><application>pg_standby</> options</title>
@@ -177,8 +208,7 @@ pg_standby <optional> <replaceable>option</> ... </optional> <replaceable>archiv
<entry><literal>-t</> <replaceable>triggerfile</></entry>
<entry>none</entry>
<entry>
- Specify a trigger file whose presence should cause recovery to end
- whether or not the next WAL file is available.
+ Specify a trigger file whose presence should perform failover.
It is recommended that you use a structured filename to
avoid confusion as to which server is being triggered
when multiple servers exist on the same system; for example
@@ -190,7 +220,7 @@ pg_standby <optional> <replaceable>option</> ... </optional> <replaceable>archiv
<entry>0</entry>
<entry>
Set the maximum number of seconds to wait for the next WAL file,
- after which recovery will end and the standby will come up.
+ after which a fast failover will be performed.
A setting of zero (the default) means wait forever.
The default setting is not necessarily recommended;
consult <xref linkend="warm-standby"> for discussion.
@@ -210,6 +240,7 @@ pg_standby <optional> <replaceable>option</> ... </optional> <replaceable>archiv
archive_command = 'cp %p .../archive/%f'
restore_command = 'pg_standby -l -d -s 2 -t /tmp/pgsql.trigger.5442 .../archive %f %p %r 2>>standby.log'
+recovery_end_command = 'rm -f /tmp/pgsql.trigger.5442'
</programlisting>
<para>
where the archive directory is physically located on the standby server,
@@ -236,7 +267,13 @@ restore_command = 'pg_standby -l -d -s 2 -t /tmp/pgsql.trigger.5442 .../archive
<listitem>
<para>
stop waiting only when a trigger file called
- <filename>/tmp/pgsql.trigger.5442</> appears
+ <filename>/tmp/pgsql.trigger.5442</> appears,
+ and perform failover according to its content
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ remove the trigger file when recovery ends
</para>
</listitem>
<listitem>
@@ -277,7 +314,8 @@ restore_command = 'pg_standby -d -s 5 -t C:\pgsql.trigger.5442 ...\archive %f %p
<listitem>
<para>
stop waiting only when a trigger file called
- <filename>C:\pgsql.trigger.5442</> appears
+ <filename>C:\pgsql.trigger.5442</> appears,
+ and perform failover according to its content
</para>
</listitem>
<listitem>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9b3fe5eafa7560d9b3bca2877db297358f1dda90..09b507500f576db6b1bc33674ab17eaaed90321a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.337 2009/05/07 11:25:25 heikki Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.338 2009/05/14 20:31:09 heikki Exp $
*
*-------------------------------------------------------------------------
*/
@@ -147,6 +147,7 @@ static bool restoredFromArchive = false;
/* options taken from recovery.conf */
static char *recoveryRestoreCommand = NULL;
+static char *recoveryEndCommand = NULL;
static bool recoveryTarget = false;
static bool recoveryTargetExact = false;
static bool recoveryTargetInclusive = true;
@@ -463,6 +464,7 @@ static int XLogFileRead(uint32 log, uint32 seg, int emode);
static void XLogFileClose(void);
static bool RestoreArchivedFile(char *path, const char *xlogfname,
const char *recovername, off_t expectedSize);
+static void ExecuteRecoveryEndCommand(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
static void RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr);
static void ValidateXLOGDirectoryStructure(void);
@@ -2849,6 +2851,114 @@ RestoreArchivedFile(char *path, const char *xlogfname,
return false;
}
+/*
+ * Attempt to execute the recovery_end_command.
+ */
+static void
+ExecuteRecoveryEndCommand(void)
+{
+ char xlogRecoveryEndCmd[MAXPGPATH];
+ char lastRestartPointFname[MAXPGPATH];
+ char *dp;
+ char *endp;
+ const char *sp;
+ int rc;
+ bool signaled;
+ uint32 restartLog;
+ uint32 restartSeg;
+
+ Assert(recoveryEndCommand);
+
+ /*
+ * Calculate the archive file cutoff point for use during log shipping
+ * replication. All files earlier than this point can be deleted
+ * from the archive, though there is no requirement to do so.
+ *
+ * We initialise this with the filename of an InvalidXLogRecPtr, which
+ * will prevent the deletion of any WAL files from the archive
+ * because of the alphabetic sorting property of WAL filenames.
+ *
+ * Once we have successfully located the redo pointer of the checkpoint
+ * from which we start recovery we never request a file prior to the redo
+ * pointer of the last restartpoint. When redo begins we know that we
+ * have successfully located it, so there is no need for additional
+ * status flags to signify the point when we can begin deleting WAL files
+ * from the archive.
+ */
+ if (InRedo)
+ {
+ XLByteToSeg(ControlFile->checkPointCopy.redo,
+ restartLog, restartSeg);
+ XLogFileName(lastRestartPointFname,
+ ControlFile->checkPointCopy.ThisTimeLineID,
+ restartLog, restartSeg);
+ }
+ else
+ XLogFileName(lastRestartPointFname, 0, 0, 0);
+
+ /*
+ * construct the command to be executed
+ */
+ dp = xlogRecoveryEndCmd;
+ endp = xlogRecoveryEndCmd + MAXPGPATH - 1;
+ *endp = '\0';
+
+ for (sp = recoveryEndCommand; *sp; sp++)
+ {
+ if (*sp == '%')
+ {
+ switch (sp[1])
+ {
+ case 'r':
+ /* %r: filename of last restartpoint */
+ sp++;
+ StrNCpy(dp, lastRestartPointFname, endp - dp);
+ dp += strlen(dp);
+ break;
+ case '%':
+ /* convert %% to a single % */
+ sp++;
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ default:
+ /* otherwise treat the % as not special */
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ }
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *sp;
+ }
+ }
+ *dp = '\0';
+
+ ereport(DEBUG3,
+ (errmsg_internal("executing recovery end command \"%s\"",
+ xlogRecoveryEndCmd)));
+
+ /*
+ * Copy xlog from archival storage to XLOGDIR
+ */
+ rc = system(xlogRecoveryEndCmd);
+ if (rc != 0)
+ {
+ /*
+ * If the failure was due to any sort of signal, it's best to punt and
+ * abort recovery. See also detailed comments on signals in
+ * RestoreArchivedFile().
+ */
+ signaled = WIFSIGNALED(rc) || WEXITSTATUS(rc) > 125;
+
+ ereport(signaled ? FATAL : WARNING,
+ (errmsg("recovery_end_command \"%s\": return code %d",
+ xlogRecoveryEndCmd, rc)));
+ }
+}
+
/*
* Preallocate log files beyond the specified log endpoint.
*
@@ -4664,6 +4774,13 @@ readRecoveryCommandFile(void)
(errmsg("restore_command = '%s'",
recoveryRestoreCommand)));
}
+ else if (strcmp(tok1, "recovery_end_command") == 0)
+ {
+ recoveryEndCommand = pstrdup(tok2);
+ ereport(LOG,
+ (errmsg("recovery_end_command = '%s'",
+ recoveryEndCommand)));
+ }
else if (strcmp(tok1, "recovery_target_timeline") == 0)
{
rtliGiven = true;
@@ -5622,6 +5739,9 @@ StartupXLOG(void)
* allows some extra error checking in xlog_redo.
*/
CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE);
+
+ if (recoveryEndCommand)
+ ExecuteRecoveryEndCommand();
}
/*