From e97512345419c35a33c07bde76ab3394652857f2 Mon Sep 17 00:00:00 2001
From: Bruce Momjian <bruce@momjian.us>
Date: Wed, 24 Apr 2002 02:21:04 +0000
Subject: [PATCH] Speed improvement for large object restore.
Mario Weilguni
---
src/bin/pg_dump/pg_backup_archiver.c | 45 +++++++++++++++++++++++-----
src/bin/pg_dump/pg_backup_archiver.h | 6 +++-
src/bin/pg_dump/pg_backup_custom.c | 8 ++++-
src/bin/pg_dump/pg_backup_files.c | 8 ++++-
src/bin/pg_dump/pg_backup_null.c | 3 +-
src/bin/pg_dump/pg_backup_tar.c | 8 ++++-
6 files changed, 64 insertions(+), 14 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 2c8ea7d828f..8d2c2792bfe 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -15,7 +15,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.42 2002/02/11 00:18:20 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.43 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
*
@@ -819,6 +819,9 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
AH->createdBlobXref = 1;
}
+ /* Initialize the LO Buffer */
+ AH->lo_buf_used = 0;
+
/*
* Start long-running TXs if necessary
*/
@@ -848,6 +851,19 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
void
EndRestoreBlob(ArchiveHandle *AH, Oid oid)
{
+ if(AH->lo_buf_used > 0) {
+ /* Write remaining bytes from the LO buffer */
+ int res;
+ res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, AH->lo_buf_used);
+
+ ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = %d)\n",
+ (int)AH->lo_buf_used, res);
+ if (res != AH->lo_buf_used)
+ die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+ res, AH->lo_buf_used);
+ AH->lo_buf_used = 0;
+ }
+
lo_close(AH->connection, AH->loFd);
AH->writingBlob = 0;
@@ -1228,14 +1244,27 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
if (AH->writingBlob)
{
- res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
- ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
- (int) (size * nmemb), res);
- if (res != size * nmemb)
- die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
- res, (int) (size * nmemb));
+ if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
+ /* Split LO buffer */
+ int remaining = AH->lo_buf_size - AH->lo_buf_used;
+ int slack = nmemb * size - remaining;
- return res;
+ memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
+ res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
+ ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+ AH->lo_buf_size, res);
+ if (res != AH->lo_buf_size)
+ die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+ res, AH->lo_buf_size);
+ memcpy(AH->lo_buf, ptr + remaining, slack);
+ AH->lo_buf_used = slack;
+ } else {
+ /* LO Buffer is still large enough, buffer it */
+ memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
+ AH->lo_buf_used += size * nmemb;
+ }
+
+ return size * nmemb;
}
else if (AH->gzOut)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index cb7671ca69d..0fbce03f52c 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -17,7 +17,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.41 2001/11/05 17:46:30 momjian Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.42 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
* - Initial version.
@@ -41,6 +41,7 @@
#include <errno.h>
#include "pqexpbuffer.h"
+#define LOBBUFSIZE 32768
#ifdef HAVE_LIBZ
#include <zlib.h>
@@ -240,6 +241,9 @@ typedef struct _archiveHandle
RestoreOptions *ropt; /* Used to check restore options in
* ahwrite etc */
+ void *lo_buf;
+ int lo_buf_used;
+ int lo_buf_size;
} ArchiveHandle;
typedef struct _tocEntry
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 314e90d4df7..93e58378f4f 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -19,7 +19,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.17 2001/11/27 23:48:12 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.18 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
*
@@ -153,6 +153,12 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
if (ctx->zp == NULL)
die_horribly(AH, modulename, "out of memory\n");
+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");
+
/*
* zlibOutSize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to
diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c
index dfc1fdd2a1c..33e49b2d380 100644
--- a/src/bin/pg_dump/pg_backup_files.c
+++ b/src/bin/pg_dump/pg_backup_files.c
@@ -20,7 +20,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.14 2001/10/25 05:49:52 momjian Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.15 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
*
@@ -113,6 +113,12 @@ InitArchiveFmt_Files(ArchiveHandle *AH)
AH->formatData = (void *) ctx;
ctx->filePos = 0;
+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");
+
/*
* Now open the TOC file
*/
diff --git a/src/bin/pg_dump/pg_backup_null.c b/src/bin/pg_dump/pg_backup_null.c
index ee3b85eb4c8..af8f52a7c20 100644
--- a/src/bin/pg_dump/pg_backup_null.c
+++ b/src/bin/pg_dump/pg_backup_null.c
@@ -17,7 +17,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.7 2001/06/27 21:21:37 petere Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.8 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 09-Jul-2000 - pjw@rhyme.com.au
*
@@ -64,7 +64,6 @@ InitArchiveFmt_Null(ArchiveHandle *AH)
*/
if (AH->mode == archModeRead)
die_horribly(AH, NULL, "this format cannot be read\n");
-
}
/*
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 557ccbecbdc..efd8a030133 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -16,7 +16,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.20 2001/10/28 06:25:58 momjian Exp $
+ * $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.21 2002/04/24 02:21:04 momjian Exp $
*
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
*
@@ -157,6 +157,12 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
ctx = (lclContext *) malloc(sizeof(lclContext));
AH->formatData = (void *) ctx;
ctx->filePos = 0;
+
+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+ if(AH->lo_buf == NULL)
+ die_horribly(AH, modulename, "out of memory\n");
/*
* Now open the TOC file
--
GitLab