From ad06db2610f479d559ee4a3b9e3241d40be07440 Mon Sep 17 00:00:00 2001
From: Bruce Momjian <bruce@momjian.us>
Date: Wed, 20 Oct 2010 00:33:19 +0000
Subject: [PATCH] In pg_upgrade, use cached copy of directory listing, rather
than calling scandir() with a pattern for every table.
Optimization after report of pg_upgrade slowness with 150k tables.
---
contrib/pg_upgrade/info.c | 15 +++---
contrib/pg_upgrade/pg_upgrade.h | 25 ++++-----
contrib/pg_upgrade/relfilenode.c | 88 ++++++++++++++++++++------------
3 files changed, 74 insertions(+), 54 deletions(-)
diff --git a/contrib/pg_upgrade/info.c b/contrib/pg_upgrade/info.c
index b4e414ec8d9..a8d0bddc191 100644
--- a/contrib/pg_upgrade/info.c
+++ b/contrib/pg_upgrade/info.c
@@ -30,8 +30,8 @@ static void map_rel_by_id(Oid oldid, Oid newid,
const char *old_tablespace, const DbInfo *old_db,
const DbInfo *new_db, const char *olddata,
const char *newdata, FileNameMap *map);
-static RelInfo *relarr_lookup_reloid(
- RelInfoArr *rel_arr, Oid oid, Cluster whichCluster);
+static RelInfo *relarr_lookup_reloid(RelInfoArr *rel_arr,
+ Oid oid, Cluster whichCluster);
static RelInfo *relarr_lookup_rel(RelInfoArr *rel_arr,
const char *nspname, const char *relname,
Cluster whichCluster);
@@ -172,8 +172,8 @@ map_rel_by_id(Oid oldid, Oid newid,
* relation belongs to the default tablespace, hence relfiles would
* exist in the data directories.
*/
- snprintf(map->old_file, sizeof(map->old_file), "%s/base/%u", olddata, old_db->db_oid);
- snprintf(map->new_file, sizeof(map->new_file), "%s/base/%u", newdata, new_db->db_oid);
+ snprintf(map->old_dir, sizeof(map->old_dir), "%s/base/%u", olddata, old_db->db_oid);
+ snprintf(map->new_dir, sizeof(map->new_dir), "%s/base/%u", newdata, new_db->db_oid);
}
else
{
@@ -181,9 +181,9 @@ map_rel_by_id(Oid oldid, Oid newid,
* relation belongs to some tablespace, hence copy its physical
* location
*/
- snprintf(map->old_file, sizeof(map->old_file), "%s%s/%u", old_tablespace,
+ snprintf(map->old_dir, sizeof(map->old_dir), "%s%s/%u", old_tablespace,
old_cluster.tablespace_suffix, old_db->db_oid);
- snprintf(map->new_file, sizeof(map->new_file), "%s%s/%u", old_tablespace,
+ snprintf(map->new_dir, sizeof(map->new_dir), "%s%s/%u", old_tablespace,
new_cluster.tablespace_suffix, new_db->db_oid);
}
}
@@ -318,6 +318,7 @@ get_rel_infos(const DbInfo *dbinfo,
* pg_largeobject_loid_pn_index's relfilenode can change if the table was
* reindexed so we get the relfilenode for each database and migrate it as
* a normal user table.
+ * Order by tablespace so we can cache the directory contents efficiently.
*/
snprintf(query, sizeof(query),
@@ -338,7 +339,7 @@ get_rel_infos(const DbInfo *dbinfo,
"GROUP BY c.oid, n.nspname, c.relname, c.relfilenode,"
" c.reltoastrelid, t.spclocation, "
" n.nspname "
- "ORDER BY n.nspname, c.relname;",
+ "ORDER BY t.spclocation, n.nspname, c.relname;",
FirstNormalObjectId,
/* see the comment at the top of old_8_3_create_sequence_script() */
(GET_MAJOR_VERSION(old_cluster.major_version) <= 803) ?
diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
index 3756f5cbb0a..2aa22081582 100644
--- a/contrib/pg_upgrade/pg_upgrade.h
+++ b/contrib/pg_upgrade/pg_upgrade.h
@@ -85,10 +85,10 @@ typedef struct
*/
typedef struct
{
+ char old_dir[MAXPGPATH];
+ char new_dir[MAXPGPATH];
Oid old_relfilenode; /* Relfilenode of the old relation */
Oid new_relfilenode; /* Relfilenode of the new relation */
- char old_file[MAXPGPATH];
- char new_file[MAXPGPATH];
char old_nspname[NAMEDATALEN]; /* old name of the namespace */
char old_relname[NAMEDATALEN]; /* old name of the relation */
char new_nspname[NAMEDATALEN]; /* new name of the namespace */
@@ -255,10 +255,8 @@ void check_old_cluster(bool live_check,
char **sequence_script_file_name);
void check_new_cluster(void);
void report_clusters_compatible(void);
-void issue_warnings(
- char *sequence_script_file_name);
-void output_completion_banner(
- char *deletion_script_file_name);
+void issue_warnings(char *sequence_script_file_name);
+void output_completion_banner(char *deletion_script_file_name);
void check_cluster_versions(void);
void check_cluster_compatibility(bool live_check);
void create_script_for_old_cluster_deletion(char **deletion_script_file_name);
@@ -319,14 +317,12 @@ typedef void *pageCnvCtx;
#endif
int dir_matching_filenames(const struct dirent * scan_ent);
-int pg_scandir(const char *dirname,
- struct dirent *** namelist,
- int (*selector) (const struct dirent *));
-const char *copyAndUpdateFile(
- pageCnvCtx *pageConverter, const char *src,
+int pg_scandir(const char *dirname, struct dirent *** namelist,
+ int (*selector) (const struct dirent *));
+const char *copyAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
const char *dst, bool force);
-const char *linkAndUpdateFile(
- pageCnvCtx *pageConverter, const char *src, const char *dst);
+const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
+ const char *dst);
void check_hard_link(void);
@@ -374,8 +370,7 @@ PGresult *executeQueryOrDie(PGconn *conn,
void start_postmaster(Cluster whichCluster, bool quiet);
void stop_postmaster(bool fast, bool quiet);
-uint32 get_major_server_version(char **verstr,
- Cluster whichCluster);
+uint32 get_major_server_version(char **verstr, Cluster whichCluster);
void check_for_libpq_envvars(void);
diff --git a/contrib/pg_upgrade/relfilenode.c b/contrib/pg_upgrade/relfilenode.c
index ff1a73cdec2..7b73b5e91f5 100644
--- a/contrib/pg_upgrade/relfilenode.c
+++ b/contrib/pg_upgrade/relfilenode.c
@@ -110,20 +110,38 @@ static void
transfer_single_new_db(pageCnvCtx *pageConverter,
FileNameMap *maps, int size)
{
+ char old_dir[MAXPGPATH];
+ struct dirent **namelist = NULL;
+ int numFiles = 0;
int mapnum;
+ int fileno;
+
+ old_dir[0] = '\0';
for (mapnum = 0; mapnum < size; mapnum++)
{
char old_file[MAXPGPATH];
char new_file[MAXPGPATH];
- struct dirent **namelist = NULL;
- int numFiles;
+
+ /* Changed tablespaces? Need a new directory scan? */
+ if (strcmp(maps[mapnum].old_dir, old_dir) != 0)
+ {
+ if (numFiles > 0)
+ {
+ for (fileno = 0; fileno < numFiles; fileno++)
+ pg_free(namelist[fileno]);
+ pg_free(namelist);
+ }
+
+ snprintf(old_dir, sizeof(old_dir), "%s", maps[mapnum].old_dir);
+ numFiles = pg_scandir(old_dir, &namelist, NULL);
+ }
/* Copying files might take some time, so give feedback. */
- snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_file,
+ snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_dir,
maps[mapnum].old_relfilenode);
- snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_file,
+ snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_dir,
maps[mapnum].new_relfilenode);
pg_log(PG_REPORT, OVERWRITE_MESSAGE, old_file);
@@ -139,28 +157,27 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804)
{
/*
- * Now copy/link any fsm and vm files, if they exist
+ * Copy/link any fsm and vm files, if they exist
*/
snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u_",
maps[mapnum].old_relfilenode);
- numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
- while (numFiles--)
+ for (fileno = 0; fileno < numFiles; fileno++)
{
- snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
- namelist[numFiles]->d_name);
- snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
- maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '_'));
-
- unlink(new_file);
- transfer_relfile(pageConverter, old_file, new_file,
- maps[mapnum].old_nspname, maps[mapnum].old_relname,
- maps[mapnum].new_nspname, maps[mapnum].new_relname);
-
- pg_free(namelist[numFiles]);
+ if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+ strlen(scandir_file_pattern)) == 0)
+ {
+ snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+ namelist[fileno]->d_name);
+ snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+ maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '_'));
+
+ unlink(new_file);
+ transfer_relfile(pageConverter, old_file, new_file,
+ maps[mapnum].old_nspname, maps[mapnum].old_relname,
+ maps[mapnum].new_nspname, maps[mapnum].new_relname);
+ }
}
-
- pg_free(namelist);
}
/*
@@ -172,23 +189,30 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
*/
snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u.",
maps[mapnum].old_relfilenode);
- numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
- while (numFiles--)
+ for (fileno = 0; fileno < numFiles; fileno++)
{
- snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
- namelist[numFiles]->d_name);
- snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
- maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '.'));
-
- unlink(new_file);
- transfer_relfile(pageConverter, old_file, new_file,
- maps[mapnum].old_nspname, maps[mapnum].old_relname,
- maps[mapnum].new_nspname, maps[mapnum].new_relname);
+ if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
+ strlen(scandir_file_pattern)) == 0)
+ {
+ snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
+ namelist[fileno]->d_name);
+ snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
+ maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '.'));
- pg_free(namelist[numFiles]);
+ unlink(new_file);
+ transfer_relfile(pageConverter, old_file, new_file,
+ maps[mapnum].old_nspname, maps[mapnum].old_relname,
+ maps[mapnum].new_nspname, maps[mapnum].new_relname);
+ }
}
+ }
+
+ if (numFiles > 0)
+ {
+ for (fileno = 0; fileno < numFiles; fileno++)
+ pg_free(namelist[fileno]);
pg_free(namelist);
}
}
--
GitLab