diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 466fca8d05a080143e88fbe4e2dce2a1011c77b7..e47edcd331ed1633fa087644e41add8902d50e98 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1,4 +1,4 @@ -<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.59 2006/10/07 20:59:03 petere Exp $ --> +<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.60 2007/01/25 02:17:25 momjian Exp $ --> <chapter id="performance-tips"> <title>Performance Tips</title> @@ -800,7 +800,9 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; prepared <command>INSERT</command> statement, and then use <command>EXECUTE</command> as many times as required. This avoids some of the overhead of repeatedly parsing and planning - <command>INSERT</command>. + <command>INSERT</command>. Different interfaces provide this facility + in different ways; look for Prepared Statements in the interface + documentation. </para> <para> @@ -809,6 +811,20 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; <command>INSERT</command>, even if <command>PREPARE</> is used and multiple insertions are batched into a single transaction. </para> + + <para> + <command>COPY</command> is fastest when used within the same + transaction as an earlier <command>CREATE TABLE</command> or + <command>TRUNCATE</command> command. In those cases, no WAL + needs to be written because in case of an error, the files + containing the newly loaded data will be removed automatically. + <command>CREATE TABLE AS SELECT</command> is also optimized + to avoid writing WAL. <command>COPY</command> and + <command>CREATE TABLE AS SELECT</command> will write WAL + when <xref linkend="guc-archive-command"> is set and will not + therefore be optimized in that case. + </para> + </sect2> <sect2 id="populate-rm-indexes"> @@ -877,6 +893,29 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; </para> </sect2> + <sect2 id="populate-pitr"> + <title>Turn off <varname>archive_command</varname></title> + + <para> + When loading large amounts of data you may want to unset the + <xref linkend="guc-archive-command"> before loading. It may be + faster to take a new base backup once the load has completed + than to allow a large archive to accumulate. + </para> + + <para> + This is particularly important advice because certain commands + will perform more slowly when <varname>archive_command</varname> + is set, as a result of their needing to write large amounts of WAL. + This applies to the following commands: + <command>CREATE TABLE AS SELECT</command>, + <command>CREATE INDEX</command> and also <command>COPY</command>, when + it is executed in the same transaction as a prior + <command>CREATE TABLE</command> or <command>TRUNCATE</command> command. + </para> + + </sect2> + <sect2 id="populate-analyze"> <title>Run <command>ANALYZE</command> Afterwards</title> @@ -914,8 +953,12 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; the first several guidelines are handled automatically. What is left for you to do is to set appropriate (i.e., larger than normal) values for <varname>maintenance_work_mem</varname> and - <varname>checkpoint_segments</varname> before loading the dump script, - and then to run <command>ANALYZE</> afterwards. + <varname>checkpoint_segments</varname>, as well as unsetting + <varname>archive_command</varname> before loading the dump script, + and then to run <command>ANALYZE</> afterwards and resetting + <varname>archive_command</varname> if required. All of the + parameters can be reset once the load has completed without needing + to restart the server, as described in <xref linkend="config-setting">. </para> <para> diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 3913391f6f1854710c36e4a656bd30af264ccf3b..c0f5f2074b1980b85caf10aa63bee7a9c0e7e84b 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.224 2007/01/09 22:00:59 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.225 2007/01/25 02:17:25 momjian Exp $ * * * INTERFACE ROUTINES @@ -28,6 +28,7 @@ * heap_update - replace a tuple in a relation with another tuple * heap_markpos - mark scan position * heap_restrpos - restore position to marked location + * heap_sync - sync heap, for when no WAL has been written * * NOTES * This file contains the heap_ routines which implement @@ -50,6 +51,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "storage/procarray.h" +#include "storage/smgr.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/relcache.h" @@ -1358,7 +1360,7 @@ heap_get_latest_tid(Relation relation, * that all new tuples go into new pages not containing any tuples from other * transactions, that the relation gets fsync'd before commit, and that the * transaction emits at least one WAL record to ensure RecordTransactionCommit - * will decide to WAL-log the commit. + * will decide to WAL-log the commit. (see heap_sync() comments also) * * use_fsm is passed directly to RelationGetBufferForTuple, which see for * more info. @@ -1418,7 +1420,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ if (HeapTupleHasExternal(tup) || (MAXALIGN(tup->t_len) > TOAST_TUPLE_THRESHOLD)) - heaptup = toast_insert_or_update(relation, tup, NULL); + heaptup = toast_insert_or_update(relation, tup, NULL, use_wal); else heaptup = tup; @@ -1535,6 +1537,18 @@ simple_heap_insert(Relation relation, HeapTuple tup) return heap_insert(relation, tup, GetCurrentCommandId(), true, true); } +/* + * fast_heap_insert - insert a tuple with options to improve speed + * + * Currently, this routine allows specifying additional options for speed + * in certain cases, such as WAL-avoiding COPY command + */ +Oid +fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal) +{ + return heap_insert(relation, tup, GetCurrentCommandId(), use_wal, use_wal); +} + /* * heap_delete - delete a tuple * @@ -2086,11 +2100,11 @@ l2: * * Note: below this point, heaptup is the data we actually intend to * store into the relation; newtup is the caller's original untoasted - * data. + * data. (We always use WAL for toast table updates.) */ if (need_toast) { - heaptup = toast_insert_or_update(relation, newtup, &oldtup); + heaptup = toast_insert_or_update(relation, newtup, &oldtup, true); newtupsize = MAXALIGN(heaptup->t_len); } else @@ -3966,3 +3980,24 @@ heap2_desc(StringInfo buf, uint8 xl_info, char *rec) else appendStringInfo(buf, "UNKNOWN"); } + +/* ---------------- + * heap_sync - sync a heap, for use when no WAL has been written + * + * ---------------- + */ +void +heap_sync(Relation rel) +{ + if (!rel->rd_istemp) + { + /* + * If we skipped using WAL, and it's not a temp relation, + * we must force the relation down to disk before it's + * safe to commit the transaction. This requires forcing + * out any dirty buffers and then doing a forced fsync. + */ + FlushRelationBuffers(rel); + smgrimmedsync(rel->rd_smgr); + } +} diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index c7d3a439d5192a3a6ffb18840e7ef3eaabbcd574..449c57268e12af7a44ced433c2b2fe21039d4be1 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.68 2007/01/05 22:19:22 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.69 2007/01/25 02:17:26 momjian Exp $ * * * INTERFACE ROUTINES @@ -42,7 +42,7 @@ #undef TOAST_DEBUG static void toast_delete_datum(Relation rel, Datum value); -static Datum toast_save_datum(Relation rel, Datum value); +static Datum toast_save_datum(Relation rel, Datum value, bool use_wal); static varattrib *toast_fetch_datum(varattrib *attr); static varattrib *toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length); @@ -342,7 +342,7 @@ toast_delete(Relation rel, HeapTuple oldtup) * ---------- */ HeapTuple -toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup) +toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, bool use_wal) { HeapTuple result_tuple; TupleDesc tupleDesc; @@ -612,7 +612,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup) i = biggest_attno; old_value = toast_values[i]; toast_action[i] = 'p'; - toast_values[i] = toast_save_datum(rel, toast_values[i]); + toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal); if (toast_free[i]) pfree(DatumGetPointer(old_value)); @@ -724,7 +724,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup) i = biggest_attno; old_value = toast_values[i]; toast_action[i] = 'p'; - toast_values[i] = toast_save_datum(rel, toast_values[i]); + toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal); if (toast_free[i]) pfree(DatumGetPointer(old_value)); @@ -972,7 +972,7 @@ toast_compress_datum(Datum value) * ---------- */ static Datum -toast_save_datum(Relation rel, Datum value) +toast_save_datum(Relation rel, Datum value, bool use_wal) { Relation toastrel; Relation toastidx; @@ -1057,7 +1057,7 @@ toast_save_datum(Relation rel, Datum value) if (!HeapTupleIsValid(toasttup)) elog(ERROR, "failed to build TOAST tuple"); - simple_heap_insert(toastrel, toasttup); + fast_heap_insert(toastrel, toasttup, use_wal); /* * Create the index entry. We cheat a little here by not using diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index d75efdcc845129715297f9656aae91d96750485f..2cfd828bd520e0b9e51f375206a9336cbfc3989a 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.276 2007/01/09 02:14:11 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.277 2007/01/25 02:17:26 momjian Exp $ * * * INTERFACE ROUTINES @@ -1245,6 +1245,9 @@ setNewRelfilenode(Relation relation) heap_close(pg_class, RowExclusiveLock); + /* Remember we did this in current transaction, to allow later optimisations */ + relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId(); + /* Make sure the relfilenode change is visible */ CommandCounterIncrement(); } diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 6a5bba64ecbf3fc35c84744c15606a7a20dffb7a..e61384beec85ba96c2138ffdd112c78e54e13d02 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.274 2007/01/05 22:19:25 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.275 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -1652,6 +1652,7 @@ CopyFrom(CopyState cstate) ExprContext *econtext; /* used for ExecEvalExpr for default atts */ MemoryContext oldcontext = CurrentMemoryContext; ErrorContextCallback errcontext; + bool use_wal = true; /* By default, we use WAL to log db changes */ Assert(cstate->rel); @@ -1843,6 +1844,28 @@ CopyFrom(CopyState cstate) nfields = file_has_oids ? (attr_count + 1) : attr_count; field_strings = (char **) palloc(nfields * sizeof(char *)); + /* + * Check for performance optimization by avoiding WAL writes + * + * If archive logging is not be enabled *and* either + * - table is created in same transaction as this COPY + * - table data is now being written to new relfilenode + * then we can safely avoid writing WAL. Why? + * The data files for the table plus toast table/index, plus any indexes + * will all be dropped at the end of the transaction if it fails, so we + * do not need to worry about inconsistent states. + * As mentioned in comments in utils/rel.h, the in-same-transaction test is + * not completely reliable, since rd_createSubId can be reset to zero in + * certain cases before the end of the creating transaction. + * We are doing this for performance only, so we only need to know: + * if rd_createSubid != InvalidSubTransactionId then it is *always* just + * created. If we have PITR enabled, then we *must* use_wal + */ + if ((cstate->rel->rd_createSubid != InvalidSubTransactionId || + cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId) + && !XLogArchivingActive()) + use_wal = false; + /* Initialize state variables */ cstate->fe_eof = false; cstate->eol_type = EOL_UNKNOWN; @@ -2076,7 +2099,7 @@ CopyFrom(CopyState cstate) ExecConstraints(resultRelInfo, slot, estate); /* OK, store the tuple and create index entries for it */ - simple_heap_insert(cstate->rel, tuple); + fast_heap_insert(cstate->rel, tuple, use_wal); if (resultRelInfo->ri_NumIndices > 0) ExecInsertIndexTuples(slot, &(tuple->t_self), estate, false); @@ -2093,6 +2116,32 @@ CopyFrom(CopyState cstate) } } + /* + * If we skipped writing WAL for heaps, then we need to sync + */ + if (!use_wal) + { + /* main heap */ + heap_sync(cstate->rel); + + /* main heap indexes, if any */ + /* we always use WAL for index inserts, so no need to sync */ + + /* toast heap, if any */ + if (OidIsValid(cstate->rel->rd_rel->reltoastrelid)) + { + Relation toastrel; + + toastrel = heap_open(cstate->rel->rd_rel->reltoastrelid, + AccessShareLock); + heap_sync(toastrel); + heap_close(toastrel, AccessShareLock); + } + + /* toast index, if toast heap */ + /* we always use WAL for index inserts, so no need to sync */ + } + /* Done, clean up */ error_context_stack = errcontext.previous; diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 1de5db0b68b846efb7b46ef6f387ba1eaf94004e..06bc519dde3432484071471b0eb9568c05ae3c18 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -26,7 +26,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.283 2007/01/05 22:19:27 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.284 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -2516,11 +2516,7 @@ CloseIntoRel(QueryDesc *queryDesc) */ if (!estate->es_into_relation_use_wal && !estate->es_into_relation_descriptor->rd_istemp) - { - FlushRelationBuffers(estate->es_into_relation_descriptor); - /* FlushRelationBuffers will have opened rd_smgr */ - smgrimmedsync(estate->es_into_relation_descriptor->rd_smgr); - } + heap_sync(estate->es_into_relation_descriptor); /* close rel, but keep lock until commit */ heap_close(estate->es_into_relation_descriptor, NoLock); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index c43846cd57a5c540e17fa2c4cfea044b4e6a060d..da1f9fd924052e2694977dd90e9c8b23483e4433 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.254 2007/01/09 02:14:15 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.255 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -836,6 +836,7 @@ RelationBuildDesc(Oid targetRelId, Relation oldrelation) relation->rd_refcnt = 0; relation->rd_isnailed = false; relation->rd_createSubid = InvalidSubTransactionId; + relation->rd_newRelfilenodeSubid = InvalidSubTransactionId; relation->rd_istemp = isTempNamespace(relation->rd_rel->relnamespace); /* @@ -1358,6 +1359,7 @@ formrdesc(const char *relationName, Oid relationReltype, */ relation->rd_isnailed = true; relation->rd_createSubid = InvalidSubTransactionId; + relation->rd_newRelfilenodeSubid = InvalidSubTransactionId; relation->rd_istemp = false; /* @@ -1769,6 +1771,7 @@ RelationClearRelation(Relation relation, bool rebuild) Oid save_relid = RelationGetRelid(relation); int old_refcnt = relation->rd_refcnt; SubTransactionId old_createSubid = relation->rd_createSubid; + SubTransactionId old_newRelfilenodeSubid = relation->rd_newRelfilenodeSubid; TupleDesc old_att = relation->rd_att; RuleLock *old_rules = relation->rd_rules; MemoryContext old_rulescxt = relation->rd_rulescxt; @@ -1787,6 +1790,8 @@ RelationClearRelation(Relation relation, bool rebuild) } relation->rd_refcnt = old_refcnt; relation->rd_createSubid = old_createSubid; + relation->rd_newRelfilenodeSubid = old_newRelfilenodeSubid; + if (equalTupleDescs(old_att, relation->rd_att)) { /* needn't flush typcache here */ @@ -1827,7 +1832,8 @@ RelationFlushRelation(Relation relation) { bool rebuild; - if (relation->rd_createSubid != InvalidSubTransactionId) + if (relation->rd_createSubid != InvalidSubTransactionId || + relation->rd_newRelfilenodeSubid != InvalidSubTransactionId) { /* * New relcache entries are always rebuilt, not flushed; else we'd @@ -1909,6 +1915,9 @@ RelationCacheInvalidateEntry(Oid relationId) * so we do not touch new-in-transaction relations; they cannot be targets * of cross-backend SI updates (and our own updates now go through a * separate linked list that isn't limited by the SI message buffer size). + * We don't do anything special for newRelfilenode-in-transaction relations, + * though since we have a lock on the relation nobody else should be + * generating cache invalidation messages for it anyhow. * * We do this in two phases: the first pass deletes deletable items, and * the second one rebuilds the rebuildable items. This is essential for @@ -2085,6 +2094,7 @@ AtEOXact_RelationCache(bool isCommit) continue; } } + relation->rd_newRelfilenodeSubid = InvalidSubTransactionId; /* * Flush any temporary index list. @@ -2146,6 +2156,13 @@ AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid, continue; } } + if (relation->rd_newRelfilenodeSubid == mySubid) + { + if (isCommit) + relation->rd_newRelfilenodeSubid = parentSubid; + else + relation->rd_newRelfilenodeSubid = InvalidSubTransactionId; + } /* * Flush any temporary index list. @@ -2235,6 +2252,7 @@ RelationBuildLocalRelation(const char *relname, /* it's being created in this transaction */ rel->rd_createSubid = GetCurrentSubTransactionId(); + rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; /* must flag that we have rels created in this transaction */ need_eoxact_work = true; @@ -3392,6 +3410,7 @@ load_relcache_init_file(void) rel->rd_indexlist = NIL; rel->rd_oidindex = InvalidOid; rel->rd_createSubid = InvalidSubTransactionId; + rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; rel->rd_amcache = NULL; MemSet(&rel->pgstat_info, 0, sizeof(rel->pgstat_info)); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index eb780109062fd15489cfb5f38365eea0b8372916..3a689599570cc14f3346b7ddd87f5df70305c0ab 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.119 2007/01/09 22:01:00 momjian Exp $ + * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.120 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -178,6 +178,9 @@ extern void simple_heap_delete(Relation relation, ItemPointer tid); extern void simple_heap_update(Relation relation, ItemPointer otid, HeapTuple tup); +extern Oid fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal); + + extern void heap_markpos(HeapScanDesc scan); extern void heap_restrpos(HeapScanDesc scan); @@ -236,4 +239,6 @@ extern MinimalTuple minimal_tuple_from_heap_tuple(HeapTuple htup); extern HeapTuple heap_addheader(int natts, bool withoid, Size structlen, void *structure); +extern void heap_sync(Relation relation); + #endif /* HEAPAM_H */ diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index c76c47abbed9340254bab90fe91e871476f8e8f3..ee8cfa74492ef6306256c588d9afbdcdc20b870a 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -6,7 +6,7 @@ * * Copyright (c) 2000-2007, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.29 2007/01/05 22:19:51 momjian Exp $ + * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.30 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -69,7 +69,7 @@ * ---------- */ extern HeapTuple toast_insert_or_update(Relation rel, - HeapTuple newtup, HeapTuple oldtup); + HeapTuple newtup, HeapTuple oldtup, bool use_wal); /* ---------- * toast_delete - diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index c8b78b954220025068e80068cc138d29ec3adba8..8ce7e118a4fc2e336970d1f473271732960c16e9 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.95 2007/01/09 02:14:16 tgl Exp $ + * $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.96 2007/01/25 02:17:26 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -137,6 +137,7 @@ typedef struct RelationData char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 = * valid, 2 = temporarily forced */ SubTransactionId rd_createSubid; /* rel was created in current xact */ + SubTransactionId rd_newRelfilenodeSubid; /* rel had new relfilenode in current xact */ /* * rd_createSubid is the ID of the highest subtransaction the rel has