diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1e22c0304620ef9dc873c555e61b36eed34a5708..d5f36e4e4d9468db715b0231209f82b50eb71473 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/access/heap/heapam.c,v 1.83 2000/08/03 19:18:54 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/access/heap/heapam.c,v 1.84 2000/08/04 04:16:06 tgl Exp $
*
*
* INTERFACE ROUTINES
@@ -1362,7 +1362,7 @@ heap_insert(Relation relation, HeapTuple tup)
* ----------
*/
if (HeapTupleHasExtended(tup) ||
- (MAXALIGN(tup->t_len) > (MaxTupleSize / 4)))
+ (MAXALIGN(tup->t_len) > TOAST_TUPLE_THRESHOLD))
heap_tuple_toast_attrs(relation, tup, NULL);
#endif
@@ -1621,13 +1621,15 @@ l2:
#ifdef TUPLE_TOASTER_ACTIVE
/* ----------
* If this relation is enabled for toasting, let the toaster
- * delete not any longer needed entries and create new ones to
- * make the new tuple fit again.
+ * delete any no-longer-needed entries and create new ones to
+ * make the new tuple fit again. Also, if there are already-
+ * toasted values from some other relation, the toaster must
+ * fix them.
* ----------
*/
if (HeapTupleHasExtended(&oldtup) ||
- HeapTupleHasExtended(newtup) ||
- (MAXALIGN(newtup->t_len) > (MaxTupleSize / 4)))
+ HeapTupleHasExtended(newtup) ||
+ (MAXALIGN(newtup->t_len) > TOAST_TUPLE_THRESHOLD))
heap_tuple_toast_attrs(relation, newtup, &oldtup);
#endif
diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index 61076444a2df525b9fb0a10e0209c0d1f07a1047..05952bc80e87c10899d301a063124adc697da0b4 100644
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.11 2000/08/03 16:33:40 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.12 2000/08/04 04:16:07 tgl Exp $
*
*
* INTERFACE ROUTINES
@@ -22,11 +22,10 @@
*-------------------------------------------------------------------------
*/
-#include <stdio.h>
-#include <stdlib.h>
+#include "postgres.h"
+
#include <unistd.h>
#include <fcntl.h>
-#include "postgres.h"
#include "access/heapam.h"
#include "access/genam.h"
@@ -39,6 +38,7 @@
#ifdef TUPLE_TOASTER_ACTIVE
+
#undef TOAST_DEBUG
static void toast_delete(Relation rel, HeapTuple oldtup);
@@ -47,7 +47,6 @@ static void toast_insert_or_update(Relation rel, HeapTuple newtup,
HeapTuple oldtup);
static Datum toast_compress_datum(Datum value);
static Datum toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value);
-
static varattrib *toast_fetch_datum(varattrib *attr);
@@ -209,7 +208,7 @@ toast_delete(Relation rel, HeapTuple oldtup)
/* ----------
* toast_insert_or_update -
*
- * Delete no more used toast-entries and create new ones to
+ * Delete no-longer-used toast-entries and create new ones to
* make the new tuple fit on INSERT or UPDATE
* ----------
*/
@@ -375,7 +374,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
}
/* ----------
- * Compress and/or save external until data fits
+ * Compress and/or save external until data fits into target length
*
* 1: Inline compress attributes with attstorage 'x'
* 2: Store attributes with attstorage 'x' or 'e' external
@@ -386,7 +385,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
maxDataLen = offsetof(HeapTupleHeaderData, t_bits);
if (has_nulls)
maxDataLen += BITMAPLEN(numAttrs);
- maxDataLen = (MaxTupleSize / 4) - MAXALIGN(maxDataLen);
+ maxDataLen = TOAST_TUPLE_TARGET - MAXALIGN(maxDataLen);
/* ----------
* Look for attributes with attstorage 'x' to compress
@@ -560,7 +559,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
/* ----------
* Search for the biggest yet inlined attribute with
- * attstorage = 'x' or 'e'
+ * attstorage = 'm'
* ----------
*/
for (i = 0; i < numAttrs; i++)
@@ -684,15 +683,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
if (toast_delold[i])
toast_delete_datum(rel,
heap_getattr(oldtup, i + 1, tupleDesc, &old_isnull));
-
- return;
}
/* ----------
* toast_compress_datum -
*
- * Create a compressed version of a datum
+ * Create a compressed version of a varlena datum
* ----------
*/
static Datum
@@ -726,9 +723,9 @@ toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value)
InsertIndexResult idxres;
TupleDesc toasttupDesc;
Datum t_values[3];
- char t_nulls[4];
+ char t_nulls[3];
varattrib *result;
- char chunk_data[MaxTupleSize];
+ char chunk_data[VARHDRSZ + TOAST_MAX_CHUNK_SIZE];
int32 chunk_size;
int32 chunk_seq = 0;
char *data_p;
@@ -769,7 +766,6 @@ toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value)
t_nulls[0] = ' ';
t_nulls[1] = ' ';
t_nulls[2] = ' ';
- t_nulls[3] = '\0';
/* ----------
* Get the data to process
@@ -783,16 +779,8 @@ toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value)
* ----------
*/
toastrel = heap_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
- if (toastrel == NULL)
- elog(ERROR, "Failed to open secondary relation of %s",
- DatumGetCString(DirectFunctionCall1(nameout,
- NameGetDatum(&(rel->rd_rel->relname)))));
toasttupDesc = toastrel->rd_att;
toastidx = index_open(rel->rd_rel->reltoastidxid);
- if (toastidx == NULL)
- elog(ERROR, "Failed to open index for secondary relation of %s",
- DatumGetCString(DirectFunctionCall1(nameout,
- NameGetDatum(&(rel->rd_rel->relname)))));
/* ----------
* Split up the item into chunks
@@ -804,14 +792,13 @@ toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value)
* Calculate the size of this chunk
* ----------
*/
- chunk_size = (TOAST_MAX_CHUNK_SIZE < data_todo) ?
- TOAST_MAX_CHUNK_SIZE : data_todo;
+ chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
/* ----------
* Build a tuple
* ----------
*/
- t_values[1] = (Datum)(chunk_seq++);
+ t_values[1] = Int32GetDatum(chunk_seq++);
VARATT_SIZEP(chunk_data) = chunk_size + VARHDRSZ;
memcpy(VARATT_DATA(chunk_data), data_p, chunk_size);
toasttup = heap_formtuple(toasttupDesc, t_values, t_nulls);
@@ -882,11 +869,7 @@ toast_delete_datum(Relation rel, Datum value)
*/
toastrel = heap_open(attr->va_content.va_external.va_toastrelid,
RowExclusiveLock);
- if (toastrel == NULL)
- elog(ERROR, "Failed to open secondary relation at TOAST fetch");
toastidx = index_open(attr->va_content.va_external.va_toastidxid);
- if (toastidx == NULL)
- elog(ERROR, "Failed to open index of secondary relation at TOAST fetch");
/* ----------
* Setup a scan key to fetch from the index by va_valueid
@@ -928,8 +911,6 @@ toast_delete_datum(Relation rel, Datum value)
index_endscan(toastscan);
index_close(toastidx);
heap_close(toastrel, RowExclusiveLock);
-
- return;
}
@@ -957,14 +938,15 @@ toast_fetch_datum(varattrib *attr)
int32 ressize;
int32 residx;
int numchunks;
- Datum chunk;
+ Pointer chunk;
bool isnull;
+ int32 chunksize;
char *chunks_found;
char *chunks_expected;
ressize = attr->va_content.va_external.va_extsize;
- numchunks = (ressize / TOAST_MAX_CHUNK_SIZE) + 1;
+ numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
chunks_found = palloc(numchunks);
chunks_expected = palloc(numchunks);
@@ -982,12 +964,8 @@ toast_fetch_datum(varattrib *attr)
*/
toastrel = heap_open(attr->va_content.va_external.va_toastrelid,
AccessShareLock);
- if (toastrel == NULL)
- elog(ERROR, "Failed to open secondary relation at TOAST fetch");
toasttupDesc = toastrel->rd_att;
toastidx = index_open(attr->va_content.va_external.va_toastidxid);
- if (toastidx == NULL)
- elog(ERROR, "Failed to open index of secondary relation at TOAST fetch");
/* ----------
* Setup a scan key to fetch from the index by va_valueid
@@ -1001,6 +979,8 @@ toast_fetch_datum(varattrib *attr)
/* ----------
* Read the chunks by index
+ *
+ * Note we will not necessarily see the chunks in sequence-number order.
* ----------
*/
toastscan = index_beginscan(toastidx, false, 1, &toastkey);
@@ -1018,30 +998,46 @@ toast_fetch_datum(varattrib *attr)
* Have a chunk, extract the sequence number and the data
* ----------
*/
- residx = (int32)heap_getattr(ttup, 2, toasttupDesc, &isnull);
- chunk = heap_getattr(ttup, 3, toasttupDesc, &isnull);
+ residx = DatumGetInt32(heap_getattr(ttup, 2, toasttupDesc, &isnull));
+ Assert(!isnull);
+ chunk = DatumGetPointer(heap_getattr(ttup, 3, toasttupDesc, &isnull));
+ Assert(!isnull);
+ chunksize = VARATT_SIZE(chunk) - VARHDRSZ;
/* ----------
* Some checks on the data we've found
* ----------
*/
- if (residx * TOAST_MAX_CHUNK_SIZE + VARATT_SIZE(chunk) - VARHDRSZ
- > ressize)
- elog(ERROR, "chunk data exceeds original data size for "
- "toast value %d",
- attr->va_content.va_external.va_valueid);
+ if (residx < 0 || residx >= numchunks)
+ elog(ERROR, "unexpected chunk number %d for toast value %d",
+ residx,
+ attr->va_content.va_external.va_valueid);
+ if (residx < numchunks-1)
+ {
+ if (chunksize != TOAST_MAX_CHUNK_SIZE)
+ elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %d",
+ chunksize, residx,
+ attr->va_content.va_external.va_valueid);
+ }
+ else
+ {
+ if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize)
+ elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %d",
+ chunksize, residx,
+ attr->va_content.va_external.va_valueid);
+ }
if (chunks_found[residx]++ > 0)
elog(ERROR, "chunk %d for toast value %d appears multiple times",
- residx,
- attr->va_content.va_external.va_valueid);
+ residx,
+ attr->va_content.va_external.va_valueid);
/* ----------
- * Copy the data into our result
+ * Copy the data into proper place in our result
* ----------
*/
memcpy(((char *)VARATT_DATA(result)) + residx * TOAST_MAX_CHUNK_SIZE,
- VARATT_DATA(chunk),
- VARATT_SIZE(chunk) - VARHDRSZ);
+ VARATT_DATA(chunk),
+ chunksize);
ReleaseBuffer(buffer);
}
diff --git a/src/backend/commands/command.c b/src/backend/commands/command.c
index de2597bd266016ea9a3b58aaf0c39605fcf8f5f0..68487f64ffce08b5e61edfa2629fcaefeb3f75d9 100644
--- a/src/backend/commands/command.c
+++ b/src/backend/commands/command.c
@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/commands/Attic/command.c,v 1.93 2000/08/03 19:19:18 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/commands/Attic/command.c,v 1.94 2000/08/04 04:16:06 tgl Exp $
*
* NOTES
* The PerformAddAttribute() code, like most of the relation
@@ -1503,6 +1503,14 @@ AlterTableCreateToastTable(const char *relationName, bool silent)
"chunk_data",
BYTEAOID,
-1, 0, false);
+ /*
+ * Ensure that the toast table doesn't itself get toasted,
+ * or we'll be toast :-(. This is essential for chunk_data because
+ * type bytea is toastable; hit the other two just to be sure.
+ */
+ tupdesc->attrs[0]->attstorage = 'p';
+ tupdesc->attrs[1]->attstorage = 'p';
+ tupdesc->attrs[2]->attstorage = 'p';
/*
* Note: the toast relation is considered a "normal" relation even if
diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index 91149f5f8362082e91bf87d1bc2fb65cb6b60060..23f31aec522572def9504b8c91623880d3d088bd 100644
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -6,7 +6,7 @@
*
* Copyright (c) 2000, PostgreSQL Development Team
*
- * $Id: tuptoaster.h,v 1.7 2000/07/22 11:18:47 wieck Exp $
+ * $Id: tuptoaster.h,v 1.8 2000/08/04 04:16:10 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -20,25 +20,57 @@
#include "access/tupmacs.h"
#include "utils/rel.h"
+
+/*
+ * This enables de-toasting of index entries. Needed until VACUUM is
+ * smart enough to rebuild indexes from scratch.
+ */
#define TOAST_INDEX_HACK
-#define TOAST_MAX_CHUNK_SIZE ((MaxTupleSize - \
- MAXALIGN( \
- MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) + \
- MAXALIGN(sizeof(Oid)) + \
- MAXALIGN(sizeof(int32)) + \
- MAXALIGN(VARHDRSZ))) / 4)
+/*
+ * These symbols control toaster activation. If a tuple is larger than
+ * TOAST_TUPLE_THRESHOLD, we will try to toast it down to no more than
+ * TOAST_TUPLE_TARGET bytes. Both numbers include all tuple header and
+ * alignment-padding overhead.
+ *
+ * The numbers need not be the same, though they currently are.
+ */
+#define TOAST_TUPLE_THRESHOLD (MaxTupleSize / 4)
+
+#define TOAST_TUPLE_TARGET (MaxTupleSize / 4)
+
+
+/*
+ * When we store an oversize datum externally, we divide it into chunks
+ * containing at most TOAST_MAX_CHUNK_SIZE data bytes. This number *must*
+ * be small enough that the completed toast-table tuple (including the
+ * ID and sequence fields and all overhead) is no more than MaxTupleSize
+ * bytes. It *should* be small enough to make toast-table tuples no more
+ * than TOAST_TUPLE_THRESHOLD bytes, else heapam.c will uselessly invoke
+ * the toaster on toast-table tuples.
+ *
+ * NB: you cannot change this value without forcing initdb, at least not
+ * if your DB contains any multi-chunk toasted values.
+ */
+#define TOAST_MAX_CHUNK_SIZE (TOAST_TUPLE_THRESHOLD - \
+ MAXALIGN( \
+ MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) + \
+ sizeof(Oid) + \
+ sizeof(int32) + \
+ VARHDRSZ))
/* ----------
* heap_tuple_toast_attrs() -
*
* Called by heap_insert(), heap_update() and heap_delete().
- * Outdates not any longer needed toast entries referenced
- * by oldtup and creates new ones until newtup is smaller
- * that ~2K (or running out of toastable values).
+ * Outdates any no-longer-needed toast entries referenced
+ * by oldtup and creates new ones until newtup is no more than
+ * TOAST_TUPLE_TARGET (or we run out of toastable values).
* Possibly modifies newtup by replacing the t_data part!
+ *
+ * oldtup is NULL if insert, newtup is NULL if delete.
* ----------
*/
extern void heap_tuple_toast_attrs(Relation rel,
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index 0a8b0d43ccef8e57b9c724c5b4a233f87f04b74a..c9ac2c39abae763497dfb1fb8bf8047084232150 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -37,7 +37,7 @@
* Portions Copyright (c) 1996-2000, PostgreSQL, Inc
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $Id: catversion.h,v 1.38 2000/07/30 22:13:59 tgl Exp $
+ * $Id: catversion.h,v 1.39 2000/08/04 04:16:17 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -53,6 +53,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 200007301
+#define CATALOG_VERSION_NO 200008031
#endif