From 1bd3a8f58bb8147628a85643b0246051a6e4e5ee Mon Sep 17 00:00:00 2001
From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Thu, 3 Aug 2000 00:58:22 +0000
Subject: [PATCH] Repair bug noted by Paul Caskey: neqsel() has been generating
a bogus result, in fact nearly the opposite of what it should, because it was
passing the not-equal operator to eqsel() which would use it to compare the
value against the most common value in the column, and of course obtain the
wrong result therefrom. Must pass the equality operator to eqsel() instead.
Fortunately that's easy to get from the oprnegate link.
---
src/backend/utils/adt/selfuncs.c | 63 +++++++++++++++++++++++++++++---
1 file changed, 58 insertions(+), 5 deletions(-)
diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 04d9d9f6a9e..16b7964392e 100644
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -15,7 +15,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.76 2000/07/29 03:26:42 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.77 2000/08/03 00:58:22 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -252,9 +252,33 @@ eqsel(PG_FUNCTION_ARGS)
Datum
neqsel(PG_FUNCTION_ARGS)
{
+ Oid opid = PG_GETARG_OID(0);
+ Oid relid = PG_GETARG_OID(1);
+ AttrNumber attno = PG_GETARG_INT16(2);
+ Datum value = PG_GETARG_DATUM(3);
+ int32 flag = PG_GETARG_INT32(4);
+ Oid eqopid;
float8 result;
- result = DatumGetFloat8(eqsel(fcinfo));
+ /*
+ * We want 1 - eqsel() where the equality operator is the one associated
+ * with this != operator, that is, its negator.
+ */
+ eqopid = get_negator(opid);
+ if (eqopid)
+ {
+ result = DatumGetFloat8(DirectFunctionCall5(eqsel,
+ ObjectIdGetDatum(eqopid),
+ ObjectIdGetDatum(relid),
+ Int16GetDatum(attno),
+ value,
+ Int32GetDatum(flag)));
+ }
+ else
+ {
+ /* Use default selectivity (should we raise an error instead?) */
+ result = DEFAULT_EQ_SEL;
+ }
result = 1.0 - result;
PG_RETURN_FLOAT8(result);
}
@@ -392,15 +416,39 @@ scalarltsel(PG_FUNCTION_ARGS)
Datum
scalargtsel(PG_FUNCTION_ARGS)
{
+ Oid opid = PG_GETARG_OID(0);
+ Oid relid = PG_GETARG_OID(1);
+ AttrNumber attno = PG_GETARG_INT16(2);
+ Datum value = PG_GETARG_DATUM(3);
+ int32 flag = PG_GETARG_INT32(4);
+ Oid ltopid;
float8 result;
/*
* Compute selectivity of "<", then invert --- but only if we were
- * able to produce a non-default estimate.
+ * able to produce a non-default estimate. Note that we get the
+ * negator which strictly speaking means we are looking at "<="
+ * for ">" or "<" for ">=". We assume this won't matter.
*/
- result = DatumGetFloat8(scalarltsel(fcinfo));
+ ltopid = get_negator(opid);
+ if (ltopid)
+ {
+ result = DatumGetFloat8(DirectFunctionCall5(scalarltsel,
+ ObjectIdGetDatum(ltopid),
+ ObjectIdGetDatum(relid),
+ Int16GetDatum(attno),
+ value,
+ Int32GetDatum(flag)));
+ }
+ else
+ {
+ /* Use default selectivity (should we raise an error instead?) */
+ result = DEFAULT_INEQ_SEL;
+ }
+
if (result != DEFAULT_INEQ_SEL)
result = 1.0 - result;
+
PG_RETURN_FLOAT8(result);
}
@@ -567,7 +615,7 @@ nlikesel(PG_FUNCTION_ARGS)
Datum
eqjoinsel(PG_FUNCTION_ARGS)
{
-#ifdef NOT_USED
+#ifdef NOT_USED /* see neqjoinsel() before removing me! */
Oid opid = PG_GETARG_OID(0);
#endif
Oid relid1 = PG_GETARG_OID(1);
@@ -620,6 +668,11 @@ neqjoinsel(PG_FUNCTION_ARGS)
{
float8 result;
+ /*
+ * XXX we skip looking up the negator operator here because we know
+ * eqjoinsel() won't look at it anyway. If eqjoinsel() ever does look,
+ * this routine will need to look more like neqsel() does.
+ */
result = DatumGetFloat8(eqjoinsel(fcinfo));
result = 1.0 - result;
PG_RETURN_FLOAT8(result);
--
GitLab