From 1dc3a62ec7c9bd3f602689b289a1126fca12fead Mon Sep 17 00:00:00 2001
From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Mon, 21 Apr 2003 00:22:24 +0000
Subject: [PATCH] stddev() and variance() should return NULL when there is just
one input value, per recent discussion on pgsql-general.
---
src/backend/utils/adt/float.c | 16 +++++----------
src/backend/utils/adt/numeric.c | 36 ++++++++++++---------------------
2 files changed, 18 insertions(+), 34 deletions(-)
diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c
index 4220b4775a9..95206174eff 100644
--- a/src/backend/utils/adt/float.c
+++ b/src/backend/utils/adt/float.c
@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/utils/adt/float.c,v 1.84 2003/03/11 21:01:33 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/float.c,v 1.85 2003/04/21 00:22:24 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -1649,12 +1649,9 @@ float8_variance(PG_FUNCTION_ARGS)
sumX = transvalues[1];
sumX2 = transvalues[2];
- /* We define VARIANCE of no values to be NULL, of 1 value to be 0 */
- if (N == 0.0)
- PG_RETURN_NULL();
-
+ /* Sample variance is undefined when N is 0 or 1, so return NULL */
if (N <= 1.0)
- PG_RETURN_FLOAT8(0.0);
+ PG_RETURN_NULL();
numerator = N * sumX2 - sumX * sumX;
@@ -1680,12 +1677,9 @@ float8_stddev(PG_FUNCTION_ARGS)
sumX = transvalues[1];
sumX2 = transvalues[2];
- /* We define STDDEV of no values to be NULL, of 1 value to be 0 */
- if (N == 0.0)
- PG_RETURN_NULL();
-
+ /* Sample stddev is undefined when N is 0 or 1, so return NULL */
if (N <= 1.0)
- PG_RETURN_FLOAT8(0.0);
+ PG_RETURN_NULL();
numerator = N * sumX2 - sumX * sumX;
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c
index e6698fea6c0..c851ae5eb35 100644
--- a/src/backend/utils/adt/numeric.c
+++ b/src/backend/utils/adt/numeric.c
@@ -14,7 +14,7 @@
* Copyright (c) 1998-2003, PostgreSQL Global Development Group
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.59 2003/03/21 01:58:04 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.60 2003/04/21 00:22:24 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -1964,24 +1964,19 @@ numeric_variance(PG_FUNCTION_ARGS)
if (NUMERIC_IS_NAN(N) || NUMERIC_IS_NAN(sumX) || NUMERIC_IS_NAN(sumX2))
PG_RETURN_NUMERIC(make_result(&const_nan));
- /* We define VARIANCE of no values to be NULL, of 1 value to be 0 */
- /* N is zero iff no digits (cf. numeric_uminus) */
- if (N->varlen == NUMERIC_HDRSZ)
- PG_RETURN_NULL();
-
+ /* Sample variance is undefined when N is 0 or 1, so return NULL */
init_var(&vN);
set_var_from_num(N, &vN);
- init_var(&vNminus1);
- sub_var(&vN, &const_one, &vNminus1);
-
- if (cmp_var(&vNminus1, &const_zero) <= 0)
+ if (cmp_var(&vN, &const_one) <= 0)
{
free_var(&vN);
- free_var(&vNminus1);
- PG_RETURN_NUMERIC(make_result(&const_zero));
+ PG_RETURN_NULL();
}
+ init_var(&vNminus1);
+ sub_var(&vN, &const_one, &vNminus1);
+
init_var(&vsumX);
set_var_from_num(sumX, &vsumX);
init_var(&vsumX2);
@@ -2045,24 +2040,19 @@ numeric_stddev(PG_FUNCTION_ARGS)
if (NUMERIC_IS_NAN(N) || NUMERIC_IS_NAN(sumX) || NUMERIC_IS_NAN(sumX2))
PG_RETURN_NUMERIC(make_result(&const_nan));
- /* We define STDDEV of no values to be NULL, of 1 value to be 0 */
- /* N is zero iff no digits (cf. numeric_uminus) */
- if (N->varlen == NUMERIC_HDRSZ)
- PG_RETURN_NULL();
-
+ /* Sample stddev is undefined when N is 0 or 1, so return NULL */
init_var(&vN);
set_var_from_num(N, &vN);
- init_var(&vNminus1);
- sub_var(&vN, &const_one, &vNminus1);
-
- if (cmp_var(&vNminus1, &const_zero) <= 0)
+ if (cmp_var(&vN, &const_one) <= 0)
{
free_var(&vN);
- free_var(&vNminus1);
- PG_RETURN_NUMERIC(make_result(&const_zero));
+ PG_RETURN_NULL();
}
+ init_var(&vNminus1);
+ sub_var(&vN, &const_one, &vNminus1);
+
init_var(&vsumX);
set_var_from_num(sumX, &vsumX);
init_var(&vsumX2);
--
GitLab