From 83c94a886c059e8d45cb2f5cd097df634ef5371d Mon Sep 17 00:00:00 2001
From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Sat, 14 Apr 2001 02:10:57 +0000
Subject: [PATCH] Another try at making numeric MODULO operator produce the
right answer. Although it was now using the right equation, it was making
bogus choices of the precision to compute intermediate results to. I'm not
sure this is really right even yet, but it's better than before ...
---
src/backend/utils/adt/numeric.c | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c
index 4399ae554b3..d4e93cf8756 100644
--- a/src/backend/utils/adt/numeric.c
+++ b/src/backend/utils/adt/numeric.c
@@ -5,7 +5,7 @@
*
* 1998 Jan Wieck
*
- * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.39 2001/03/22 06:16:17 momjian Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.40 2001/04/14 02:10:57 tgl Exp $
*
* ----------
*/
@@ -1134,7 +1134,6 @@ numeric_mod(PG_FUNCTION_ARGS)
mod_var(&arg1, &arg2, &result);
- result.dscale = result.rscale;
res = make_result(&result);
free_var(&result);
@@ -3281,29 +3280,42 @@ mod_var(NumericVar *var1, NumericVar *var2, NumericVar *result)
{
NumericVar tmp;
int save_global_rscale;
+ int div_dscale;
init_var(&tmp);
/* ---------
* We do this using the equation
* mod(x,y) = x - trunc(x/y)*y
- * We fiddle a bit with global_rscale to control result precision.
+ * We set global_rscale the same way numeric_div and numeric_mul do
+ * to get the right answer from the equation. The final result,
+ * however, need not be displayed to more precision than the inputs.
* ----------
*/
save_global_rscale = global_rscale;
- global_rscale = var2->rscale + 2;
+
+ div_dscale = MAX(var1->dscale + var2->dscale, NUMERIC_MIN_DISPLAY_SCALE);
+ div_dscale = MIN(div_dscale, NUMERIC_MAX_DISPLAY_SCALE);
+ global_rscale = MAX(var1->rscale + var2->rscale,
+ NUMERIC_MIN_RESULT_SCALE);
+ global_rscale = MAX(global_rscale, div_dscale + 4);
+ global_rscale = MIN(global_rscale, NUMERIC_MAX_RESULT_SCALE);
div_var(var1, var2, &tmp);
+ tmp.dscale = div_dscale;
+
/* do trunc() by forgetting digits to the right of the decimal point */
tmp.ndigits = MAX(0, MIN(tmp.ndigits, tmp.weight + 1));
- tmp.rscale = var2->rscale;
- global_rscale = var2->rscale;
+ global_rscale = var2->rscale + tmp.rscale;
+
mul_var(var2, &tmp, &tmp);
sub_var(var1, &tmp, result);
+ result->dscale = MAX(var1->dscale, var2->dscale);
+
global_rscale = save_global_rscale;
free_var(&tmp);
}
--
GitLab