use doubles to avoid multiple integer overflows in slope() function

This commit is contained in:
Axel Kohlmeyer
2020-07-02 16:43:00 -04:00
parent f598ae7ebf
commit 307f54611f

View File

@ -4057,7 +4057,7 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
Tree **treestack, int &ntreestack, Tree **treestack, int &ntreestack,
double *argstack, int &nargstack, int ivar) double *argstack, int &nargstack, int ivar)
{ {
bigint sx,sxx; double sx,sxx;
double value,sy,sxy; double value,sy,sxy;
// word not a match to any special function // word not a match to any special function
@ -4213,10 +4213,7 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
"variable formula",ivar); "variable formula",ivar);
value = 0.0; value = 0.0;
if (method == SLOPE) { if (method == SLOPE) sx = sxx = sy = sxy = 0.0;
sx = sxx = 0;
sy = sxy = 0.0;
}
else if (method == XMIN) value = BIG; else if (method == XMIN) value = BIG;
else if (method == XMAX) value = -BIG; else if (method == XMAX) value = -BIG;
@ -4235,10 +4232,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += vec[j]; else if (method == AVE) value += vec[j];
else if (method == TRAP) value += vec[j]; else if (method == TRAP) value += vec[j];
else if (method == SLOPE) { else if (method == SLOPE) {
sx += i; sx += (double)i;
sy += vec[j]; sy += vec[j];
sxx += i*i; sxx += (double)i * (double)i;
sxy += i*vec[j]; sxy += (double)i * vec[j];
} }
j += nstride; j += nstride;
} }
@ -4256,10 +4253,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += one; else if (method == AVE) value += one;
else if (method == TRAP) value += one; else if (method == TRAP) value += one;
else if (method == SLOPE) { else if (method == SLOPE) {
sx += i; sx += (double)i;
sy += one; sy += one;
sxx += i*i; sxx += (double)i * (double)i;
sxy += i*one; sxy += (double)i * one;
} }
} }
if (method == TRAP) { if (method == TRAP) {
@ -4281,10 +4278,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += one; else if (method == AVE) value += one;
else if (method == TRAP) value += one; else if (method == TRAP) value += one;
else if (method == SLOPE) { else if (method == SLOPE) {
sx += i; sx += (double) i;
sy += one; sy += one;
sxx += i*i; sxx += (double)i * (double)i;
sxy += i*one; sxy += (double)i * one;
} }
} }
if (method == TRAP) value -= 0.5*vec[0] + 0.5*vec[nvec-1]; if (method == TRAP) value -= 0.5*vec[0] + 0.5*vec[nvec-1];