use doubles to avoid multiple integer overflows in slope() function

This commit is contained in:
Axel Kohlmeyer
2020-07-02 16:43:00 -04:00
parent f598ae7ebf
commit 307f54611f

View File

@ -4057,7 +4057,7 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
Tree **treestack, int &ntreestack,
double *argstack, int &nargstack, int ivar)
{
bigint sx,sxx;
double sx,sxx;
double value,sy,sxy;
// word not a match to any special function
@ -4213,10 +4213,7 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
"variable formula",ivar);
value = 0.0;
if (method == SLOPE) {
sx = sxx = 0;
sy = sxy = 0.0;
}
if (method == SLOPE) sx = sxx = sy = sxy = 0.0;
else if (method == XMIN) value = BIG;
else if (method == XMAX) value = -BIG;
@ -4235,10 +4232,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += vec[j];
else if (method == TRAP) value += vec[j];
else if (method == SLOPE) {
sx += i;
sx += (double)i;
sy += vec[j];
sxx += i*i;
sxy += i*vec[j];
sxx += (double)i * (double)i;
sxy += (double)i * vec[j];
}
j += nstride;
}
@ -4256,10 +4253,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += one;
else if (method == TRAP) value += one;
else if (method == SLOPE) {
sx += i;
sx += (double)i;
sy += one;
sxx += i*i;
sxy += i*one;
sxx += (double)i * (double)i;
sxy += (double)i * one;
}
}
if (method == TRAP) {
@ -4281,10 +4278,10 @@ int Variable::special_function(char *word, char *contents, Tree **tree,
else if (method == AVE) value += one;
else if (method == TRAP) value += one;
else if (method == SLOPE) {
sx += i;
sx += (double) i;
sy += one;
sxx += i*i;
sxy += i*one;
sxx += (double)i * (double)i;
sxy += (double)i * one;
}
}
if (method == TRAP) value -= 0.5*vec[0] + 0.5*vec[nvec-1];