Skip to content

Commit

Permalink
Decimal: SetInt: default precision uses whole input magnitude
Browse files Browse the repository at this point in the history
When z.prec is zero, compute precision based on all decimal digits in
source big.Int, including trailing zeros (i.e. the precision should
match the magnitude of the input, regardless of its actual value).

Before the change, trailing zero digits were ignored.
  • Loading branch information
db47h committed Jun 1, 2020
1 parent 81d67aa commit ab4d78a
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 9 deletions.
16 changes: 10 additions & 6 deletions decimal.go
Original file line number Diff line number Diff line change
Expand Up @@ -1147,12 +1147,16 @@ func (z *Decimal) SetInt(x *big.Int) *Decimal {
// TODO(db47h) truncating x could be more efficient if z.prec > 0
// but small compared to the size of x.
z.mant = z.mant.make(int((prec + _DW - 1) / _DW)).setNat(x.Bits())
exp := dnorm(z.mant)
if z.prec == 0 {
// adjust precision
z.prec = uint32(max(len(z.mant)*_DW-int(z.mant.trailingZeroDigits()), DefaultDecimalPrec))
// adjust precision. z.mant is not notmalized yet. use nlz() and accept
// trailing zeros as part of the required precision
digits := int64(len(z.mant))*_DW - int64(nlz10(z.mant[len(z.mant)-1]))
if digits > MaxPrec {
digits = MaxPrec
}
z.prec = umax32(uint32(digits), DefaultDecimalPrec)
}
z.setExpAndRound(int64(len(z.mant))*_DW-exp, 0)
z.setExpAndRound(int64(len(z.mant))*_DW-dnorm(z.mant), 0)
return z
}

Expand Down Expand Up @@ -1584,9 +1588,9 @@ func dnorm(m dec) int64 {
if debugDecimal && (len(m) == 0 || m[len(m)-1] == 0) {
panic("msw of mantissa is 0")
}
s := _DW - decDigits(uint(m[len(m)-1]))
// partial shift
s := nlz10(m[len(m)-1])
if s > 0 {
// partial shift
c := shl10VU(m, m, s)
if debugDecimal && c != 0 {
panic("nlz or shlVU incorrect")
Expand Down
4 changes: 2 additions & 2 deletions decimal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -721,7 +721,7 @@ func TestDecimalSetInt(t *testing.T) {
f.SetInt(&x)

// check precision
n := len(strings.TrimRight(want, "0"))
n := len(want)
if n < DefaultDecimalPrec {
n = DefaultDecimalPrec
}
Expand All @@ -747,7 +747,7 @@ func TestDecimalSetRat(t *testing.T) {
{"-1", DefaultDecimalPrec},
{"1234567890", DefaultDecimalPrec},
{"123456789012345678901234567890", DefaultDecimalPrec},
{"123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", 89},
{"123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", 90},
{"1.2", DefaultDecimalPrec},
{"3.14159265", DefaultDecimalPrec},
// TODO(db47h) expand
Expand Down
2 changes: 1 addition & 1 deletion decimal_toa.go
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ func (x *Decimal) fmtP(buf []byte) []byte {
mant, exp := x.toa(10)
buf = append(buf, bytes.TrimRight(mant, "0")...)
buf = append(buf, 'e')
if x.exp >= 0 {
if exp >= 0 {
buf = append(buf, '+')
}
return strconv.AppendInt(buf, int64(exp), 10)
Expand Down

0 comments on commit ab4d78a

Please sign in to comment.