Skip to content

Commit

Permalink
Fix incorrect calculation of const precision, add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
mwoss committed Dec 29, 2023
1 parent add0605 commit 0f927d0
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 6 deletions.
12 changes: 7 additions & 5 deletions const.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func newConstApproximation(value string) constApproximation {

var approximations []Decimal
for p := 1; p < maxPrecision; p *= 2 {
r := RequireFromString(strLn10[:coeffLen+p])
r := RequireFromString(value[:coeffLen+p])
approximations = append(approximations, r)
}

Expand All @@ -36,13 +36,15 @@ func newConstApproximation(value string) constApproximation {
}
}

// Returns the smallest approximation available that's at least as precise as the passed precision
// i.e. Ceil[ log2(precision) ] = 1 + Floor[ log2(precision-1) ]
// Returns the smallest approximation available that's at least as precise
// as the passed precision (places after decimal point), i.e. Floor[ log2(precision) ] + 1
func (c constApproximation) withPrecision(precision int32) Decimal {
i := 0

if precision > 1 {
precision--
// 0, 1, 2-3, 4-7, 8-15, 16-31, 32-63
// 0, 1, 2, 3, 4, 5, 6

if precision >= 1 {
i++
}

Expand Down
34 changes: 34 additions & 0 deletions const_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package decimal

import "testing"

func TestConstApproximation(t *testing.T) {
for _, testCase := range []struct {
Const string
Precision int32
ExpectedApproximation string
}{
{"2.3025850929940456840179914546", 0, "2"},
{"2.3025850929940456840179914546", 1, "2.3"},
{"2.3025850929940456840179914546", 3, "2.302"},
{"2.3025850929940456840179914546", 5, "2.302585"},
{"2.3025850929940456840179914546", 10, "2.302585092994045"},
{"2.3025850929940456840179914546", 100, "2.3025850929940456840179914546"},
{"2.3025850929940456840179914546", -1, "2"},
{"2.3025850929940456840179914546", -5, "2"},
{"3.14159265359", 0, "3"},
{"3.14159265359", 1, "3.1"},
{"3.14159265359", 2, "3.141"},
{"3.14159265359", 4, "3.1415926"},
{"3.14159265359", 13, "3.14159265359"},
} {
ca := newConstApproximation(testCase.Const)
expected, _ := NewFromString(testCase.ExpectedApproximation)

approximation := ca.withPrecision(testCase.Precision)

if approximation.Cmp(expected) != 0 {
t.Errorf("expected approximation %s, got %s - for const with %s precision %d", testCase.ExpectedApproximation, approximation.String(), testCase.Const, testCase.Precision)
}
}
}
2 changes: 1 addition & 1 deletion decimal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2774,7 +2774,7 @@ func TestDecimal_Ln(t *testing.T) {
{"839101.0351", 25, "13.6400864014410013994397240"},
{"839101.0351094726488848490572028502", 50, "13.64008640145229044389152437468283605382056561604272"},
{"5023583755703750094849.03519358513093500275017501750602739169823", 25, "49.9684305274348922267409953"},
{"5023583755703750094849.03519358513093500275017501750602739169823", -1, "40.0"},
{"5023583755703750094849.03519358513093500275017501750602739169823", -1, "50.0"},
} {
d, _ := NewFromString(testCase.Dec)
expected, _ := NewFromString(testCase.Expected)
Expand Down

0 comments on commit 0f927d0

Please sign in to comment.