Skip to content

Commit

Permalink
test: enhance data race detection
Browse files Browse the repository at this point in the history
  • Loading branch information
liuq19 committed Jun 14, 2024
1 parent 3b2fa46 commit a2fc197
Show file tree
Hide file tree
Showing 16 changed files with 468 additions and 193 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/unit_test-linux-x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on: push
jobs:
build:
strategy:
max-parallel: 4
matrix:
go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
runs-on: [self-hosted, X64]
Expand All @@ -28,9 +29,15 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Data Race
run: |
./scripts/test_race.sh
- name: PCSP Test
run: python3 ./scripts/test_pcsp.py



- name: Unit Test
run: |
go test -race -covermode=atomic -coverprofile=coverage.txt ./...
Expand Down
35 changes: 0 additions & 35 deletions decoder/decoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
`runtime`
`runtime/debug`
`strings`
`sync`
`testing`
`time`

Expand All @@ -46,40 +45,6 @@ func TestMain(m *testing.M) {
m.Run()
}

func TestGC(t *testing.T) {
if debugSyncGC {
return
}
var w interface{}
out, err := decode(TwitterJson, &w, true)
if err != nil {
t.Fatal(err)
}
if out != len(TwitterJson) {
t.Fatal(out)
}
wg := &sync.WaitGroup{}
N := 10000
for i:=0; i<N; i++ {
wg.Add(1)
go func (wg *sync.WaitGroup) {
defer wg.Done()
var w interface{}
out, err := decode(TwitterJson, &w, true)
if err != nil {
t.Error(err)
return
}
if out != len(TwitterJson) {
t.Error(out)
return
}
runtime.GC()
}(wg)
}
wg.Wait()
}

var _BindingValue TwitterStruct

func init() {
Expand Down
31 changes: 0 additions & 31 deletions encoder/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
`runtime`
`runtime/debug`
`strconv`
`sync`
`testing`
`time`

Expand All @@ -47,36 +46,6 @@ func TestMain(m *testing.M) {
m.Run()
}

func TestGC(t *testing.T) {
if debugSyncGC {
return
}
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
n := len(out)
wg := &sync.WaitGroup{}
N := 10000
for i:=0; i<N; i++ {
wg.Add(1)
go func (wg *sync.WaitGroup, size int) {
defer wg.Done()
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Error(err)
return
}
if len(out) != size {
t.Error(len(out), size)
return
}
runtime.GC()
}(wg, n)
}
wg.Wait()
}

type sample struct {
M map[string]interface{}
S []interface{}
Expand Down
73 changes: 73 additions & 0 deletions internal/decoder/decode_norace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
//go:build !race
// +build !race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/utf8`
)

// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)
return err
}
80 changes: 80 additions & 0 deletions internal/decoder/decode_race.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
//go:build race
// +build race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/utf8`
)

func helpDetectDataRace(val interface{}) {
_, _ = json.Marshal(val)
}

// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)

/* put last to make the panic from sonic will always be caught at first */
helpDetectDataRace(val)
return err
}
47 changes: 0 additions & 47 deletions internal/decoder/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,12 @@
package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/utf8`
)

const (
Expand Down Expand Up @@ -106,49 +102,6 @@ func (self *Decoder) CheckTrailings() error {
}


// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)
return err
}

// UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
// int64 instead of as a float64.
func (self *Decoder) UseInt64() {
Expand Down
Loading

0 comments on commit a2fc197

Please sign in to comment.