Skip to content

Commit

Permalink
shortcheck: run with -race (#970)
Browse files Browse the repository at this point in the history
* shortcheck: run with -race

Signed-off-by: Alexey Ermakov <[email protected]>

* clone routes in the file watch data client as it is storing the same instances that it passes to the routing

Signed-off-by: Arpad Ryszka <[email protected]>

* add missing read lock and wait for the refresh goroutine to finish before test assertion

Signed-off-by: Arpad Ryszka <[email protected]>

* fix race in test code with legacy table tests

Signed-off-by: Arpad Ryszka <[email protected]>

* fix tests for audit logging during upgrade requests and connection checking during roundtripper retry

Signed-off-by: Arpad Ryszka <[email protected]>

* fix access log time sensitivity during testing with race

Signed-off-by: Arpad Ryszka <[email protected]>

* update timeout values for breaker tests for race testing

Signed-off-by: Arpad Ryszka <[email protected]>

* remove some time sensitive tests from under -race testing

Signed-off-by: Arpad Ryszka <[email protected]>

* fix race condition in test data client

Signed-off-by: Arpad Ryszka <[email protected]>

* adjust or disable some remaining time sensitive tests

Signed-off-by: Arpad Ryszka <[email protected]>

* exclude test writing to global var from race detection tests

Signed-off-by: Arpad Ryszka <[email protected]>

* split more time sensitive tests from race detection

Signed-off-by: Arpad Ryszka <[email protected]>

* split -race testing from shortcheck because some of the tests are not executed now with -race enabled

Signed-off-by: Arpad Ryszka <[email protected]>
  • Loading branch information
aermakov-zalando authored and szuecs committed Mar 11, 2019
1 parent 4e2b8e7 commit 7b3f470
Show file tree
Hide file tree
Showing 28 changed files with 602 additions and 415 deletions.
14 changes: 11 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ shortcheck: build check-plugins
#
for p in $(PACKAGES); do GO111MODULE=on go test -test.short -run ^Test $$p || break -1; done

check-race: build
# go test -race -test.short -run ^Test $(PACKAGES)
#
# due to vendoring and how go test ./... is not the same as go test ./a/... ./b/...
# probably can be reverted once etcd is fully mocked away for tests
#
for p in $(PACKAGES); do GO111MODULE=on go test -race -test.short -run ^Test $$p || break -1; done

check-plugins: $(TEST_PLUGINS)
GO111MODULE=on go test -run LoadPlugins

Expand Down Expand Up @@ -106,9 +114,9 @@ fmt: $(SOURCES)
check-fmt: $(SOURCES)
@if [ "$$(gofmt -d $(SOURCES))" != "" ]; then false; else true; fi

precommit: fmt build vet staticcheck shortcheck
precommit: fmt build vet staticcheck check-race shortcheck

check-precommit: check-fmt build vet staticcheck shortcheck
check-precommit: check-fmt build vet staticcheck check-race shortcheck

.coverprofile-all: $(SOURCES) $(TEST_PLUGINS)
# go list -f \
Expand Down Expand Up @@ -169,5 +177,5 @@ else ifeq ($(TRAVIS_BRANCH)_$(TRAVIS_PULL_REQUEST), master_false)
else ifeq ($(TRAVIS_BRANCH), master)
make deps check-precommit
else
make deps shortcheck check-plugins
make deps check-race shortcheck check-plugins
endif
2 changes: 1 addition & 1 deletion circuit/breaker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestConsecutiveFailures(t *testing.T) {
Type: ConsecutiveFailures,
Failures: 3,
HalfOpenRequests: 3,
Timeout: 3 * time.Millisecond,
Timeout: 15 * time.Millisecond,
}

waitTimeout := func() {
Expand Down
153 changes: 2 additions & 151 deletions circuit/registry_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
/* +build !race */

package circuit

import (
"math/rand"
"testing"
"time"
)
Expand Down Expand Up @@ -319,153 +320,3 @@ func TestIndividualIdle(t *testing.T) {
shouldBeClosed(t, "foo")
})
}

func TestRegistryFuzzy(t *testing.T) {
if testing.Short() {
t.Skip()
}

const (
hostCount = 1200
customSettingsCount = 120
concurrentRequests = 2048
requestDurationMean = 120 * time.Microsecond
requestDurationDeviation = 60 * time.Microsecond
idleTTL = time.Second
duration = 3 * time.Second
)

genHost := func() string {
const (
minHostLength = 12
maxHostLength = 36
)

h := make([]byte, minHostLength+rand.Intn(maxHostLength-minHostLength))
for i := range h {
h[i] = 'a' + byte(rand.Intn(int('z'+1-'a')))
}

return string(h)
}

hosts := make([]string, hostCount)
for i := range hosts {
hosts[i] = genHost()
}

settings := []BreakerSettings{{IdleTTL: idleTTL}}

settingsMap := make(map[string]BreakerSettings)
for _, h := range hosts {
s := BreakerSettings{
Host: h,
Type: ConsecutiveFailures,
Failures: 5,
IdleTTL: idleTTL,
}
settings = append(settings, s)
settingsMap[h] = s
}

r := NewRegistry(settings...)

// the first customSettingsCount hosts can have corresponding custom settings
customSettings := make(map[string]BreakerSettings)
for _, h := range hosts[:customSettingsCount] {
s := settingsMap[h]
s.Failures = 15
s.IdleTTL = idleTTL
customSettings[h] = s
}

var syncToken struct{}
sync := make(chan struct{}, 1)
sync <- syncToken
synced := func(f func()) {
t := <-sync
f()
sync <- t
}

replaceHostSettings := func(settings map[string]BreakerSettings, old, nu string) {
if s, ok := settings[old]; ok {
delete(settings, old)
s.Host = nu
settings[nu] = s
}
}

replaceHost := func() {
synced(func() {
i := rand.Intn(len(hosts))
old := hosts[i]
nu := genHost()
hosts[i] = nu
replaceHostSettings(settingsMap, old, nu)
replaceHostSettings(customSettings, old, nu)
})
}

stop := make(chan struct{})

getSettings := func(useCustom bool) BreakerSettings {
var s BreakerSettings
synced(func() {
if useCustom {
s = customSettings[hosts[rand.Intn(customSettingsCount)]]
return
}

s = settingsMap[hosts[rand.Intn(hostCount)]]
})

return s
}

requestDuration := func() time.Duration {
mean := float64(requestDurationMean)
deviation := float64(requestDurationDeviation)
return time.Duration(rand.NormFloat64()*deviation + mean)
}

makeRequest := func(useCustom bool) {
s := getSettings(useCustom)
b := r.Get(s)
if b.settings != s {
t.Error("invalid breaker received")
t.Log(b.settings, s)
close(stop)
}

time.Sleep(requestDuration())
}

runAgent := func() {
for {
select {
case <-stop:
return
default:
}

// 1% percent chance for getting a host replaced:
if rand.Intn(100) == 0 {
replaceHost()
}

// 3% percent of the requests is custom:
makeRequest(rand.Intn(100) < 3)
}
}

time.AfterFunc(duration, func() {
close(stop)
})

for i := 0; i < concurrentRequests; i++ {
go runAgent()
}

<-stop
}
157 changes: 157 additions & 0 deletions circuit/registryrace_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package circuit

import (
"math/rand"
"testing"
"time"
)

func TestRegistryFuzzy(t *testing.T) {
if testing.Short() {
t.Skip()
}

const (
hostCount = 1200
customSettingsCount = 120
concurrentRequests = 2048
requestDurationMean = 120 * time.Microsecond
requestDurationDeviation = 60 * time.Microsecond
idleTTL = time.Second
duration = 3 * time.Second
)

genHost := func() string {
const (
minHostLength = 12
maxHostLength = 36
)

h := make([]byte, minHostLength+rand.Intn(maxHostLength-minHostLength))
for i := range h {
h[i] = 'a' + byte(rand.Intn(int('z'+1-'a')))
}

return string(h)
}

hosts := make([]string, hostCount)
for i := range hosts {
hosts[i] = genHost()
}

settings := []BreakerSettings{{IdleTTL: idleTTL}}

settingsMap := make(map[string]BreakerSettings)
for _, h := range hosts {
s := BreakerSettings{
Host: h,
Type: ConsecutiveFailures,
Failures: 5,
IdleTTL: idleTTL,
}
settings = append(settings, s)
settingsMap[h] = s
}

r := NewRegistry(settings...)

// the first customSettingsCount hosts can have corresponding custom settings
customSettings := make(map[string]BreakerSettings)
for _, h := range hosts[:customSettingsCount] {
s := settingsMap[h]
s.Failures = 15
s.IdleTTL = idleTTL
customSettings[h] = s
}

var syncToken struct{}
sync := make(chan struct{}, 1)
sync <- syncToken
synced := func(f func()) {
t := <-sync
f()
sync <- t
}

replaceHostSettings := func(settings map[string]BreakerSettings, old, nu string) {
if s, ok := settings[old]; ok {
delete(settings, old)
s.Host = nu
settings[nu] = s
}
}

replaceHost := func() {
synced(func() {
i := rand.Intn(len(hosts))
old := hosts[i]
nu := genHost()
hosts[i] = nu
replaceHostSettings(settingsMap, old, nu)
replaceHostSettings(customSettings, old, nu)
})
}

stop := make(chan struct{})

getSettings := func(useCustom bool) BreakerSettings {
var s BreakerSettings
synced(func() {
if useCustom {
s = customSettings[hosts[rand.Intn(customSettingsCount)]]
return
}

s = settingsMap[hosts[rand.Intn(hostCount)]]
})

return s
}

requestDuration := func() time.Duration {
mean := float64(requestDurationMean)
deviation := float64(requestDurationDeviation)
return time.Duration(rand.NormFloat64()*deviation + mean)
}

makeRequest := func(useCustom bool) {
s := getSettings(useCustom)
b := r.Get(s)
if b.settings != s {
t.Error("invalid breaker received")
t.Log(b.settings, s)
close(stop)
}

time.Sleep(requestDuration())
}

runAgent := func() {
for {
select {
case <-stop:
return
default:
}

// 1% percent chance for getting a host replaced:
if rand.Intn(100) == 0 {
replaceHost()
}

// 3% percent of the requests is custom:
makeRequest(rand.Intn(100) < 3)
}
}

time.AfterFunc(duration, func() {
close(stop)
})

for i := 0; i < concurrentRequests; i++ {
go runAgent()
}

<-stop
}
2 changes: 2 additions & 0 deletions dataclients/routestring/example_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build !race

package routestring_test

import (
Expand Down
Loading

0 comments on commit 7b3f470

Please sign in to comment.