Skip to content

Commit

Permalink
test: remove testify from benchmarking code (#4711)
Browse files Browse the repository at this point in the history 
test: Remove `require.NoError` and similar functions which skew
benchmark results.
test: Use `b.Loop()` to simplify benchmarks and reset timers properly.
refactor: Use `range` over integers to simplify code.
  • Loading branch information
@romshark
romshark authored Feb 25, 2025
1 parent 90da24d commit 32ca823
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 46 deletions.
4 changes: 2 additions & 2 deletions acceptance/router_benchmark/brload/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,12 @@ func run(cmd *cobra.Command) int {
numPkt := 0
for time.Since(begin) < testDuration {
// we break every 1000 batches to check the time
for i := 0; i < 1000; i++ {
for range 1000 {
// Rotate through flowIDs. We patch it directly into the SCION header of the packet. The
// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no
// more than 64K flows).
for j := 0; j < batchSize; j++ {
for j := range batchSize {
binary.BigEndian.PutUint16(allPkts[j][44:46], uint16(numPkt%int(numStreams)))
numPkt++
}
Expand Down
46 changes: 27 additions & 19 deletions pkg/slayers/scion_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Copyright 2020 Anapaya Systems
// Copyright 2025 SCION Association
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -450,52 +451,59 @@ func TestUnkownAddrType(t *testing.T) {
func BenchmarkDecodePreallocNoParse(b *testing.B) {
raw := prepRawPacket(b)
s := &slayers.SCION{}
for i := 0; i < b.N; i++ {
err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback)
require.NoError(b, err)
for b.Loop() {
if err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
}

func BenchmarkDecodeNoPreallocNoParse(b *testing.B) {
raw := prepRawPacket(b)
for i := 0; i < b.N; i++ {
for b.Loop() {
s := &slayers.SCION{}
err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback)
require.NoError(b, err)
if err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
}

func BenchmarkDecodePreallocFull(b *testing.B) {
raw := prepRawPacket(b)
s := &slayers.SCION{}
for i := 0; i < b.N; i++ {
err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback)
require.NoError(b, err)
for b.Loop() {
if err := s.DecodeFromBytes(raw, gopacket.NilDecodeFeedback); err != nil {
b.Fatalf("unexpected error: %v", err)
}
p := s.Path.(*scion.Raw)
_, err = p.ToDecoded()
require.NoError(b, err)
if _, err := p.ToDecoded(); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
}

func BenchmarkSerializeReuseBuffer(b *testing.B) {
s := prepPacket(b, slayers.L4UDP)
buffer := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{FixLengths: true}
for i := 0; i < b.N; i++ {
err := s.SerializeTo(buffer, opts)
require.NoError(b, err)
err = buffer.Clear()
require.NoError(b, err)
for b.Loop() {
if err := s.SerializeTo(buffer, opts); err != nil {
b.Fatalf("unexpected error: %v", err)
}
if err := buffer.Clear(); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
}

func BenchmarkSerializeNoReuseBuffer(b *testing.B) {
s := prepPacket(b, slayers.L4UDP)
opts := gopacket.SerializeOptions{FixLengths: true}
for i := 0; i < b.N; i++ {
for b.Loop() {
buffer := gopacket.NewSerializeBuffer()
err := s.SerializeTo(buffer, opts)
require.NoError(b, err)
if err := s.SerializeTo(buffer, opts); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
}

Expand Down
11 changes: 6 additions & 5 deletions pkg/slayers/slayers_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Copyright 2020 Anapaya Systems
// Copyright 2025 SCION Association
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -453,7 +454,7 @@ func TestPacketDecodeIsInverseOfSerialize(t *testing.T) {
func BenchmarkDecodeEager(b *testing.B) {
raw := xtest.MustReadFromFile(b, rawUDPPktFilename)

for i := 0; i < b.N; i++ {
for b.Loop() {
gopacket.NewPacket(raw, slayers.LayerTypeSCION, gopacket.Default)
}
}
Expand All @@ -470,7 +471,7 @@ func BenchmarkDecodeLayerParser(b *testing.B) {
slayers.LayerTypeSCION, &scn, &hbh, &e2e, &udp, &scmp, &pld,
)
decoded := []gopacket.LayerType{}
for i := 0; i < b.N; i++ {
for b.Loop() {
if err := parser.DecodeLayers(raw, &decoded); err != nil {
b.Fatalf("error: %v\n", err)
}
Expand All @@ -489,7 +490,7 @@ func BenchmarkDecodeLayerParserExtn(b *testing.B) {
slayers.LayerTypeSCION, &scn, &hbh, &e2e, &udp, &scmp, &pld,
)
decoded := []gopacket.LayerType{}
for i := 0; i < b.N; i++ {
for b.Loop() {
if err := parser.DecodeLayers(raw, &decoded); err != nil {
b.Fatalf("error: %v\n", err)
}
Expand All @@ -508,7 +509,7 @@ func BenchmarkDecodeLayerParserExtnSkipper(b *testing.B) {
slayers.LayerTypeSCION, &scn, &hbh, &e2e, &udp, &scmp, &pld,
)
decoded := []gopacket.LayerType{}
for i := 0; i < b.N; i++ {
for b.Loop() {
if err := parser.DecodeLayers(raw, &decoded); err != nil {
b.Fatalf("error: %v\n", err)
}
Expand All @@ -517,7 +518,7 @@ func BenchmarkDecodeLayerParserExtnSkipper(b *testing.B) {

func mkPayload(plen int) []byte {
b := make([]byte, plen)
for i := 0; i < plen; i++ {
for i := range b {
b[i] = uint8(i % 256)
}
return b
Expand Down
15 changes: 7 additions & 8 deletions private/file/periodicview_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Copyright 2021 Anapaya Systems
// Copyright 2025 SCION Association
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,9 +29,7 @@ import (
"github.com/scionproto/scion/private/file"
)

var (
update = xtest.UpdateGoldenFiles()
)
var update = xtest.UpdateGoldenFiles()

const (
// fileName points to the PEM test file used for both some tests and benchmarks. Only
Expand All @@ -48,7 +47,7 @@ func TestPeriodicViewWithParser(t *testing.T) {
if *update {
b, err := scrypto.EncodePEMSymmetricKey(key)
require.NoError(t, err)
err = os.WriteFile(fileName, b, 0644)
err = os.WriteFile(fileName, b, 0o644)
require.NoError(t, err)
}

Expand All @@ -68,7 +67,7 @@ func TestPeriodicViewWithoutParser(t *testing.T) {
f := "testdata/data.raw"
b := []byte{1, 2, 3, 4}
if *update {
err := os.WriteFile(f, b, 0644)
err := os.WriteFile(f, b, 0o644)
require.NoError(t, err)
}
view := &file.PeriodicView{
Expand All @@ -86,7 +85,7 @@ func TestPeriodicViewTwoReaders(t *testing.T) {
f := "testdata/two.raw"
b := []byte{1, 2, 3, 4}
if *update {
err := os.WriteFile(f, b, 0644)
err := os.WriteFile(f, b, 0o644)
require.NoError(t, err)
}
viewOne := &file.PeriodicView{
Expand Down Expand Up @@ -125,7 +124,7 @@ func TestPeriodicViewMultipleReads(t *testing.T) {
f := "testdata/multiple.raw"
b := []byte{1, 2, 3, 4}
if *update {
err := os.WriteFile(f, b, 0644)
err := os.WriteFile(f, b, 0o644)
require.NoError(t, err)
}
view := &file.PeriodicView{
Expand Down Expand Up @@ -211,7 +210,7 @@ func BenchmarkPeriodicView(b *testing.B) {
}

func benchmarkView(b *testing.B, view file.View) {
for n := 0; n < b.N; n++ {
for b.Loop() {
_, err := view.Get()
if err != nil {
b.Fatalf("Error: %v", err)
Expand Down
23 changes: 11 additions & 12 deletions private/trust/verifier_bench_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Copyright 2020 Anapaya Systems
// Copyright 2025 SCION Association
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -47,7 +48,7 @@ func BenchmarkConcurrent10(b *testing.B) {
_, err = trust.LoadChains(context.Background(), filepath.Join(dir, "certs"), db)
require.NoError(b, err)

signer := loadTrustSigner(b, dir, db)
signer := loadTrustSigner(b, dir)

associated := [][]byte{make([]byte, 300), make([]byte, 300), make([]byte, 300)}
msg, err := signer.Sign(context.Background(), make([]byte, 5000), associated...)
Expand All @@ -62,12 +63,10 @@ func BenchmarkConcurrent10(b *testing.B) {
},
}

b.ResetTimer()

var wg sync.WaitGroup
for i := 0; i < b.N; i++ {
for b.Loop() {
wg.Add(10)
for j := 0; j < 10; j++ {
for range 10 {
go func() {
defer wg.Done()
_, err := verifier.Verify(context.Background(), msg, associated...)
Expand All @@ -90,7 +89,7 @@ func BenchmarkConcurrentCache10(b *testing.B) {
_, err = trust.LoadChains(context.Background(), filepath.Join(dir, "certs"), db)
require.NoError(b, err)

signer := loadTrustSigner(b, dir, db)
signer := loadTrustSigner(b, dir)

associated := [][]byte{make([]byte, 300), make([]byte, 300), make([]byte, 300)}
msg, err := signer.Sign(context.Background(), make([]byte, 5000), associated...)
Expand All @@ -106,23 +105,23 @@ func BenchmarkConcurrentCache10(b *testing.B) {
Cache: cache.New(time.Second, time.Minute),
}

b.ResetTimer()

var wg sync.WaitGroup
for i := 0; i < b.N; i++ {
for b.Loop() {
wg.Add(10)
for j := 0; j < 10; j++ {
for range 10 {
go func() {
defer wg.Done()
_, err := verifier.Verify(context.Background(), msg, associated...)
require.NoError(b, err)
if err != nil {
panic(err)
}
}()
}
wg.Wait()
}
}

func loadTrustSigner(b *testing.B, dir string, db trust.DB) trust.Signer {
func loadTrustSigner(b *testing.B, dir string) trust.Signer {
raw, err := os.ReadFile(filepath.Join(dir, "ISD1/ASff00_0_110/crypto/as/cp-as.key"))
require.NoError(b, err)
block, _ := pem.Decode(raw)
Expand Down

0 comments on commit 32ca823

Please sign in to comment.