Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use proto.Buffer API for protobuf codec and cache proto.Buffer structs #1010

Merged
merged 27 commits into from
Apr 13, 2017
Merged
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e946d53
use a global sharded pool of proto.Buffer caches in protoCodec
apolcyn Nov 17, 2016
f8e2d0b
fix goimports
apolcyn Dec 5, 2016
d9756fc
make global buffer pool index counter atomic
apolcyn Dec 9, 2016
3ab447e
hack to remove alloc in encode_len_struct
apolcyn Dec 28, 2016
3a7b0e8
remove extra slice alloc in proto codec marshal
apolcyn Dec 28, 2016
0f18cff
replce magic number for proto size field length with constant
apolcyn Dec 28, 2016
0b1ccd4
replace custom cache with sync.Pool
apolcyn Dec 30, 2016
a1e068f
remove 1 line functions in codec.go and add protoCodec microbenchmarks
apolcyn Feb 7, 2017
a21119d
add concurrent usage test for protoCodec
apolcyn Feb 7, 2017
e16c62c
fix golint.gofmt,goimport checks
apolcyn Feb 7, 2017
586bba2
fix issues in codec.go and codec_test.go
apolcyn Feb 10, 2017
510b880
use go parallel benchmark helpers
apolcyn Feb 12, 2017
34bf3db
replace proto.Codec with a guess of size needed
apolcyn Feb 12, 2017
c8c475c
update Fatalf -> Errorf in tests
apolcyn Feb 13, 2017
8431a0d
wrap proto.Buffer along with cached last size into larger struct for …
apolcyn Feb 14, 2017
f2b8177
make wrapped proto buffer only a literal
apolcyn Feb 14, 2017
225b3ae
fix style and imports
apolcyn Feb 14, 2017
f841695
move b.Run into inner function
apolcyn Feb 14, 2017
776137a
reverse micro benchmark op order to unmarshal-marshal and fix benchma…
apolcyn Feb 14, 2017
93d4020
add test for large message
apolcyn Feb 15, 2017
d30dcba
remove use of defer in codec.marshal
apolcyn Feb 22, 2017
126640b
revert recent changes to codec bencmarks
apolcyn Feb 27, 2017
18485fe
move sub-benchmarks into >= go-1.7 only file
apolcyn Feb 27, 2017
93c3e12
add commentfor marshaler and tweak benchmark subtests for easier usage
apolcyn Feb 27, 2017
5ef6c9b
move build tag for go1.7 on benchmarks to inside file
apolcyn Feb 27, 2017
42d712a
move build tag to top of file
apolcyn Feb 27, 2017
1517ac9
comment Codec, embed proto.Buffer into cached struct and add an int32…
apolcyn Mar 16, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
revert recent changes to codec bencmarks
  • Loading branch information
apolcyn committed Mar 27, 2017
commit 126640b0994f58ceadca914377cb9c024cfe4a37
42 changes: 17 additions & 25 deletions codec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,7 @@ func TestStaggeredMarshalAndUnmarshalUsingSamePool(t *testing.T) {
}
}

func setupBenchmarkProtoCodecInputs(b *testing.B) [][]byte {
codec := protoCodec{}
func setupBenchmarkProtoCodecInputs(b *testing.B) []proto.Message {
// small, arbitrary byte slices
protoBodies := [][]byte{
[]byte("one"),
Expand All @@ -154,37 +153,31 @@ func setupBenchmarkProtoCodecInputs(b *testing.B) [][]byte {
[]byte("four"),
[]byte("five"),
}

marshaled := make([][]byte, 0, len(protoBodies))
protoStructs := make([]proto.Message, 0)

for _, p := range protoBodies {
protoStruct := &codec_perf.Buffer{}
protoStruct.Body = p
marshaledBytes, err := codec.Marshal(protoStruct)
if err != nil {
b.Errorf("protoCodec.Marshal(_) returned an error")
}
marshaled = append(marshaled, marshaledBytes)
ps := &codec_perf.Buffer{}
ps.Body = p
protoStructs = append(protoStructs, ps)
}

return marshaled
return protoStructs
}

// The possible use of certain protobuf APIs like the proto.Buffer API potentially involves caching
// on our side. This can add checks around memory allocations and possible contention.
// Example run: go test -v -run=^$ -bench=BenchmarkProtoCodec -benchmem
<<<<<<< HEAD
func BenchmarkProtoCodec(b *testing.B) {
i := uint32(0)
marshaledBytes := setupBenchmarkProtoCodecInputs(b)
protoStructs := setupBenchmarkProtoCodecInputs(b)
for i < 20 {
i += 2
func(p int) {
b.Run(fmt.Sprintf("BenchmarkProtoCodec_SetParallelism(%v)", p), func(b *testing.B) {
codec := &protoCodec{}
b.SetParallelism(p)
b.RunParallel(func(pb *testing.PB) {
benchmarkProtoCodec(codec, marshaledBytes, pb, b)
benchmarkProtoCodec(codec, protoStructs, pb, b)
})
})
}(1 << i)
Expand All @@ -207,23 +200,22 @@ func benchmarkProtoCodecWithParallelism(b *testing.B, p int, protos []proto.Mess
})
}

func benchmarkProtoCodec(codec *protoCodec, marshaledBytes [][]byte, pb *testing.PB, b *testing.B) {
protoStruct := &codec_perf.Buffer{}
func benchmarkProtoCodec(codec *protoCodec, protoStructs []proto.Message, pb *testing.PB, b *testing.B) {
counter := 0
for pb.Next() {
counter++
m := marshaledBytes[counter%len(marshaledBytes)]
fastMarshalAndUnmarshal(codec, m, protoStruct, b)
ps := protoStructs[counter%len(protoStructs)]
fastMarshalAndUnmarshal(codec, ps, b)
}
}

func fastMarshalAndUnmarshal(protoCodec Codec, marshaledBytes []byte, protoStruct proto.Message, b *testing.B) {
if err := protoCodec.Unmarshal(marshaledBytes, protoStruct); err != nil {
b.Errorf("protoCodec.Unmarshal(_) returned an error")
}

_, err := protoCodec.Marshal(protoStruct)
func fastMarshalAndUnmarshal(protoCodec Codec, protoStruct proto.Message, b *testing.B) {
marshaledBytes, err := protoCodec.Marshal(protoStruct)
if err != nil {
b.Errorf("protoCodec.Marshal(_) returned an error")
}

if err := protoCodec.Unmarshal(marshaledBytes, protoStruct); err != nil {
b.Errorf("protoCodec.Unmarshal(_) returned an error")
}
}