github.com/gogo/protobuf@v1.3.2/test/packed/packed_test.go

// Protocol Buffers for Go with Gadgets
//
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
// http://github.com/gogo/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package packed

import (
	"bytes"
	"fmt"
	math_rand "math/rand"
	"runtime"
	"testing"
	"time"
	"unsafe"

	"github.com/gogo/protobuf/proto"
)

func BenchmarkVarintIssue436withCount(b *testing.B) {
	var arraySizes = []struct {
		name  string
		value int64
	}{
		{"2^0_ints", 1 << 0},
		{"2^1_ints", 1 << 1},
		{"2^2_ints", 1 << 2},
		{"2^3_ints", 1 << 3},
		{"2^4_ints", 1 << 4},
		{"2^8_ints", 1 << 8},
		{"2^16_ints", 1 << 16},
		{"2^20_ints", 1 << 20},
		{"2^24_ints", 1 << 24},
	}

	var varintSizes = []struct {
		name  string
		value int64
	}{
		{"max_int 2^7-4", 1<<7 - 4},
		{"max_int 2^15-4", 1<<15 - 4},
		{"max_int 2^31-4", 1<<31 - 4},
		{"max_int 2^63-4", 1<<63 - 4},
	}

	for _, arraySize := range arraySizes {
		for _, varintSize := range varintSizes {
			seed := time.Now().UnixNano()
			rng := math_rand.New(math_rand.NewSource(seed))
			buf := make([]int64, arraySize.value)
			for j := range buf {
				buf[j] = rng.Int63n(varintSize.value)
			}

			b.Run(arraySize.name+", "+varintSize.name, func(b *testing.B) {
				msg := &NinRepNative{
					Field8: buf,
				}

				data, err := proto.Marshal(msg)
				if err != nil {
					b.Fatal(err)
				}

				normalmsg := &NinRepNative{}

				for i := 0; i < b.N; i++ {
					err = proto.Unmarshal(data, normalmsg)
					if err != nil {
						b.Fatal(err)
					}
				}
			})
		}
	}
}

func TestVarintIssue436(t *testing.T) {
	n := 1 << 22 // Makes for 32 MiB

	m := &runtime.MemStats{}

	msgNormal := &NinRepNative{
		Field8: make([]int64, n),
	}
	dataNormal, err := proto.Marshal(msgNormal)
	if err != nil {
		t.Fatal(err)
	}

	normalmsg := &NinRepNative{}
	runtime.ReadMemStats(m)
	beforeNormal := m.TotalAlloc
	err = proto.Unmarshal(dataNormal, normalmsg)
	runtime.ReadMemStats(m)
	afterNormal := m.TotalAlloc
	if err != nil {
		t.Fatal(err)
	}

	msgPacked := &NinRepPackedNative{
		Field8: make([]int64, n),
	}
	dataPacked, err := proto.Marshal(msgPacked)
	if err != nil {
		t.Fatal(err)
	}

	packedmsg := &NinRepPackedNative{}
	runtime.ReadMemStats(m)
	beforePacked := m.TotalAlloc
	err = proto.Unmarshal(dataPacked, packedmsg)
	runtime.ReadMemStats(m)
	afterPacked := m.TotalAlloc
	if err != nil {
		t.Fatal(err)
	}

	totalNormal := afterNormal - beforeNormal
	totalPacked := afterPacked - beforePacked
	usedRatio := float64(totalPacked) / float64(totalNormal)
	if usedRatio > 0.5 {
		t.Fatalf("unmarshaling packed msg allocated too much memory:\nnormal:\t\t%d bytes\npacked:\t\t%d bytes\nused ratio:\t%.2f%%", totalNormal, totalPacked, usedRatio*100)
	}
}

func TestIssue436(t *testing.T) {
	n := 1 << 22 // Makes for 32 MiB

	m := &runtime.MemStats{}

	msgNormal := &NinRepNative{
		Field1: make([]float64, n),
	}
	dataNormal, err := proto.Marshal(msgNormal)
	if err != nil {
		t.Fatal(err)
	}

	normalmsg := &NinRepNative{}
	runtime.ReadMemStats(m)
	beforeNormal := m.TotalAlloc
	err = proto.Unmarshal(dataNormal, normalmsg)
	runtime.ReadMemStats(m)
	afterNormal := m.TotalAlloc
	if err != nil {
		t.Fatal(err)
	}

	msgPacked := &NinRepPackedNative{
		Field1: make([]float64, n),
	}
	dataPacked, err := proto.Marshal(msgPacked)
	if err != nil {
		t.Fatal(err)
	}

	packedmsg := &NinRepPackedNative{}
	runtime.ReadMemStats(m)
	beforePacked := m.TotalAlloc
	err = proto.Unmarshal(dataPacked, packedmsg)
	runtime.ReadMemStats(m)
	afterPacked := m.TotalAlloc
	if err != nil {
		t.Fatal(err)
	}

	totalNormal := afterNormal - beforeNormal
	totalPacked := afterPacked - beforePacked
	usedRatio := float64(totalPacked) / float64(totalNormal)
	if usedRatio > 0.5 {
		t.Fatalf("unmarshaling packed msg allocated too much memory:\nnormal:\t\t%d bytes\npacked:\t\t%d bytes\nused ratio:\t%.2f%%", totalNormal, totalPacked, usedRatio*100)
	}
}

/*
https://github.com/gogo/protobuf/issues/503
preallocation was estimating length over all data vs. just that for the given field.
*/
func TestTestPackedPreallocation(t *testing.T) {
	n1 := 900
	n2 := 700

	msgPacked := &NinRepPackedNative{
		Field3: make([]int32, n1),
		Field4: make([]int64, n2),
	}

	dataPacked, err := proto.Marshal(msgPacked)
	if err != nil {
		t.Fatal(err)
	}

	packedmsg := &NinRepPackedNative{}
	err = proto.Unmarshal(dataPacked, packedmsg)
	if err != nil {
		t.Fatal(err)
	}

	if v := len(packedmsg.Field3); v != n1 {
		t.Errorf("Field3 incorrect len: %v != %v", v, n1)
	}

	if v := len(packedmsg.Field4); v != n2 {
		t.Errorf("Field4 incorrect len: %v != %v", v, n2)
	}

	if v := cap(packedmsg.Field3); v != n1 {
		t.Errorf("Field3 incorrect cap: %v != %v", v, n1)
	}

	if v := cap(packedmsg.Field4); v != n2 {
		t.Errorf("Field4 incorrect cap: %v != %v", v, n2)
	}
}

/*
https://github.com/gogo/protobuf/issues/detail?id=21
https://developers.google.com/protocol-buffers/docs/proto#options
In 2.3.0 and later, this change is safe, as parsers for packable fields will always accept both formats,
*/
func TestSafeIssue21(t *testing.T) {
	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
	msg1 := NewPopulatedNinRepNative(popr, true)
	data1, err := proto.Marshal(msg1)
	if err != nil {
		t.Fatal(err)
	}
	packedmsg := &NinRepPackedNative{}
	err = proto.Unmarshal(data1, packedmsg)
	if err != nil {
		t.Fatal(err)
	}
	if len(packedmsg.XXX_unrecognized) != 0 {
		t.Fatalf("packed msg unmarshaled unrecognized fields, even though there aren't any")
	}
	if err := VerboseEqual(msg1, packedmsg); err != nil {
		t.Fatalf("%v", err)
	}
}

func TestUnsafeIssue21(t *testing.T) {
	var bigendian uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&bigendian)) == 1 {
		t.Skip("unsafe does not work on big endian architectures")
	}
	popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
	msg1 := NewPopulatedNinRepNativeUnsafe(popr, true)
	data1, err := proto.Marshal(msg1)
	if err != nil {
		t.Fatal(err)
	}
	packedmsg := &NinRepPackedNativeUnsafe{}
	err = proto.Unmarshal(data1, packedmsg)
	if err != nil {
		t.Fatal(err)
	}
	if len(packedmsg.XXX_unrecognized) != 0 {
		t.Fatalf("packed msg unmarshaled unrecognized fields, even though there aren't any")
	}
	if err := VerboseEqualUnsafe(msg1, packedmsg); err != nil {
		t.Fatalf("%v", err)
	}
}

func VerboseEqual(this *NinRepNative, that *NinRepPackedNative) error {
	if that == nil {
		if this == nil {
			return nil
		}
		return fmt.Errorf("that == nil && this != nil")
	} else if this == nil {
		return fmt.Errorf("that != nil && this == nil")
	}

	if len(this.Field1) != len(that.Field1) {
		return fmt.Errorf("Field1 this(%v) Not Equal that(%v)", len(this.Field1), len(that.Field1))
	}
	for i := range this.Field1 {
		if this.Field1[i] != that.Field1[i] {
			return fmt.Errorf("Field1 this[%v](%v) Not Equal that[%v](%v)", i, this.Field1[i], i, that.Field1[i])
		}
	}
	if len(this.Field2) != len(that.Field2) {
		return fmt.Errorf("Field2 this(%v) Not Equal that(%v)", len(this.Field2), len(that.Field2))
	}
	for i := range this.Field2 {
		if this.Field2[i] != that.Field2[i] {
			return fmt.Errorf("Field2 this[%v](%v) Not Equal that[%v](%v)", i, this.Field2[i], i, that.Field2[i])
		}
	}
	if len(this.Field3) != len(that.Field3) {
		return fmt.Errorf("Field3 this(%v) Not Equal that(%v)", len(this.Field3), len(that.Field3))
	}
	for i := range this.Field3 {
		if this.Field3[i] != that.Field3[i] {
			return fmt.Errorf("Field3 this[%v](%v) Not Equal that[%v](%v)", i, this.Field3[i], i, that.Field3[i])
		}
	}
	if len(this.Field4) != len(that.Field4) {
		return fmt.Errorf("Field4 this(%v) Not Equal that(%v)", len(this.Field4), len(that.Field4))
	}
	for i := range this.Field4 {
		if this.Field4[i] != that.Field4[i] {
			return fmt.Errorf("Field4 this[%v](%v) Not Equal that[%v](%v)", i, this.Field4[i], i, that.Field4[i])
		}
	}
	if len(this.Field5) != len(that.Field5) {
		return fmt.Errorf("Field5 this(%v) Not Equal that(%v)", len(this.Field5), len(that.Field5))
	}
	for i := range this.Field5 {
		if this.Field5[i] != that.Field5[i] {
			return fmt.Errorf("Field5 this[%v](%v) Not Equal that[%v](%v)", i, this.Field5[i], i, that.Field5[i])
		}
	}
	if len(this.Field6) != len(that.Field6) {
		return fmt.Errorf("Field6 this(%v) Not Equal that(%v)", len(this.Field6), len(that.Field6))
	}
	for i := range this.Field6 {
		if this.Field6[i] != that.Field6[i] {
			return fmt.Errorf("Field6 this[%v](%v) Not Equal that[%v](%v)", i, this.Field6[i], i, that.Field6[i])
		}
	}
	if len(this.Field7) != len(that.Field7) {
		return fmt.Errorf("Field7 this(%v) Not Equal that(%v)", len(this.Field7), len(that.Field7))
	}
	for i := range this.Field7 {
		if this.Field7[i] != that.Field7[i] {
			return fmt.Errorf("Field7 this[%v](%v) Not Equal that[%v](%v)", i, this.Field7[i], i, that.Field7[i])
		}
	}
	if len(this.Field8) != len(that.Field8) {
		return fmt.Errorf("Field8 this(%v) Not Equal that(%v)", len(this.Field8), len(that.Field8))
	}
	for i := range this.Field8 {
		if this.Field8[i] != that.Field8[i] {
			return fmt.Errorf("Field8 this[%v](%v) Not Equal that[%v](%v)", i, this.Field8[i], i, that.Field8[i])
		}
	}
	if len(this.Field9) != len(that.Field9) {
		return fmt.Errorf("Field9 this(%v) Not Equal that(%v)", len(this.Field9), len(that.Field9))
	}
	for i := range this.Field9 {
		if this.Field9[i] != that.Field9[i] {
			return fmt.Errorf("Field9 this[%v](%v) Not Equal that[%v](%v)", i, this.Field9[i], i, that.Field9[i])
		}
	}
	if len(this.Field10) != len(that.Field10) {
		return fmt.Errorf("Field10 this(%v) Not Equal that(%v)", len(this.Field10), len(that.Field10))
	}
	for i := range this.Field10 {
		if this.Field10[i] != that.Field10[i] {
			return fmt.Errorf("Field10 this[%v](%v) Not Equal that[%v](%v)", i, this.Field10[i], i, that.Field10[i])
		}
	}
	if len(this.Field11) != len(that.Field11) {
		return fmt.Errorf("Field11 this(%v) Not Equal that(%v)", len(this.Field11), len(that.Field11))
	}
	for i := range this.Field11 {
		if this.Field11[i] != that.Field11[i] {
			return fmt.Errorf("Field11 this[%v](%v) Not Equal that[%v](%v)", i, this.Field11[i], i, that.Field11[i])
		}
	}
	if len(this.Field12) != len(that.Field12) {
		return fmt.Errorf("Field12 this(%v) Not Equal that(%v)", len(this.Field12), len(that.Field12))
	}
	for i := range this.Field12 {
		if this.Field12[i] != that.Field12[i] {
			return fmt.Errorf("Field12 this[%v](%v) Not Equal that[%v](%v)", i, this.Field12[i], i, that.Field12[i])
		}
	}
	if len(this.Field13) != len(that.Field13) {
		return fmt.Errorf("Field13 this(%v) Not Equal that(%v)", len(this.Field13), len(that.Field13))
	}
	for i := range this.Field13 {
		if this.Field13[i] != that.Field13[i] {
			return fmt.Errorf("Field13 this[%v](%v) Not Equal that[%v](%v)", i, this.Field13[i], i, that.Field13[i])
		}
	}
	if !bytes.Equal(this.XXX_unrecognized, that.XXX_unrecognized) {
		return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that.XXX_unrecognized)
	}
	return nil
}

func VerboseEqualUnsafe(this *NinRepNativeUnsafe, that *NinRepPackedNativeUnsafe) error {
	if that == nil {
		if this == nil {
			return nil
		}
		return fmt.Errorf("that == nil && this != nil")
	} else if this == nil {
		return fmt.Errorf("that != nil && this == nil")
	}

	if len(this.Field1) != len(that.Field1) {
		return fmt.Errorf("Field1 this(%v) Not Equal that(%v)", len(this.Field1), len(that.Field1))
	}
	for i := range this.Field1 {
		if this.Field1[i] != that.Field1[i] {
			return fmt.Errorf("Field1 this[%v](%v) Not Equal that[%v](%v)", i, this.Field1[i], i, that.Field1[i])
		}
	}
	if len(this.Field2) != len(that.Field2) {
		return fmt.Errorf("Field2 this(%v) Not Equal that(%v)", len(this.Field2), len(that.Field2))
	}
	for i := range this.Field2 {
		if this.Field2[i] != that.Field2[i] {
			return fmt.Errorf("Field2 this[%v](%v) Not Equal that[%v](%v)", i, this.Field2[i], i, that.Field2[i])
		}
	}
	if len(this.Field3) != len(that.Field3) {
		return fmt.Errorf("Field3 this(%v) Not Equal that(%v)", len(this.Field3), len(that.Field3))
	}
	for i := range this.Field3 {
		if this.Field3[i] != that.Field3[i] {
			return fmt.Errorf("Field3 this[%v](%v) Not Equal that[%v](%v)", i, this.Field3[i], i, that.Field3[i])
		}
	}
	if len(this.Field4) != len(that.Field4) {
		return fmt.Errorf("Field4 this(%v) Not Equal that(%v)", len(this.Field4), len(that.Field4))
	}
	for i := range this.Field4 {
		if this.Field4[i] != that.Field4[i] {
			return fmt.Errorf("Field4 this[%v](%v) Not Equal that[%v](%v)", i, this.Field4[i], i, that.Field4[i])
		}
	}
	if len(this.Field5) != len(that.Field5) {
		return fmt.Errorf("Field5 this(%v) Not Equal that(%v)", len(this.Field5), len(that.Field5))
	}
	for i := range this.Field5 {
		if this.Field5[i] != that.Field5[i] {
			return fmt.Errorf("Field5 this[%v](%v) Not Equal that[%v](%v)", i, this.Field5[i], i, that.Field5[i])
		}
	}
	if len(this.Field6) != len(that.Field6) {
		return fmt.Errorf("Field6 this(%v) Not Equal that(%v)", len(this.Field6), len(that.Field6))
	}
	for i := range this.Field6 {
		if this.Field6[i] != that.Field6[i] {
			return fmt.Errorf("Field6 this[%v](%v) Not Equal that[%v](%v)", i, this.Field6[i], i, that.Field6[i])
		}
	}
	if len(this.Field7) != len(that.Field7) {
		return fmt.Errorf("Field7 this(%v) Not Equal that(%v)", len(this.Field7), len(that.Field7))
	}
	for i := range this.Field7 {
		if this.Field7[i] != that.Field7[i] {
			return fmt.Errorf("Field7 this[%v](%v) Not Equal that[%v](%v)", i, this.Field7[i], i, that.Field7[i])
		}
	}
	if len(this.Field8) != len(that.Field8) {
		return fmt.Errorf("Field8 this(%v) Not Equal that(%v)", len(this.Field8), len(that.Field8))
	}
	for i := range this.Field8 {
		if this.Field8[i] != that.Field8[i] {
			return fmt.Errorf("Field8 this[%v](%v) Not Equal that[%v](%v)", i, this.Field8[i], i, that.Field8[i])
		}
	}
	if len(this.Field9) != len(that.Field9) {
		return fmt.Errorf("Field9 this(%v) Not Equal that(%v)", len(this.Field9), len(that.Field9))
	}
	for i := range this.Field9 {
		if this.Field9[i] != that.Field9[i] {
			return fmt.Errorf("Field9 this[%v](%v) Not Equal that[%v](%v)", i, this.Field9[i], i, that.Field9[i])
		}
	}
	if len(this.Field10) != len(that.Field10) {
		return fmt.Errorf("Field10 this(%v) Not Equal that(%v)", len(this.Field10), len(that.Field10))
	}
	for i := range this.Field10 {
		if this.Field10[i] != that.Field10[i] {
			return fmt.Errorf("Field10 this[%v](%v) Not Equal that[%v](%v)", i, this.Field10[i], i, that.Field10[i])
		}
	}
	if len(this.Field11) != len(that.Field11) {
		return fmt.Errorf("Field11 this(%v) Not Equal that(%v)", len(this.Field11), len(that.Field11))
	}
	for i := range this.Field11 {
		if this.Field11[i] != that.Field11[i] {
			return fmt.Errorf("Field11 this[%v](%v) Not Equal that[%v](%v)", i, this.Field11[i], i, that.Field11[i])
		}
	}
	if len(this.Field12) != len(that.Field12) {
		return fmt.Errorf("Field12 this(%v) Not Equal that(%v)", len(this.Field12), len(that.Field12))
	}
	for i := range this.Field12 {
		if this.Field12[i] != that.Field12[i] {
			return fmt.Errorf("Field12 this[%v](%v) Not Equal that[%v](%v)", i, this.Field12[i], i, that.Field12[i])
		}
	}
	if len(this.Field13) != len(that.Field13) {
		return fmt.Errorf("Field13 this(%v) Not Equal that(%v)", len(this.Field13), len(that.Field13))
	}
	for i := range this.Field13 {
		if this.Field13[i] != that.Field13[i] {
			return fmt.Errorf("Field13 this[%v](%v) Not Equal that[%v](%v)", i, this.Field13[i], i, that.Field13[i])
		}
	}
	if !bytes.Equal(this.XXX_unrecognized, that.XXX_unrecognized) {
		return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that.XXX_unrecognized)
	}
	return nil
}