github.com/TeaOSLab/EdgeNode@v1.3.8/internal/nodes/http_access_log_queue_test.go

// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.

package nodes_test

import (
	"bytes"
	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
	"github.com/TeaOSLab/EdgeNode/internal/nodes"
	"github.com/TeaOSLab/EdgeNode/internal/rpc"
	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
	_ "github.com/iwind/TeaGo/bootstrap"
	"google.golang.org/grpc/status"
	"reflect"
	"runtime"
	"runtime/debug"
	"strconv"
	"strings"
	"testing"
	"time"
	"unicode/utf8"
)

func TestHTTPAccessLogQueue_Push(t *testing.T) {
	// 发送到API
	client, err := rpc.SharedRPC()
	if err != nil {
		t.Fatal(err)
	}

	var requestId = 1_000_000

	var utf8Bytes = []byte{}
	for i := 0; i < 254; i++ {
		utf8Bytes = append(utf8Bytes, uint8(i))
	}

	//bytes = []byte("真不错")

	var accessLog = &pb.HTTPAccessLog{
		ServerId:    23,
		RequestId:   strconv.FormatInt(time.Now().Unix(), 10) + strconv.Itoa(requestId) + strconv.FormatInt(1, 10),
		NodeId:      48,
		Host:        "www.hello.com",
		RequestURI:  string(utf8Bytes),
		RequestPath: string(utf8Bytes),
		Timestamp:   time.Now().Unix(),
		Cookie:      map[string]string{"test": string(utf8Bytes)},

		Header: map[string]*pb.Strings{
			"test": {Values: []string{string(utf8Bytes)}},
		},
	}

	new(nodes.HTTPAccessLogQueue).ToValidUTF8(accessLog)

	//	logs.PrintAsJSON(accessLog)

	//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
	_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
		accessLog,
	}})
	if err != nil {
		// 这里只是为了重现错误
		t.Logf("%#v, %s", err, err.Error())

		statusErr, ok := status.FromError(err)
		if ok {
			t.Logf("%#v", statusErr)
		}
		return
	}
	t.Log("ok")
}

func TestHTTPAccessLogQueue_Push2(t *testing.T) {
	var utf8Bytes = []byte{}
	for i := 0; i < 254; i++ {
		utf8Bytes = append(utf8Bytes, uint8(i))
	}

	var accessLog = &pb.HTTPAccessLog{
		ServerId:    23,
		RequestId:   strconv.FormatInt(time.Now().Unix(), 10) + strconv.Itoa(1) + strconv.FormatInt(1, 10),
		NodeId:      48,
		Host:        "www.hello.com",
		RequestURI:  string(utf8Bytes),
		RequestPath: string(utf8Bytes),
		Timestamp:   time.Now().Unix(),
	}
	var v = reflect.Indirect(reflect.ValueOf(accessLog))
	var countFields = v.NumField()
	for i := 0; i < countFields; i++ {
		var field = v.Field(i)
		if field.Kind() == reflect.String {
			field.SetString(strings.ToValidUTF8(field.String(), ""))
		}
	}

	client, err := rpc.SharedRPC()
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
		accessLog,
	}})
	if err != nil {
		t.Fatal(err)
	}
	t.Log("ok")
}

func TestHTTPAccessLogQueue_Memory(t *testing.T) {
	if !testutils.IsSingleTesting() {
		return
	}

	testutils.StartMemoryStats(t)

	debug.SetGCPercent(10)

	var accessLogs = []*pb.HTTPAccessLog{}
	for i := 0; i < 20_000; i++ {
		accessLogs = append(accessLogs, &pb.HTTPAccessLog{
			RequestPath: "https://goedge.cn/hello/world",
		})
	}

	runtime.GC()
	_ = accessLogs

	// will not release automatically
	func() {
		var accessLogs1 = []*pb.HTTPAccessLog{}
		for i := 0; i < 2_000_000; i++ {
			accessLogs1 = append(accessLogs1, &pb.HTTPAccessLog{
				RequestPath: "https://goedge.cn/hello/world",
			})
		}
		_ = accessLogs1
	}()

	time.Sleep(5 * time.Second)
}

func TestUTF8_IsValid(t *testing.T) {
	t.Log(utf8.ValidString("abc"))

	var noneUTF8Bytes = []byte{}
	for i := 0; i < 254; i++ {
		noneUTF8Bytes = append(noneUTF8Bytes, uint8(i))
	}
	t.Log(utf8.ValidString(string(noneUTF8Bytes)))
}

func BenchmarkHTTPAccessLogQueue_ToValidUTF8(b *testing.B) {
	runtime.GOMAXPROCS(1)

	var utf8Bytes = []byte{}
	for i := 0; i < 254; i++ {
		utf8Bytes = append(utf8Bytes, uint8(i))
	}

	for i := 0; i < b.N; i++ {
		_ = bytes.ToValidUTF8(utf8Bytes, nil)
	}
}

func BenchmarkHTTPAccessLogQueue_ToValidUTF8String(b *testing.B) {
	runtime.GOMAXPROCS(1)

	var utf8Bytes = []byte{}
	for i := 0; i < 254; i++ {
		utf8Bytes = append(utf8Bytes, uint8(i))
	}

	var s = string(utf8Bytes)
	for i := 0; i < b.N; i++ {
		_ = strings.ToValidUTF8(s, "")
	}
}

func BenchmarkAppendAccessLogs(b *testing.B) {
	b.ReportAllocs()

	var stat1 = &runtime.MemStats{}
	runtime.ReadMemStats(stat1)

	const count = 20000
	var a = make([]*pb.HTTPAccessLog, 0, count)
	for i := 0; i < b.N; i++ {
		a = append(a, &pb.HTTPAccessLog{
			RequestPath: "/hello/world",
			Host:        "example.com",
			RequestBody: bytes.Repeat([]byte{'A'}, 1024),
		})
		if len(a) == count {
			a = make([]*pb.HTTPAccessLog, 0, count)
		}
	}

	_ = len(a)

	var stat2 = &runtime.MemStats{}
	runtime.ReadMemStats(stat2)
	b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
}

func BenchmarkAppendAccessLogs2(b *testing.B) {
	b.ReportAllocs()

	var stat1 = &runtime.MemStats{}
	runtime.ReadMemStats(stat1)

	const count = 20000
	var a = []*pb.HTTPAccessLog{}
	for i := 0; i < b.N; i++ {
		a = append(a, &pb.HTTPAccessLog{
			RequestPath: "/hello/world",
			Host:        "example.com",
			RequestBody: bytes.Repeat([]byte{'A'}, 1024),
		})
		if len(a) == count {
			a = []*pb.HTTPAccessLog{}
		}
	}

	_ = len(a)

	var stat2 = &runtime.MemStats{}
	runtime.ReadMemStats(stat2)
	b.Log((stat2.TotalAlloc-stat1.TotalAlloc)>>20, "MB")
}