github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/chunk_test.go (about) 1 package chunk 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/pkg/errors" 9 "github.com/prometheus/common/model" 10 "github.com/prometheus/prometheus/model/labels" 11 "github.com/stretchr/testify/require" 12 13 "github.com/grafana/loki/pkg/ingester/client" 14 "github.com/grafana/loki/pkg/logproto" 15 "github.com/grafana/loki/pkg/storage/config" 16 ) 17 18 const userID = "userID" 19 20 var labelsForDummyChunks = labels.Labels{ 21 {Name: labels.MetricName, Value: "foo"}, 22 {Name: "bar", Value: "baz"}, 23 {Name: "toms", Value: "code"}, 24 } 25 26 func dummyChunk(now model.Time) Chunk { 27 return dummyChunkFor(now, labelsForDummyChunks) 28 } 29 30 func dummyChunkForEncoding(now model.Time, metric labels.Labels, samples int) Chunk { 31 c, _ := NewForEncoding(Bigchunk) 32 chunkStart := now.Add(-time.Hour) 33 34 for i := 0; i < samples; i++ { 35 t := time.Duration(i) * 15 * time.Second 36 nc, err := c.(*bigchunk).Add(model.SamplePair{Timestamp: chunkStart.Add(t), Value: model.SampleValue(i)}) 37 if err != nil { 38 panic(err) 39 } 40 if nc != nil { 41 panic("returned chunk was not nil") 42 } 43 } 44 45 chunk := NewChunk( 46 userID, 47 client.Fingerprint(metric), 48 metric, 49 c, 50 chunkStart, 51 now, 52 ) 53 // Force checksum calculation. 54 err := chunk.Encode() 55 if err != nil { 56 panic(err) 57 } 58 return chunk 59 } 60 61 func dummyChunkFor(now model.Time, metric labels.Labels) Chunk { 62 return dummyChunkForEncoding(now, metric, 1) 63 } 64 65 func TestChunkCodec(t *testing.T) { 66 dummy := dummyChunk(model.Now()) 67 decodeContext := NewDecodeContext() 68 for i, c := range []struct { 69 chunk Chunk 70 err error 71 f func(*Chunk, []byte) 72 }{ 73 // Basic round trip 74 {chunk: dummy}, 75 76 // Checksum should fail 77 { 78 chunk: dummy, 79 err: ErrInvalidChecksum, 80 f: func(_ *Chunk, buf []byte) { buf[4]++ }, 81 }, 82 83 // Checksum should fail 84 { 85 chunk: dummy, 86 err: ErrInvalidChecksum, 87 f: func(c *Chunk, _ []byte) { c.Checksum = 123 }, 88 }, 89 90 // Metadata test should fail 91 { 92 chunk: dummy, 93 err: ErrWrongMetadata, 94 f: func(c *Chunk, _ []byte) { c.Fingerprint++ }, 95 }, 96 97 // Metadata test should fail 98 { 99 chunk: dummy, 100 err: ErrWrongMetadata, 101 f: func(c *Chunk, _ []byte) { c.UserID = "foo" }, 102 }, 103 } { 104 t.Run(fmt.Sprintf("[%d]", i), func(t *testing.T) { 105 err := c.chunk.Encode() 106 require.NoError(t, err) 107 encoded, err := c.chunk.Encoded() 108 require.NoError(t, err) 109 110 s := config.SchemaConfig{ 111 Configs: []config.PeriodConfig{ 112 { 113 From: config.DayTime{Time: 0}, 114 Schema: "v11", 115 RowShards: 16, 116 }, 117 }, 118 } 119 120 have, err := ParseExternalKey(userID, s.ExternalKey(c.chunk.ChunkRef)) 121 require.NoError(t, err) 122 123 buf := make([]byte, len(encoded)) 124 copy(buf, encoded) 125 if c.f != nil { 126 c.f(&have, buf) 127 } 128 129 err = have.Decode(decodeContext, buf) 130 require.Equal(t, c.err, errors.Cause(err)) 131 132 if c.err == nil { 133 require.Equal(t, have.encoded, c.chunk.encoded) 134 } 135 }) 136 } 137 } 138 139 const fixedTimestamp = model.Time(1557654321000) 140 141 func TestChunkDecodeBackwardsCompatibility(t *testing.T) { 142 // lets build a new chunk same as what was built using code at commit b1777a50ab19 143 c, _ := NewForEncoding(Bigchunk) 144 nc, err := c.(*bigchunk).Add(model.SamplePair{Timestamp: fixedTimestamp, Value: 0}) 145 require.NoError(t, err) 146 require.Equal(t, nil, nc, "returned chunk should be nil") 147 148 chunk := NewChunk( 149 userID, 150 client.Fingerprint(labelsForDummyChunks), 151 labelsForDummyChunks, 152 c, 153 fixedTimestamp.Add(-time.Hour), 154 fixedTimestamp, 155 ) 156 // Force checksum calculation. 157 require.NoError(t, chunk.Encode()) 158 159 // Chunk encoded using code at commit b1777a50ab19 160 rawData := []byte("\x00\x00\x00\xb7\xff\x06\x00\x00sNaPpY\x01\xa5\x00\x00\xfcB\xb4\xc9{\"fingerprint\":18245339272195143978,\"userID\":\"userID\",\"from\":1557650721,\"through\":1557654321,\"metric\":{\"__name__\":\"foo\",\"bar\":\"baz\",\"toms\":\"code\"},\"encoding\":0}\n\x00\x00\x00\x15\x01\x00\x11\x00\x00\x01\xd0\xdd\xf5\xb6\xd5Z\x00\x00\x00\x00\x00\x00\x00\x00\x00") 161 decodeContext := NewDecodeContext() 162 have, err := ParseExternalKey(userID, "userID/fd3477666dacf92a:16aab37c8e8:16aab6eb768:70b431bb") 163 require.NoError(t, err) 164 require.NoError(t, have.Decode(decodeContext, rawData)) 165 want := chunk 166 // We can't just compare these two chunks, since the Bigchunk internals are different on construction and read-in. 167 // Compare the serialised version instead 168 require.NoError(t, have.Encode()) 169 require.NoError(t, want.Encode()) 170 haveEncoded, _ := have.Encoded() 171 wantEncoded, _ := want.Encoded() 172 require.Equal(t, haveEncoded, wantEncoded) 173 174 s := config.SchemaConfig{ 175 Configs: []config.PeriodConfig{ 176 { 177 From: config.DayTime{Time: 0}, 178 Schema: "v11", 179 RowShards: 16, 180 }, 181 }, 182 } 183 require.Equal(t, s.ExternalKey(have.ChunkRef), s.ExternalKey(want.ChunkRef)) 184 } 185 186 func TestParseExternalKey(t *testing.T) { 187 for _, c := range []struct { 188 key string 189 chunk Chunk 190 err error 191 }{ 192 {key: userID + "/2:270d8f00:270d8f00:f84c5745", chunk: Chunk{ 193 ChunkRef: logproto.ChunkRef{ 194 UserID: userID, 195 Fingerprint: uint64(2), 196 From: model.Time(655200000), 197 Through: model.Time(655200000), 198 Checksum: 4165752645, 199 }, 200 }}, 201 202 {key: userID + "/2/270d8f00:270d8f00:f84c5745", chunk: Chunk{ 203 ChunkRef: logproto.ChunkRef{ 204 UserID: userID, 205 Fingerprint: uint64(2), 206 From: model.Time(655200000), 207 Through: model.Time(655200000), 208 Checksum: 4165752645, 209 }, 210 }}, 211 212 {key: "invalidUserID/2:270d8f00:270d8f00:f84c5745", chunk: Chunk{}, err: ErrWrongMetadata}, 213 } { 214 chunk, err := ParseExternalKey(userID, c.key) 215 require.Equal(t, c.err, errors.Cause(err)) 216 require.Equal(t, c.chunk, chunk) 217 } 218 } 219 220 // BenchmarkLabels is a real example from Kubernetes' embedded cAdvisor metrics, lightly obfuscated 221 var BenchmarkLabels = labels.Labels{ 222 {Name: model.MetricNameLabel, Value: "container_cpu_usage_seconds_total"}, 223 {Name: "beta_kubernetes_io_arch", Value: "amd64"}, 224 {Name: "beta_kubernetes_io_instance_type", Value: "c3.somesize"}, 225 {Name: "beta_kubernetes_io_os", Value: "linux"}, 226 {Name: "container_name", Value: "some-name"}, 227 {Name: "cpu", Value: "cpu01"}, 228 {Name: "failure_domain_beta_kubernetes_io_region", Value: "somewhere-1"}, 229 {Name: "failure_domain_beta_kubernetes_io_zone", Value: "somewhere-1b"}, 230 {Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"}, 231 {Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"}, 232 {Name: "instance", Value: "ip-111-11-1-11.ec2.internal"}, 233 {Name: "job", Value: "kubernetes-cadvisor"}, 234 {Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"}, 235 {Name: "monitor", Value: "prod"}, 236 {Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"}, 237 {Name: "namespace", Value: "kube-system"}, 238 {Name: "pod_name", Value: "some-other-name-5j8s8"}, 239 } 240 241 func benchmarkChunk(now model.Time) Chunk { 242 return dummyChunkFor(now, BenchmarkLabels) 243 } 244 245 func BenchmarkEncode(b *testing.B) { 246 chunk := dummyChunk(model.Now()) 247 248 b.ResetTimer() 249 250 for i := 0; i < b.N; i++ { 251 chunk.encoded = nil 252 err := chunk.Encode() 253 require.NoError(b, err) 254 } 255 } 256 257 func BenchmarkDecode1(b *testing.B) { benchmarkDecode(b, 1) } 258 func BenchmarkDecode100(b *testing.B) { benchmarkDecode(b, 100) } 259 func BenchmarkDecode10000(b *testing.B) { benchmarkDecode(b, 10000) } 260 261 func benchmarkDecode(b *testing.B, batchSize int) { 262 chunk := benchmarkChunk(model.Now()) 263 err := chunk.Encode() 264 require.NoError(b, err) 265 buf, err := chunk.Encoded() 266 require.NoError(b, err) 267 268 b.ResetTimer() 269 270 for i := 0; i < b.N; i++ { 271 decodeContext := NewDecodeContext() 272 b.StopTimer() 273 chunks := make([]Chunk, batchSize) 274 // Copy across the metadata so the check works out ok 275 for j := 0; j < batchSize; j++ { 276 chunks[j] = chunk 277 chunks[j].Metric = nil 278 chunks[j].Data = nil 279 } 280 b.StartTimer() 281 for j := 0; j < batchSize; j++ { 282 err := chunks[j].Decode(decodeContext, buf) 283 require.NoError(b, err) 284 } 285 } 286 } 287 288 func TestChunkKeys(t *testing.T) { 289 for _, tc := range []struct { 290 name string 291 chunk Chunk 292 schemaCfg config.SchemaConfig 293 }{ 294 { 295 name: "Legacy key (pre-checksum)", 296 chunk: Chunk{ 297 ChunkRef: logproto.ChunkRef{ 298 Fingerprint: 100, 299 UserID: "fake", 300 From: model.TimeFromUnix(1000), 301 Through: model.TimeFromUnix(5000), 302 Checksum: 12345, 303 }, 304 }, 305 schemaCfg: config.SchemaConfig{ 306 Configs: []config.PeriodConfig{ 307 { 308 From: config.DayTime{Time: 0}, 309 Schema: "v11", 310 RowShards: 16, 311 }, 312 }, 313 }, 314 }, 315 { 316 name: "Newer key (post-v12)", 317 chunk: Chunk{ 318 ChunkRef: logproto.ChunkRef{ 319 Fingerprint: 100, 320 UserID: "fake", 321 From: model.TimeFromUnix(1000), 322 Through: model.TimeFromUnix(5000), 323 Checksum: 12345, 324 }, 325 }, 326 schemaCfg: config.SchemaConfig{ 327 Configs: []config.PeriodConfig{ 328 { 329 From: config.DayTime{Time: 0}, 330 Schema: "v12", 331 RowShards: 16, 332 }, 333 }, 334 }, 335 }, 336 } { 337 t.Run(tc.name, func(t *testing.T) { 338 key := tc.schemaCfg.ExternalKey(tc.chunk.ChunkRef) 339 newChunk, err := ParseExternalKey("fake", key) 340 require.NoError(t, err) 341 require.Equal(t, tc.chunk, newChunk) 342 require.Equal(t, key, tc.schemaCfg.ExternalKey(newChunk.ChunkRef)) 343 }) 344 } 345 } 346 347 func BenchmarkParseNewerExternalKey(b *testing.B) { 348 benchmarkParseExternalKey(b, "fake/57f628c7f6d57aad/162c699f000:162c69a07eb:eb242d99") 349 } 350 351 func BenchmarkParseNewExternalKey(b *testing.B) { 352 benchmarkParseExternalKey(b, "fake/57f628c7f6d57aad:162c699f000:162c69a07eb:eb242d99") 353 } 354 355 func BenchmarkParseLegacyExternalKey(b *testing.B) { 356 benchmarkParseExternalKey(b, "2:1484661279394:1484664879394") 357 } 358 359 func BenchmarkRootParseNewExternalKey(b *testing.B) { 360 for i := 0; i < b.N; i++ { 361 _, err := parseNewExternalKey("fake", "fake/57f628c7f6d57aad:162c699f000:162c69a07eb:eb242d99") 362 require.NoError(b, err) 363 } 364 } 365 366 func BenchmarkRootParseNewerExternalKey(b *testing.B) { 367 for i := 0; i < b.N; i++ { 368 _, err := parseNewerExternalKey("fake", "fake/57f628c7f6d57aad/162c699f000:162c69a07eb:eb242d99") 369 require.NoError(b, err) 370 } 371 } 372 373 func benchmarkParseExternalKey(b *testing.B, key string) { 374 for i := 0; i < b.N; i++ { 375 _, err := ParseExternalKey("fake", key) 376 require.NoError(b, err) 377 } 378 }