github.com/aclisp/heapster@v0.19.2-0.20160613100040-51756f899a96/metrics/sinks/hawkular/driver_test.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package hawkular 16 17 import ( 18 "encoding/json" 19 "fmt" 20 "io/ioutil" 21 "net/http" 22 "net/http/httptest" 23 "net/url" 24 "strings" 25 "sync" 26 "testing" 27 "time" 28 29 "github.com/hawkular/hawkular-client-go/metrics" 30 "k8s.io/heapster/metrics/core" 31 32 assert "github.com/stretchr/testify/require" 33 ) 34 35 func dummySink() *hawkularSink { 36 return &hawkularSink{ 37 reg: make(map[string]*metrics.MetricDefinition), 38 models: make(map[string]*metrics.MetricDefinition), 39 } 40 } 41 42 func TestDescriptorTransform(t *testing.T) { 43 44 hSink := dummySink() 45 46 ld := core.LabelDescriptor{ 47 Key: "k1", 48 Description: "d1", 49 } 50 smd := core.MetricDescriptor{ 51 Name: "test/metric/1", 52 Units: core.UnitsBytes, 53 ValueType: core.ValueInt64, 54 Type: core.MetricGauge, 55 Labels: []core.LabelDescriptor{ld}, 56 } 57 58 md := hSink.descriptorToDefinition(&smd) 59 60 assert.Equal(t, smd.Name, md.Id) 61 assert.Equal(t, 3, len(md.Tags)) // descriptorTag, unitsTag, typesTag, k1 62 63 assert.Equal(t, smd.Units.String(), md.Tags[unitsTag]) 64 assert.Equal(t, "d1", md.Tags["k1_description"]) 65 66 smd.Type = core.MetricCumulative 67 68 md = hSink.descriptorToDefinition(&smd) 69 assert.Equal(t, md.Type, metrics.Counter) 70 } 71 72 func TestMetricTransform(t *testing.T) { 73 hSink := dummySink() 74 75 l := make(map[string]string) 76 l["spooky"] = "notvisible" 77 l[core.LabelHostname.Key] = "localhost" 78 l[core.LabelHostID.Key] = "localhost" 79 l[core.LabelContainerName.Key] = "docker" 80 l[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 81 l[core.LabelNodename.Key] = "myNode" 82 83 metricName := "test/metric/1" 84 labeledMetricNameA := "test/labeledmetric/A" 85 labeledMetricNameB := "test/labeledmetric/B" 86 87 metricSet := core.MetricSet{ 88 Labels: l, 89 MetricValues: map[string]core.MetricValue{ 90 metricName: { 91 ValueType: core.ValueInt64, 92 MetricType: core.MetricGauge, 93 IntValue: 123456, 94 }, 95 }, 96 LabeledMetrics: []core.LabeledMetric{ 97 { 98 Name: labeledMetricNameA, 99 Labels: map[string]string{ 100 core.LabelResourceID.Key: "XYZ", 101 }, 102 MetricValue: core.MetricValue{ 103 MetricType: core.MetricGauge, 104 FloatValue: 124.456, 105 }, 106 }, 107 { 108 Name: labeledMetricNameB, 109 MetricValue: core.MetricValue{ 110 MetricType: core.MetricGauge, 111 FloatValue: 454, 112 }, 113 }, 114 }, 115 } 116 117 metricSet.LabeledMetrics = append(metricSet.LabeledMetrics, metricValueToLabeledMetric(metricSet.MetricValues)...) 118 119 now := time.Now() 120 // 121 m, err := hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[2], now) 122 assert.NoError(t, err) 123 124 assert.Equal(t, fmt.Sprintf("%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], 125 metricSet.Labels[core.LabelPodId.Key], metricName), m.Id) 126 127 assert.Equal(t, 1, len(m.Data)) 128 _, ok := m.Data[0].Value.(float64) 129 assert.True(t, ok, "Value should have been converted to float64") 130 131 delete(l, core.LabelPodId.Key) 132 133 // 134 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[2], now) 135 assert.NoError(t, err) 136 137 assert.Equal(t, fmt.Sprintf("%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], metricSet.Labels[core.LabelNodename.Key], metricName), m.Id) 138 139 // 140 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 141 assert.NoError(t, err) 142 143 assert.Equal(t, fmt.Sprintf("%s/%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], 144 metricSet.Labels[core.LabelNodename.Key], labeledMetricNameA, 145 metricSet.LabeledMetrics[0].Labels[core.LabelResourceID.Key]), m.Id) 146 147 // 148 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[1], now) 149 assert.NoError(t, err) 150 assert.Equal(t, fmt.Sprintf("%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], 151 metricSet.Labels[core.LabelNodename.Key], labeledMetricNameB), m.Id) 152 } 153 154 func TestMetricIds(t *testing.T) { 155 hSink := dummySink() 156 157 l := make(map[string]string) 158 l["spooky"] = "notvisible" 159 l[core.LabelHostname.Key] = "localhost" 160 l[core.LabelHostID.Key] = "localhost" 161 l[core.LabelContainerName.Key] = "docker" 162 l[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 163 l[core.LabelNodename.Key] = "myNode" 164 l[core.LabelNamespaceName.Key] = "myNamespace" 165 166 metricName := "test/metric/nodeType" 167 168 metricSet := core.MetricSet{ 169 Labels: l, 170 MetricValues: map[string]core.MetricValue{ 171 metricName: { 172 ValueType: core.ValueInt64, 173 MetricType: core.MetricGauge, 174 IntValue: 123456, 175 }, 176 }, 177 } 178 metricSet.LabeledMetrics = metricValueToLabeledMetric(metricSet.MetricValues) 179 180 now := time.Now() 181 // 182 m, err := hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 183 assert.NoError(t, err) 184 assert.Equal(t, fmt.Sprintf("%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], metricSet.Labels[core.LabelPodId.Key], metricName), m.Id) 185 186 // 187 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypeNode 188 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 189 assert.NoError(t, err) 190 assert.Equal(t, fmt.Sprintf("%s/%s/%s", "machine", metricSet.Labels[core.LabelNodename.Key], metricName), m.Id) 191 192 // 193 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypePod 194 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 195 assert.NoError(t, err) 196 assert.Equal(t, fmt.Sprintf("%s/%s/%s", core.MetricSetTypePod, metricSet.Labels[core.LabelPodId.Key], metricName), m.Id) 197 198 // 199 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypePodContainer 200 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 201 assert.NoError(t, err) 202 assert.Equal(t, fmt.Sprintf("%s/%s/%s", metricSet.Labels[core.LabelContainerName.Key], metricSet.Labels[core.LabelPodId.Key], metricName), m.Id) 203 204 // 205 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypeSystemContainer 206 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 207 assert.NoError(t, err) 208 assert.Equal(t, fmt.Sprintf("%s/%s/%s/%s", core.MetricSetTypeSystemContainer, metricSet.Labels[core.LabelContainerName.Key], metricSet.Labels[core.LabelPodId.Key], metricName), m.Id) 209 210 // 211 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypeCluster 212 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 213 assert.NoError(t, err) 214 assert.Equal(t, fmt.Sprintf("%s/%s", core.MetricSetTypeCluster, metricName), m.Id) 215 216 // 217 metricSet.Labels[core.LabelMetricSetType.Key] = core.MetricSetTypeNamespace 218 m, err = hSink.pointToLabeledMetricHeader(&metricSet, metricSet.LabeledMetrics[0], now) 219 assert.NoError(t, err) 220 assert.Equal(t, fmt.Sprintf("%s/%s/%s", core.MetricSetTypeNamespace, metricSet.Labels[core.LabelNamespaceName.Key], metricName), m.Id) 221 222 } 223 224 func TestRecentTest(t *testing.T) { 225 hSink := dummySink() 226 227 modelT := make(map[string]string) 228 229 id := "test.name" 230 modelT[descriptorTag] = "d" 231 modelT[groupTag] = id 232 modelT["hep"+descriptionTag] = "n" 233 234 model := metrics.MetricDefinition{ 235 Id: id, 236 Tags: modelT, 237 } 238 239 liveT := make(map[string]string) 240 for k, v := range modelT { 241 liveT[k] = v 242 } 243 244 live := metrics.MetricDefinition{ 245 Id: "test/" + id, 246 Tags: liveT, 247 } 248 249 assert.True(t, hSink.recent(&live, &model), "Tags are equal, live is newest") 250 251 delete(liveT, "hep"+descriptionTag) 252 live.Tags = liveT 253 254 assert.False(t, hSink.recent(&live, &model), "Tags are not equal, live isn't recent") 255 256 } 257 258 func TestParseFiltersErrors(t *testing.T) { 259 _, err := parseFilters([]string{"(missingcommand)"}) 260 assert.Error(t, err) 261 262 _, err = parseFilters([]string{"missingeverything"}) 263 assert.Error(t, err) 264 265 _, err = parseFilters([]string{"labelstart:^missing$)"}) 266 assert.Error(t, err) 267 268 _, err = parseFilters([]string{"label(endmissing"}) 269 assert.Error(t, err) 270 271 _, err = parseFilters([]string{"label(wrongsyntax)"}) 272 assert.Error(t, err) 273 } 274 275 // Integration tests 276 func integSink(uri string) (*hawkularSink, error) { 277 278 u, err := url.Parse(uri) 279 if err != nil { 280 return nil, err 281 } 282 283 sink := &hawkularSink{ 284 uri: u, 285 } 286 if err = sink.init(); err != nil { 287 return nil, err 288 } 289 290 return sink, nil 291 } 292 293 // Test that Definitions is called for Gauges & Counters 294 // Test that we have single registered model 295 // Test that the tags for metric is updated.. 296 func TestRegister(t *testing.T) { 297 m := &sync.Mutex{} 298 definitionsCalled := make(map[string]bool) 299 updateTagsCalled := false 300 301 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 302 m.Lock() 303 defer m.Unlock() 304 w.Header().Set("Content-Type", "application/json") 305 306 if strings.Contains(r.RequestURI, "metrics?type=") { 307 typ := r.RequestURI[strings.Index(r.RequestURI, "type=")+5:] 308 definitionsCalled[typ] = true 309 if typ == "gauge" { 310 fmt.Fprintln(w, `[{ "id": "test.create.gauge.1", "tenantId": "test-heapster", "type": "gauge", "tags": { "descriptor_name": "test/metric/1" } }]`) 311 } else { 312 w.WriteHeader(http.StatusNoContent) 313 } 314 } else if strings.Contains(r.RequestURI, "/tags") && r.Method == "PUT" { 315 updateTagsCalled = true 316 // assert.True(t, strings.Contains(r.RequestURI, "k1:d1"), "Tag k1 was not updated with value d1") 317 defer r.Body.Close() 318 b, err := ioutil.ReadAll(r.Body) 319 assert.NoError(t, err) 320 321 tags := make(map[string]string) 322 err = json.Unmarshal(b, &tags) 323 assert.NoError(t, err) 324 325 _, kt1 := tags["k1_description"] 326 _, dt := tags["descriptor_name"] 327 328 assert.True(t, kt1, "k1_description tag is missing") 329 assert.True(t, dt, "descriptor_name is missing") 330 331 w.WriteHeader(http.StatusOK) 332 } 333 })) 334 defer s.Close() 335 336 hSink, err := integSink(s.URL + "?tenant=test-heapster") 337 assert.NoError(t, err) 338 339 md := make([]core.MetricDescriptor, 0, 1) 340 ld := core.LabelDescriptor{ 341 Key: "k1", 342 Description: "d1", 343 } 344 smd := core.MetricDescriptor{ 345 Name: "test/metric/1", 346 Units: core.UnitsBytes, 347 ValueType: core.ValueInt64, 348 Type: core.MetricGauge, 349 Labels: []core.LabelDescriptor{ld}, 350 } 351 smdg := core.MetricDescriptor{ 352 Name: "test/metric/2", 353 Units: core.UnitsBytes, 354 ValueType: core.ValueFloat, 355 Type: core.MetricCumulative, 356 Labels: []core.LabelDescriptor{}, 357 } 358 359 md = append(md, smd, smdg) 360 361 err = hSink.Register(md) 362 assert.NoError(t, err) 363 364 assert.Equal(t, 2, len(hSink.models)) 365 assert.Equal(t, 1, len(hSink.reg)) 366 367 assert.True(t, definitionsCalled["gauge"], "Gauge definitions were not fetched") 368 assert.True(t, definitionsCalled["counter"], "Counter definitions were not fetched") 369 assert.True(t, updateTagsCalled, "Updating outdated tags was not called") 370 } 371 372 // Store timeseries with both gauges and cumulatives 373 func TestStoreTimeseries(t *testing.T) { 374 m := &sync.Mutex{} 375 ids := make([]string, 0, 2) 376 calls := make([]string, 0, 2) 377 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 378 m.Lock() 379 defer m.Unlock() 380 calls = append(calls, r.RequestURI) 381 w.Header().Set("Content-Type", "application/json") 382 383 typ := r.RequestURI[strings.Index(r.RequestURI, "hawkular/metrics/")+17:] 384 typ = typ[:len(typ)-5] 385 386 switch typ { 387 case "counters": 388 assert.Equal(t, "test-label", r.Header.Get("Hawkular-Tenant")) 389 break 390 case "gauges": 391 assert.Equal(t, "test-heapster", r.Header.Get("Hawkular-Tenant")) 392 break 393 default: 394 assert.FailNow(t, "Unrecognized type "+typ) 395 } 396 397 defer r.Body.Close() 398 b, err := ioutil.ReadAll(r.Body) 399 assert.NoError(t, err) 400 401 mH := []metrics.MetricHeader{} 402 err = json.Unmarshal(b, &mH) 403 assert.NoError(t, err) 404 405 assert.Equal(t, 1, len(mH)) 406 407 ids = append(ids, mH[0].Id) 408 })) 409 defer s.Close() 410 411 hSink, err := integSink(s.URL + "?tenant=test-heapster&labelToTenant=projectId") 412 assert.NoError(t, err) 413 414 l := make(map[string]string) 415 l["projectId"] = "test-label" 416 l[core.LabelContainerName.Key] = "test-container" 417 l[core.LabelPodId.Key] = "test-podid" 418 419 lg := make(map[string]string) 420 lg[core.LabelContainerName.Key] = "test-container" 421 lg[core.LabelPodId.Key] = "test-podid" 422 423 metricSet1 := core.MetricSet{ 424 Labels: l, 425 MetricValues: map[string]core.MetricValue{ 426 "test/metric/1": { 427 ValueType: core.ValueInt64, 428 MetricType: core.MetricCumulative, 429 IntValue: 123456, 430 }, 431 }, 432 } 433 434 metricSet2 := core.MetricSet{ 435 Labels: lg, 436 MetricValues: map[string]core.MetricValue{ 437 "test/metric/2": { 438 ValueType: core.ValueFloat, 439 MetricType: core.MetricGauge, 440 FloatValue: 123.456, 441 }, 442 }, 443 } 444 445 data := core.DataBatch{ 446 Timestamp: time.Now(), 447 MetricSets: map[string]*core.MetricSet{ 448 "pod1": &metricSet1, 449 "pod2": &metricSet2, 450 }, 451 } 452 453 hSink.ExportData(&data) 454 assert.Equal(t, 2, len(calls)) 455 assert.Equal(t, 2, len(ids)) 456 457 assert.NotEqual(t, ids[0], ids[1]) 458 } 459 460 func TestUserPass(t *testing.T) { 461 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 462 w.Header().Set("X-Authorization", r.Header.Get("Authorization")) 463 auth := strings.SplitN(r.Header.Get("Authorization"), " ", 2) 464 if len(auth) != 2 || auth[0] != "Basic" { 465 assert.FailNow(t, "Could not find Basic authentication") 466 } 467 assert.True(t, len(auth[1]) > 0) 468 w.WriteHeader(http.StatusNoContent) 469 })) 470 defer s.Close() 471 472 hSink, err := integSink(s.URL + "?user=tester&pass=hidden") 473 assert.NoError(t, err) 474 475 // md := make([]core.MetricDescriptor, 0, 1) 476 ld := core.LabelDescriptor{ 477 Key: "k1", 478 Description: "d1", 479 } 480 smd := core.MetricDescriptor{ 481 Name: "test/metric/1", 482 Units: core.UnitsBytes, 483 ValueType: core.ValueInt64, 484 Type: core.MetricGauge, 485 Labels: []core.LabelDescriptor{ld}, 486 } 487 err = hSink.Register([]core.MetricDescriptor{smd}) 488 assert.NoError(t, err) 489 } 490 491 func TestFiltering(t *testing.T) { 492 m := &sync.Mutex{} 493 mH := []metrics.MetricHeader{} 494 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 495 m.Lock() 496 defer m.Unlock() 497 if strings.Contains(r.RequestURI, "data") { 498 defer r.Body.Close() 499 b, err := ioutil.ReadAll(r.Body) 500 assert.NoError(t, err) 501 502 err = json.Unmarshal(b, &mH) 503 assert.NoError(t, err) 504 } 505 })) 506 defer s.Close() 507 508 hSink, err := integSink(s.URL + "?filter=label(namespace_id:^$)&filter=label(container_name:^[/system.slice/|/user.slice].*)&filter=name(remove*)") 509 assert.NoError(t, err) 510 511 l := make(map[string]string) 512 l["namespace_id"] = "123" 513 l["container_name"] = "/system.slice/-.mount" 514 l[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 515 516 l2 := make(map[string]string) 517 l2["namespace_id"] = "123" 518 l2["container_name"] = "/system.slice/dbus.service" 519 l2[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 520 521 l3 := make(map[string]string) 522 l3["namespace_id"] = "123" 523 l3[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 524 525 l4 := make(map[string]string) 526 l4["namespace_id"] = "" 527 l4[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 528 529 l5 := make(map[string]string) 530 l5["namespace_id"] = "123" 531 l5[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" 532 533 metricSet1 := core.MetricSet{ 534 Labels: l, 535 MetricValues: map[string]core.MetricValue{ 536 "/system.slice/-.mount//cpu/limit": { 537 ValueType: core.ValueInt64, 538 MetricType: core.MetricCumulative, 539 IntValue: 123456, 540 }, 541 }, 542 } 543 544 metricSet2 := core.MetricSet{ 545 Labels: l2, 546 MetricValues: map[string]core.MetricValue{ 547 "/system.slice/dbus.service//cpu/usage": { 548 ValueType: core.ValueInt64, 549 MetricType: core.MetricCumulative, 550 IntValue: 123456, 551 }, 552 }, 553 } 554 555 metricSet3 := core.MetricSet{ 556 Labels: l3, 557 MetricValues: map[string]core.MetricValue{ 558 "test/metric/1": { 559 ValueType: core.ValueInt64, 560 MetricType: core.MetricCumulative, 561 IntValue: 123456, 562 }, 563 }, 564 } 565 566 metricSet4 := core.MetricSet{ 567 Labels: l4, 568 MetricValues: map[string]core.MetricValue{ 569 "test/metric/1": { 570 ValueType: core.ValueInt64, 571 MetricType: core.MetricCumulative, 572 IntValue: 123456, 573 }, 574 }, 575 } 576 577 metricSet5 := core.MetricSet{ 578 Labels: l5, 579 MetricValues: map[string]core.MetricValue{ 580 "removeme": { 581 ValueType: core.ValueInt64, 582 MetricType: core.MetricCumulative, 583 IntValue: 123456, 584 }, 585 }, 586 } 587 588 data := core.DataBatch{ 589 Timestamp: time.Now(), 590 MetricSets: map[string]*core.MetricSet{ 591 "pod1": &metricSet1, 592 "pod2": &metricSet2, 593 "pod3": &metricSet3, 594 "pod4": &metricSet4, 595 "pod5": &metricSet5, 596 }, 597 } 598 hSink.ExportData(&data) 599 600 assert.Equal(t, 1, len(mH)) 601 } 602 603 func TestBatchingTimeseries(t *testing.T) { 604 total := 1000 605 m := &sync.Mutex{} 606 ids := make([]string, 0, total) 607 calls := 0 608 609 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 610 m.Lock() 611 defer m.Unlock() 612 613 w.Header().Set("Content-Type", "application/json") 614 615 defer r.Body.Close() 616 b, err := ioutil.ReadAll(r.Body) 617 assert.NoError(t, err) 618 619 mH := []metrics.MetricHeader{} 620 err = json.Unmarshal(b, &mH) 621 assert.NoError(t, err) 622 623 for _, v := range mH { 624 ids = append(ids, v.Id) 625 } 626 627 calls++ 628 })) 629 defer s.Close() 630 631 hSink, err := integSink(s.URL + "?tenant=test-heapster&labelToTenant=projectId&batchSize=20&concurrencyLimit=5") 632 assert.NoError(t, err) 633 634 l := make(map[string]string) 635 l["projectId"] = "test-label" 636 l[core.LabelContainerName.Key] = "test-container" 637 l[core.LabelPodId.Key] = "test-podid" 638 639 metrics := make(map[string]core.MetricValue) 640 for i := 0; i < total; i++ { 641 id := fmt.Sprintf("test/metric/%d", i) 642 metrics[id] = core.MetricValue{ 643 ValueType: core.ValueInt64, 644 MetricType: core.MetricCumulative, 645 IntValue: 123 * int64(i), 646 } 647 } 648 649 metricSet := core.MetricSet{ 650 Labels: l, 651 MetricValues: metrics, 652 } 653 654 data := core.DataBatch{ 655 Timestamp: time.Now(), 656 MetricSets: map[string]*core.MetricSet{ 657 "pod1": &metricSet, 658 }, 659 } 660 661 hSink.ExportData(&data) 662 assert.Equal(t, total, len(ids)) 663 assert.Equal(t, calls, 50) 664 665 // Verify that all ids are unique 666 newIds := make(map[string]bool) 667 for _, v := range ids { 668 if newIds[v] { 669 t.Errorf("Key %s was duplicate", v) 670 } 671 newIds[v] = true 672 } 673 }