github.com/grailbio/bigslice@v0.0.0-20230519005545-30c4c12152ad/scan_test.go (about)

     1  // Copyright 2019 GRAIL, Inc. All rights reserved.
     2  // Use of this source code is governed by the Apache 2.0
     3  // license that can be found in the LICENSE file.
     4  
     5  package bigslice_test
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io"
    11  	"io/ioutil"
    12  	"strconv"
    13  	"testing"
    14  
    15  	"github.com/grailbio/bigslice"
    16  	"github.com/grailbio/bigslice/slicetest"
    17  )
    18  
    19  func TestScanReader(t *testing.T) {
    20  	const (
    21  		N      = 1000
    22  		Nshard = 13
    23  	)
    24  	var b bytes.Buffer
    25  	for i := 0; i < N; i++ {
    26  		fmt.Fprint(&b, i, "\n")
    27  	}
    28  	slice := bigslice.ScanReader(Nshard, func() (io.ReadCloser, error) {
    29  		return ioutil.NopCloser(bytes.NewReader(b.Bytes())), nil
    30  	})
    31  	slice = bigslice.Map(slice, func(s string) (struct{}, int) {
    32  		i, _ := strconv.ParseInt(s, 10, 64)
    33  		return struct{}{}, int(i)
    34  	})
    35  	slice = bigslice.Reduce(slice, func(a, e int) int { return a + e })
    36  	slice = bigslice.Map(slice, func(k struct{}, v int) int { return v })
    37  	assertEqual(t, slice, false, []int{499500})
    38  }
    39  
    40  func ExampleScanReader() {
    41  	var b bytes.Buffer
    42  	for i := 0; i < 10; i++ {
    43  		fmt.Fprint(&b, i, "\n")
    44  	}
    45  	slice := bigslice.ScanReader(2, func() (io.ReadCloser, error) {
    46  		return ioutil.NopCloser(bytes.NewReader(b.Bytes())), nil
    47  	})
    48  	slicetest.Print(slice)
    49  	// Output:
    50  	// 0
    51  	// 1
    52  	// 2
    53  	// 3
    54  	// 4
    55  	// 5
    56  	// 6
    57  	// 7
    58  	// 8
    59  	// 9
    60  }