github.com/attic-labs/noms@v0.0.0-20210827224422-e5fa29d95e8b/samples/go/csv/csv-import/perf_test.go (about)

     1  // Copyright 2016 Attic Labs, Inc. All rights reserved.
     2  // Licensed under the Apache License, version 2.0:
     3  // http://www.apache.org/licenses/LICENSE-2.0
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"io"
    10  	"os"
    11  	"os/exec"
    12  	pp "path"
    13  	"testing"
    14  
    15  	"github.com/attic-labs/noms/go/perf/suite"
    16  	"github.com/attic-labs/noms/go/types"
    17  	"github.com/attic-labs/noms/samples/go/csv"
    18  	humanize "github.com/dustin/go-humanize"
    19  	"github.com/stretchr/testify/assert"
    20  )
    21  
    22  // CSV perf suites require the testdata directory to be checked out at $GOPATH/src/github.com/attic-labs/testdata (i.e. ../testdata relative to the noms directory).
    23  
    24  type perfSuite struct {
    25  	suite.PerfSuite
    26  	csvImportExe string
    27  }
    28  
    29  func (s *perfSuite) SetupSuite() {
    30  	// Trick the temp file logic into creating a unique path for the csv-import binary.
    31  	f := s.TempFile()
    32  	f.Close()
    33  	os.Remove(f.Name())
    34  
    35  	s.csvImportExe = f.Name()
    36  	err := exec.Command("go", "build", "-o", s.csvImportExe, "github.com/attic-labs/noms/samples/go/csv/csv-import").Run()
    37  	assert.NoError(s.T, err)
    38  }
    39  
    40  func (s *perfSuite) Test01ImportSfCrimeBlobFromTestdata() {
    41  	assert := s.NewAssert()
    42  
    43  	files := s.OpenGlob(s.Testdata, "sf-crime", "2016-07-28.*")
    44  	defer s.CloseGlob(files)
    45  
    46  	blob := types.NewBlob(s.Database, files...)
    47  	fmt.Fprintf(s.W, "\tsf-crime is %s\n", humanize.Bytes(blob.Len()))
    48  
    49  	ds := s.Database.GetDataset("sf-crime/raw")
    50  	_, err := s.Database.CommitValue(ds, blob)
    51  	assert.NoError(err)
    52  }
    53  
    54  func (s *perfSuite) Test02ImportSfCrimeCSVFromBlob() {
    55  	s.execCsvImportExe("sf-crime")
    56  }
    57  
    58  func (s *perfSuite) Test03ImportSfRegisteredBusinessesFromBlobAsMap() {
    59  	assert := s.NewAssert()
    60  
    61  	files := s.OpenGlob(s.Testdata, "sf-registered-businesses", "2016-07-25.csv")
    62  	defer s.CloseGlob(files)
    63  
    64  	blob := types.NewBlob(s.Database, files...)
    65  	fmt.Fprintf(s.W, "\tsf-reg-bus is %s\n", humanize.Bytes(blob.Len()))
    66  
    67  	ds := s.Database.GetDataset("sf-reg-bus/raw")
    68  	_, err := s.Database.CommitValue(ds, blob)
    69  	assert.NoError(err)
    70  
    71  	s.execCsvImportExe("sf-reg-bus", "--dest-type", "map:0")
    72  }
    73  
    74  func (s *perfSuite) Test04ImportSfRegisteredBusinessesFromBlobAsMultiKeyMap() {
    75  	s.execCsvImportExe("sf-reg-bus", "--dest-type", "map:Zip_Code,Business_Start_Date")
    76  }
    77  
    78  func (s *perfSuite) execCsvImportExe(dsName string, args ...string) {
    79  	assert := s.NewAssert()
    80  
    81  	blobSpec := fmt.Sprintf("%s::%s/raw.value", s.DatabaseSpec, dsName)
    82  	destSpec := fmt.Sprintf("%s::%s", s.DatabaseSpec, dsName)
    83  	args = append(args, "-p", blobSpec, destSpec)
    84  	importCmd := exec.Command(s.csvImportExe, args...)
    85  	importCmd.Stdout = s.W
    86  	importCmd.Stderr = os.Stderr
    87  
    88  	assert.NoError(importCmd.Run())
    89  }
    90  
    91  func (s *perfSuite) TestParseSfCrime() {
    92  	assert := s.NewAssert()
    93  
    94  	files := s.OpenGlob(pp.Join(s.Testdata, "sf-crime", "2016-07-28.*"))
    95  	defer s.CloseGlob(files)
    96  
    97  	reader := csv.NewCSVReader(io.MultiReader(files...), ',')
    98  	for {
    99  		_, err := reader.Read()
   100  		if err != nil {
   101  			assert.Equal(io.EOF, err)
   102  			break
   103  		}
   104  	}
   105  }
   106  
   107  func TestPerf(t *testing.T) {
   108  	suite.Run("csv-import", t, &perfSuite{})
   109  }