github.com/pbberlin/tools@v0.0.0-20160910141205-7aa5421c2169/net/http/dedup/t2_dedup_test.go (about)

     1  // +build dedup2
     2  // go test -tags=dedup2
     3  
     4  package dedup
     5  
     6  import (
     7  	"testing"
     8  	"time"
     9  
    10  	"appengine/aetest"
    11  
    12  	"github.com/pbberlin/tools/net/http/domclean2"
    13  	"github.com/pbberlin/tools/net/http/loghttp"
    14  )
    15  
    16  func Test2(t *testing.T) {
    17  
    18  	start := time.Now()
    19  
    20  	lg, b := loghttp.BuffLoggerUniversal(nil, nil)
    21  	_ = b
    22  	// closureOverBuf := func(bUnused *bytes.Buffer) {
    23  	// 	loghttp.Pf(nil, nil, b.String())
    24  	// }
    25  	// defer closureOverBuf(b) // the argument is ignored,
    26  
    27  	var c aetest.Context
    28  	if false {
    29  		var err error
    30  		c, err = aetest.NewContext(nil)
    31  		lg(err)
    32  		if err != nil {
    33  			return
    34  		}
    35  		defer c.Close()
    36  	}
    37  	fs := GetFS(c, 2)
    38  
    39  	lg("took1 %4.2v secs", time.Now().Sub(start).Seconds())
    40  
    41  	least3Files := FetchAndDecodeJSON(nil, URLs[0], "", lg, fs)
    42  
    43  	lg("took2 %4.2v secs", time.Now().Sub(start).Seconds())
    44  
    45  	doc := Dedup(least3Files, lg, fs)
    46  
    47  	fNamer := domclean2.FileNamer(logDir, 0)
    48  	fNamer() // first call yields key
    49  	fsPerm := GetFS(c, 2)
    50  	fileDump(lg, fsPerm, doc, fNamer, "_fin.html")
    51  
    52  	pf("MapSimiliarCompares: %v SimpleCompares: %v LevenstheinComp: %v\n", breakMapsTooDistinct, appliedLevenshtein, appliedCompare)
    53  	pf("Finish\n")
    54  
    55  }