github.com/unigraph-dev/dgraph@v1.1.1-0.20200923154953-8b52b426f765/dgraph/cmd/bulk/progress.go (about)

     1  /*
     2   * Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package bulk
    18  
    19  import (
    20  	"fmt"
    21  	"sync/atomic"
    22  	"time"
    23  
    24  	"github.com/dgraph-io/dgraph/x"
    25  )
    26  
    27  type phase int32
    28  
    29  const (
    30  	nothing phase = iota
    31  	mapPhase
    32  	reducePhase
    33  )
    34  
    35  type progress struct {
    36  	nquadCount      int64
    37  	errCount        int64
    38  	mapEdgeCount    int64
    39  	reduceEdgeCount int64
    40  	reduceKeyCount  int64
    41  
    42  	start       time.Time
    43  	startReduce time.Time
    44  
    45  	// shutdown is a bidirectional channel used to manage the stopping of the
    46  	// report goroutine. It handles both the request to stop the report
    47  	// goroutine, as well as the message back to say that the goroutine has
    48  	// stopped. The channel MUST be unbuffered for this to work.
    49  	shutdown chan struct{}
    50  
    51  	phase phase
    52  }
    53  
    54  func newProgress() *progress {
    55  	return &progress{
    56  		start:    time.Now(),
    57  		shutdown: make(chan struct{}),
    58  	}
    59  }
    60  
    61  func (p *progress) setPhase(ph phase) {
    62  	atomic.StoreInt32((*int32)(&p.phase), int32(ph))
    63  }
    64  
    65  func (p *progress) report() {
    66  	for {
    67  		select {
    68  		case <-time.After(time.Second):
    69  			p.reportOnce()
    70  		case <-p.shutdown:
    71  			p.shutdown <- struct{}{}
    72  			return
    73  		}
    74  	}
    75  }
    76  
    77  func (p *progress) reportOnce() {
    78  	mapEdgeCount := atomic.LoadInt64(&p.mapEdgeCount)
    79  	timestamp := time.Now().Format("15:04:05Z0700")
    80  	switch phase(atomic.LoadInt32((*int32)(&p.phase))) {
    81  	case nothing:
    82  	case mapPhase:
    83  		rdfCount := atomic.LoadInt64(&p.nquadCount)
    84  		errCount := atomic.LoadInt64(&p.errCount)
    85  		elapsed := time.Since(p.start)
    86  		fmt.Printf("[%s] MAP %s nquad_count:%s err_count:%s nquad_speed:%s/sec "+
    87  			"edge_count:%s edge_speed:%s/sec\n",
    88  			timestamp,
    89  			x.FixedDuration(elapsed),
    90  			niceFloat(float64(rdfCount)),
    91  			niceFloat(float64(errCount)),
    92  			niceFloat(float64(rdfCount)/elapsed.Seconds()),
    93  			niceFloat(float64(mapEdgeCount)),
    94  			niceFloat(float64(mapEdgeCount)/elapsed.Seconds()),
    95  		)
    96  	case reducePhase:
    97  		now := time.Now()
    98  		elapsed := time.Since(p.startReduce)
    99  		if p.startReduce.IsZero() {
   100  			p.startReduce = time.Now()
   101  			elapsed = time.Second
   102  		}
   103  		reduceKeyCount := atomic.LoadInt64(&p.reduceKeyCount)
   104  		reduceEdgeCount := atomic.LoadInt64(&p.reduceEdgeCount)
   105  		pct := ""
   106  		if mapEdgeCount != 0 {
   107  			pct = fmt.Sprintf("%.2f%% ", 100*float64(reduceEdgeCount)/float64(mapEdgeCount))
   108  		}
   109  		fmt.Printf("[%s] REDUCE %s %sedge_count:%s edge_speed:%s/sec "+
   110  			"plist_count:%s plist_speed:%s/sec\n",
   111  			timestamp,
   112  			x.FixedDuration(now.Sub(p.start)),
   113  			pct,
   114  			niceFloat(float64(reduceEdgeCount)),
   115  			niceFloat(float64(reduceEdgeCount)/elapsed.Seconds()),
   116  			niceFloat(float64(reduceKeyCount)),
   117  			niceFloat(float64(reduceKeyCount)/elapsed.Seconds()),
   118  		)
   119  	default:
   120  		x.AssertTruef(false, "invalid phase")
   121  	}
   122  }
   123  
   124  func (p *progress) endSummary() {
   125  	p.shutdown <- struct{}{}
   126  	<-p.shutdown
   127  
   128  	p.reportOnce()
   129  
   130  	total := x.FixedDuration(time.Since(p.start))
   131  	fmt.Printf("Total: %v\n", total)
   132  }
   133  
   134  var suffixes = [...]string{"", "k", "M", "G", "T"}
   135  
   136  func niceFloat(f float64) string {
   137  	idx := 0
   138  	for f >= 1000 {
   139  		f /= 1000
   140  		idx++
   141  	}
   142  	if idx >= len(suffixes) {
   143  		return fmt.Sprintf("%f", f)
   144  	}
   145  	suf := suffixes[idx]
   146  	switch {
   147  	case f >= 100:
   148  		return fmt.Sprintf("%.1f%s", f, suf)
   149  	case f >= 10:
   150  		return fmt.Sprintf("%.2f%s", f, suf)
   151  	default:
   152  		return fmt.Sprintf("%.3f%s", f, suf)
   153  	}
   154  }