github.com/Cloud-Foundations/Dominator@v0.3.4/lib/stringutil/dedup.go (about)

     1  package stringutil
     2  
     3  func deduplicateList(list []string, makeIfEmpty bool) (
     4  	[]string, map[string]struct{}) {
     5  	if len(list) < 1 && !makeIfEmpty {
     6  		return list, nil
     7  	}
     8  	copiedEntries := make(map[string]struct{}, len(list))
     9  	outputList := make([]string, 0, len(list))
    10  	for _, entry := range list {
    11  		if _, ok := copiedEntries[entry]; !ok {
    12  			outputList = append(outputList, entry)
    13  			copiedEntries[entry] = struct{}{}
    14  		}
    15  	}
    16  	if len(outputList) == len(list) {
    17  		return list, copiedEntries
    18  	}
    19  	return outputList, copiedEntries
    20  }
    21  
    22  func (d *StringDeduplicator) clear() {
    23  	if d.lock {
    24  		d.mutex.Lock()
    25  		defer d.mutex.Unlock()
    26  	}
    27  	d.mapping = make(map[string]string)
    28  	d.statistics = StringDuplicationStatistics{}
    29  }
    30  
    31  func (d *StringDeduplicator) deDuplicate(str string) string {
    32  	if str == "" {
    33  		return ""
    34  	}
    35  	if d.lock {
    36  		d.mutex.Lock()
    37  		defer d.mutex.Unlock()
    38  	}
    39  	if cached, ok := d.mapping[str]; ok {
    40  		d.statistics.DuplicateBytes += uint64(len(str))
    41  		d.statistics.DuplicateStrings++
    42  		return cached
    43  	} else {
    44  		d.mapping[str] = str
    45  		d.statistics.UniqueBytes += uint64(len(str))
    46  		d.statistics.UniqueStrings++
    47  		return str
    48  	}
    49  }
    50  
    51  func (d *StringDeduplicator) deleteUnregistered() {
    52  	if d.lock {
    53  		d.mutex.Lock()
    54  		defer d.mutex.Unlock()
    55  	}
    56  	for str := range d.mapping {
    57  		if _, registered := d.registered[str]; !registered {
    58  			delete(d.mapping, str)
    59  		}
    60  	}
    61  	d.registered = nil
    62  }
    63  
    64  func (d *StringDeduplicator) getStatistics() StringDuplicationStatistics {
    65  	if d.lock {
    66  		d.mutex.Lock()
    67  		defer d.mutex.Unlock()
    68  	}
    69  	return d.statistics
    70  }
    71  
    72  func (d *StringDeduplicator) register(str string) {
    73  	if d.lock {
    74  		d.mutex.Lock()
    75  		defer d.mutex.Unlock()
    76  	}
    77  	if d.registered == nil {
    78  		d.registered = make(map[string]struct{})
    79  	}
    80  	d.registered[str] = struct{}{}
    81  }