github.com/andybalholm/brotli@v1.0.6/block_splitter.go (about)

     1  package brotli
     2  
     3  /* Copyright 2013 Google Inc. All Rights Reserved.
     4  
     5     Distributed under MIT license.
     6     See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
     7  */
     8  
     9  /* Block split point selection utilities. */
    10  
    11  type blockSplit struct {
    12  	num_types          uint
    13  	num_blocks         uint
    14  	types              []byte
    15  	lengths            []uint32
    16  	types_alloc_size   uint
    17  	lengths_alloc_size uint
    18  }
    19  
    20  const (
    21  	kMaxLiteralHistograms        uint    = 100
    22  	kMaxCommandHistograms        uint    = 50
    23  	kLiteralBlockSwitchCost      float64 = 28.1
    24  	kCommandBlockSwitchCost      float64 = 13.5
    25  	kDistanceBlockSwitchCost     float64 = 14.6
    26  	kLiteralStrideLength         uint    = 70
    27  	kCommandStrideLength         uint    = 40
    28  	kSymbolsPerLiteralHistogram  uint    = 544
    29  	kSymbolsPerCommandHistogram  uint    = 530
    30  	kSymbolsPerDistanceHistogram uint    = 544
    31  	kMinLengthForBlockSplitting  uint    = 128
    32  	kIterMulForRefining          uint    = 2
    33  	kMinItersForRefining         uint    = 100
    34  )
    35  
    36  func countLiterals(cmds []command) uint {
    37  	var total_length uint = 0
    38  	/* Count how many we have. */
    39  
    40  	for i := range cmds {
    41  		total_length += uint(cmds[i].insert_len_)
    42  	}
    43  
    44  	return total_length
    45  }
    46  
    47  func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) {
    48  	var pos uint = 0
    49  	var from_pos uint = offset & mask
    50  	for i := range cmds {
    51  		var insert_len uint = uint(cmds[i].insert_len_)
    52  		if from_pos+insert_len > mask {
    53  			var head_size uint = mask + 1 - from_pos
    54  			copy(literals[pos:], data[from_pos:][:head_size])
    55  			from_pos = 0
    56  			pos += head_size
    57  			insert_len -= head_size
    58  		}
    59  
    60  		if insert_len > 0 {
    61  			copy(literals[pos:], data[from_pos:][:insert_len])
    62  			pos += insert_len
    63  		}
    64  
    65  		from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
    66  	}
    67  }
    68  
    69  func myRand(seed *uint32) uint32 {
    70  	/* Initial seed should be 7. In this case, loop length is (1 << 29). */
    71  	*seed *= 16807
    72  
    73  	return *seed
    74  }
    75  
    76  func bitCost(count uint) float64 {
    77  	if count == 0 {
    78  		return -2.0
    79  	} else {
    80  		return fastLog2(count)
    81  	}
    82  }
    83  
    84  const histogramsPerBatch = 64
    85  
    86  const clustersPerBatch = 16
    87  
    88  func initBlockSplit(self *blockSplit) {
    89  	self.num_types = 0
    90  	self.num_blocks = 0
    91  	self.types = self.types[:0]
    92  	self.lengths = self.lengths[:0]
    93  	self.types_alloc_size = 0
    94  	self.lengths_alloc_size = 0
    95  }
    96  
    97  func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
    98  	{
    99  		var literals_count uint = countLiterals(cmds)
   100  		var literals []byte = make([]byte, literals_count)
   101  
   102  		/* Create a continuous array of literals. */
   103  		copyLiteralsToByteArray(cmds, data, pos, mask, literals)
   104  
   105  		/* Create the block split on the array of literals.
   106  		   Literal histograms have alphabet size 256. */
   107  		splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
   108  
   109  		literals = nil
   110  	}
   111  	{
   112  		var insert_and_copy_codes []uint16 = make([]uint16, len(cmds))
   113  		/* Compute prefix codes for commands. */
   114  
   115  		for i := range cmds {
   116  			insert_and_copy_codes[i] = cmds[i].cmd_prefix_
   117  		}
   118  
   119  		/* Create the block split on the array of command prefixes. */
   120  		splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
   121  
   122  		/* TODO: reuse for distances? */
   123  
   124  		insert_and_copy_codes = nil
   125  	}
   126  	{
   127  		var distance_prefixes []uint16 = make([]uint16, len(cmds))
   128  		var j uint = 0
   129  		/* Create a continuous array of distance prefixes. */
   130  
   131  		for i := range cmds {
   132  			var cmd *command = &cmds[i]
   133  			if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
   134  				distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
   135  				j++
   136  			}
   137  		}
   138  
   139  		/* Create the block split on the array of distance prefixes. */
   140  		splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
   141  
   142  		distance_prefixes = nil
   143  	}
   144  }