huffman_code.go

Documentation: compress/flate

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package flate
     6  
     7  import (
     8  	"math"
     9  	"math/bits"
    10  	"sort"
    11  )
    12  
    13  // hcode is a huffman code with a bit code and bit length.
    14  type hcode struct {
    15  	code, len uint16
    16  }
    17  
    18  type huffmanEncoder struct {
    19  	codes     []hcode
    20  	freqcache []literalNode
    21  	bitCount  [17]int32
    22  	lns       byLiteral // stored to avoid repeated allocation in generate
    23  	lfs       byFreq    // stored to avoid repeated allocation in generate
    24  }
    25  
    26  type literalNode struct {
    27  	literal uint16
    28  	freq    int32
    29  }
    30  
    31  // A levelInfo describes the state of the constructed tree for a given depth.
    32  type levelInfo struct {
    33  	// Our level.  for better printing
    34  	level int32
    35  
    36  	// The frequency of the last node at this level
    37  	lastFreq int32
    38  
    39  	// The frequency of the next character to add to this level
    40  	nextCharFreq int32
    41  
    42  	// The frequency of the next pair (from level below) to add to this level.
    43  	// Only valid if the "needed" value of the next lower level is 0.
    44  	nextPairFreq int32
    45  
    46  	// The number of chains remaining to generate for this level before moving
    47  	// up to the next level
    48  	needed int32
    49  }
    50  
    51  // set sets the code and length of an hcode.
    52  func (h *hcode) set(code uint16, length uint16) {
    53  	h.len = length
    54  	h.code = code
    55  }
    56  
    57  func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
    58  
    59  func newHuffmanEncoder(size int) *huffmanEncoder {
    60  	return &huffmanEncoder{codes: make([]hcode, size)}
    61  }
    62  
    63  // Generates a HuffmanCode corresponding to the fixed literal table
    64  func generateFixedLiteralEncoding() *huffmanEncoder {
    65  	h := newHuffmanEncoder(maxNumLit)
    66  	codes := h.codes
    67  	var ch uint16
    68  	for ch = 0; ch < maxNumLit; ch++ {
    69  		var bits uint16
    70  		var size uint16
    71  		switch {
    72  		case ch < 144:
    73  			// size 8, 000110000  .. 10111111
    74  			bits = ch + 48
    75  			size = 8
    76  			break
    77  		case ch < 256:
    78  			// size 9, 110010000 .. 111111111
    79  			bits = ch + 400 - 144
    80  			size = 9
    81  			break
    82  		case ch < 280:
    83  			// size 7, 0000000 .. 0010111
    84  			bits = ch - 256
    85  			size = 7
    86  			break
    87  		default:
    88  			// size 8, 11000000 .. 11000111
    89  			bits = ch + 192 - 280
    90  			size = 8
    91  		}
    92  		codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
    93  	}
    94  	return h
    95  }
    96  
    97  func generateFixedOffsetEncoding() *huffmanEncoder {
    98  	h := newHuffmanEncoder(30)
    99  	codes := h.codes
   100  	for ch := range codes {
   101  		codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
   102  	}
   103  	return h
   104  }
   105  
   106  var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
   107  var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
   108  
   109  func (h *huffmanEncoder) bitLength(freq []int32) int {
   110  	var total int
   111  	for i, f := range freq {
   112  		if f != 0 {
   113  			total += int(f) * int(h.codes[i].len)
   114  		}
   115  	}
   116  	return total
   117  }
   118  
   119  const maxBitsLimit = 16
   120  
   121  // bitCounts computes the number of literals assigned to each bit size in the Huffman encoding.
   122  // It is only called when list.length >= 3.
   123  // The cases of 0, 1, and 2 literals are handled by special case code.
   124  //
   125  // list is an array of the literals with non-zero frequencies
   126  // and their associated frequencies. The array is in order of increasing
   127  // frequency and has as its last element a special element with frequency
   128  // MaxInt32.
   129  //
   130  // maxBits is the maximum number of bits that should be used to encode any literal.
   131  // It must be less than 16.
   132  //
   133  // bitCounts returns an integer slice in which slice[i] indicates the number of literals
   134  // that should be encoded in i bits.
   135  func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
   136  	if maxBits >= maxBitsLimit {
   137  		panic("flate: maxBits too large")
   138  	}
   139  	n := int32(len(list))
   140  	list = list[0 : n+1]
   141  	list[n] = maxNode()
   142  
   143  	// The tree can't have greater depth than n - 1, no matter what. This
   144  	// saves a little bit of work in some small cases
   145  	if maxBits > n-1 {
   146  		maxBits = n - 1
   147  	}
   148  
   149  	// Create information about each of the levels.
   150  	// A bogus "Level 0" whose sole purpose is so that
   151  	// level1.prev.needed==0.  This makes level1.nextPairFreq
   152  	// be a legitimate value that never gets chosen.
   153  	var levels [maxBitsLimit]levelInfo
   154  	// leafCounts[i] counts the number of literals at the left
   155  	// of ancestors of the rightmost node at level i.
   156  	// leafCounts[i][j] is the number of literals at the left
   157  	// of the level j ancestor.
   158  	var leafCounts [maxBitsLimit][maxBitsLimit]int32
   159  
   160  	for level := int32(1); level <= maxBits; level++ {
   161  		// For every level, the first two items are the first two characters.
   162  		// We initialize the levels as if we had already figured this out.
   163  		levels[level] = levelInfo{
   164  			level:        level,
   165  			lastFreq:     list[1].freq,
   166  			nextCharFreq: list[2].freq,
   167  			nextPairFreq: list[0].freq + list[1].freq,
   168  		}
   169  		leafCounts[level][level] = 2
   170  		if level == 1 {
   171  			levels[level].nextPairFreq = math.MaxInt32
   172  		}
   173  	}
   174  
   175  	// We need a total of 2*n - 2 items at top level and have already generated 2.
   176  	levels[maxBits].needed = 2*n - 4
   177  
   178  	level := maxBits
   179  	for {
   180  		l := &levels[level]
   181  		if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
   182  			// We've run out of both leafs and pairs.
   183  			// End all calculations for this level.
   184  			// To make sure we never come back to this level or any lower level,
   185  			// set nextPairFreq impossibly large.
   186  			l.needed = 0
   187  			levels[level+1].nextPairFreq = math.MaxInt32
   188  			level++
   189  			continue
   190  		}
   191  
   192  		prevFreq := l.lastFreq
   193  		if l.nextCharFreq < l.nextPairFreq {
   194  			// The next item on this row is a leaf node.
   195  			n := leafCounts[level][level] + 1
   196  			l.lastFreq = l.nextCharFreq
   197  			// Lower leafCounts are the same of the previous node.
   198  			leafCounts[level][level] = n
   199  			l.nextCharFreq = list[n].freq
   200  		} else {
   201  			// The next item on this row is a pair from the previous row.
   202  			// nextPairFreq isn't valid until we generate two
   203  			// more values in the level below
   204  			l.lastFreq = l.nextPairFreq
   205  			// Take leaf counts from the lower level, except counts[level] remains the same.
   206  			copy(leafCounts[level][:level], leafCounts[level-1][:level])
   207  			levels[l.level-1].needed = 2
   208  		}
   209  
   210  		if l.needed--; l.needed == 0 {
   211  			// We've done everything we need to do for this level.
   212  			// Continue calculating one level up. Fill in nextPairFreq
   213  			// of that level with the sum of the two nodes we've just calculated on
   214  			// this level.
   215  			if l.level == maxBits {
   216  				// All done!
   217  				break
   218  			}
   219  			levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
   220  			level++
   221  		} else {
   222  			// If we stole from below, move down temporarily to replenish it.
   223  			for levels[level-1].needed > 0 {
   224  				level--
   225  			}
   226  		}
   227  	}
   228  
   229  	// Somethings is wrong if at the end, the top level is null or hasn't used
   230  	// all of the leaves.
   231  	if leafCounts[maxBits][maxBits] != n {
   232  		panic("leafCounts[maxBits][maxBits] != n")
   233  	}
   234  
   235  	bitCount := h.bitCount[:maxBits+1]
   236  	bits := 1
   237  	counts := &leafCounts[maxBits]
   238  	for level := maxBits; level > 0; level-- {
   239  		// chain.leafCount gives the number of literals requiring at least "bits"
   240  		// bits to encode.
   241  		bitCount[bits] = counts[level] - counts[level-1]
   242  		bits++
   243  	}
   244  	return bitCount
   245  }
   246  
   247  // Look at the leaves and assign them a bit count and an encoding as specified
   248  // in RFC 1951 3.2.2
   249  func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
   250  	code := uint16(0)
   251  	for n, bits := range bitCount {
   252  		code <<= 1
   253  		if n == 0 || bits == 0 {
   254  			continue
   255  		}
   256  		// The literals list[len(list)-bits] .. list[len(list)-bits]
   257  		// are encoded using "bits" bits, and get the values
   258  		// code, code + 1, ....  The code values are
   259  		// assigned in literal order (not frequency order).
   260  		chunk := list[len(list)-int(bits):]
   261  
   262  		h.lns.sort(chunk)
   263  		for _, node := range chunk {
   264  			h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
   265  			code++
   266  		}
   267  		list = list[0 : len(list)-int(bits)]
   268  	}
   269  }
   270  
   271  // Update this Huffman Code object to be the minimum code for the specified frequency count.
   272  //
   273  // freq is an array of frequencies, in which freq[i] gives the frequency of literal i.
   274  // maxBits  The maximum number of bits to use for any literal.
   275  func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
   276  	if h.freqcache == nil {
   277  		// Allocate a reusable buffer with the longest possible frequency table.
   278  		// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
   279  		// The largest of these is maxNumLit, so we allocate for that case.
   280  		h.freqcache = make([]literalNode, maxNumLit+1)
   281  	}
   282  	list := h.freqcache[:len(freq)+1]
   283  	// Number of non-zero literals
   284  	count := 0
   285  	// Set list to be the set of all non-zero literals and their frequencies
   286  	for i, f := range freq {
   287  		if f != 0 {
   288  			list[count] = literalNode{uint16(i), f}
   289  			count++
   290  		} else {
   291  			h.codes[i].len = 0
   292  		}
   293  	}
   294  
   295  	list = list[:count]
   296  	if count <= 2 {
   297  		// Handle the small cases here, because they are awkward for the general case code. With
   298  		// two or fewer literals, everything has bit length 1.
   299  		for i, node := range list {
   300  			// "list" is in order of increasing literal value.
   301  			h.codes[node.literal].set(uint16(i), 1)
   302  		}
   303  		return
   304  	}
   305  	h.lfs.sort(list)
   306  
   307  	// Get the number of literals for each bit count
   308  	bitCount := h.bitCounts(list, maxBits)
   309  	// And do the assignment
   310  	h.assignEncodingAndSize(bitCount, list)
   311  }
   312  
   313  type byLiteral []literalNode
   314  
   315  func (s *byLiteral) sort(a []literalNode) {
   316  	*s = byLiteral(a)
   317  	sort.Sort(s)
   318  }
   319  
   320  func (s byLiteral) Len() int { return len(s) }
   321  
   322  func (s byLiteral) Less(i, j int) bool {
   323  	return s[i].literal < s[j].literal
   324  }
   325  
   326  func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
   327  
   328  type byFreq []literalNode
   329  
   330  func (s *byFreq) sort(a []literalNode) {
   331  	*s = byFreq(a)
   332  	sort.Sort(s)
   333  }
   334  
   335  func (s byFreq) Len() int { return len(s) }
   336  
   337  func (s byFreq) Less(i, j int) bool {
   338  	if s[i].freq == s[j].freq {
   339  		return s[i].literal < s[j].literal
   340  	}
   341  	return s[i].freq < s[j].freq
   342  }
   343  
   344  func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
   345  
   346  func reverseBits(number uint16, bitLength byte) uint16 {
   347  	return bits.Reverse16(number << (16 - bitLength))
   348  }
   349
View as plain text