      1
      2
      3
      4
      5
      6
      7
      8
      9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
     24
     25
     26
     27
     28
     29
     30
     31
     32
     33
     34
     35
     36
     37
     38
     39
     40
     41
     42
     43
     44
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234
    235
    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246
    247
    248
    249
    250
    251
    252
    253
    254
    255
    256
    257
    258
    259
    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277
    278
    279
    280
    281
    282
    283
    284
    285
    286
    287
    288
    289
    290
    291
    292
    293
    294
    295
    296
    297
    298
    299
    300
    301
    302
    303
    304
    305
    306
    307
    308
    309
    310
    311
    312
    313
    314
    315
    316
    317
    318
    319
    320
    321
    322
    323
    324
    325
    326
    327
    328
    329
    330
    331
    332
    333
    334
    335
    336
    337
    338
    339
    340
    341
    342
    343
    344
    345
    346
    347
    348
    349
    350
    351
    352
    353
    354
    355
    356
    357
    358
    359
    360
    361
    362
    363
    364
    365
    366
    367
    368
    369
    370
    371
    372
    373
    374
    375
    376
    377
    378
    379
    380
    381
    382
    383
    384
    385
    386
    387
    388
    389
    390
    391
    392
    393
    394
    395
    396

    use length_encode::{EncodedLength, encode_lengths_m, huffman_lengths_from_frequency_m,
                        COPY_PREVIOUS, REPEAT_ZERO_3_BITS, REPEAT_ZERO_7_BITS};
    use huffman_table::{HuffmanTable, create_codes_in_place, num_extra_bits_for_length_code,
                        num_extra_bits_for_distance_code, NUM_LITERALS_AND_LENGTHS,
                        NUM_DISTANCE_CODES, MAX_CODE_LENGTH, FIXED_CODE_LENGTHS, LENGTH_BITS_START};
    use bitstream::LsbWriter;
    use output_writer::FrequencyType;
    use stored_block::MAX_STORED_BLOCK_LENGTH;
    use deflate_state::LengthBuffers;

    use std::cmp;

    // The minimum number of literal/length values
    pub const MIN_NUM_LITERALS_AND_LENGTHS: usize = 257;
    // The minimum number of distances
    pub const MIN_NUM_DISTANCES: usize = 1;

    const NUM_HUFFMAN_LENGTHS: usize = 19;

    // The output ordering of the lengths for the huffman codes used to encode the lengths
    // used to build the full huffman tree for length/literal codes.
    // http://www.gzip.org/zlib/rfc-deflate.html#dyn
    const HUFFMAN_LENGTH_ORDER: [u8; NUM_HUFFMAN_LENGTHS] = [
        16,
        17,
        18,
        0,
        8,
        7,
        9,
        6,
        10,
        5,
        11,
        4,
        12,
        3,
        13,
        2,
        14,
        1,
        15,
    ];

    // Number of bits used for the values specifying the number of codes
    const HLIT_BITS: u8 = 5;
    const HDIST_BITS: u8 = 5;
    const HCLEN_BITS: u8 = 4;

    // The longest a huffman code describing another huffman length can be
    const MAX_HUFFMAN_CODE_LENGTH: usize = 7;

    // How many bytes (not including padding and the 3-bit block type) the stored block header takes up.
    const STORED_BLOCK_HEADER_LENGTH: u64 = 4;
    const BLOCK_MARKER_LENGTH: u8 = 3;

    /// Creates a new slice from the input slice that stops at the final non-zero value
    pub fn remove_trailing_zeroes<T: From<u8> + PartialEq>(input: &[T], min_length: usize) -> &[T] {
        let num_zeroes = input.iter().rev().take_while(|&a| *a == T::from(0)).count();
        &input[0..cmp::max(input.len() - num_zeroes, min_length)]
    }

    /// How many extra bits the huffman length code uses to represent a value.
    fn extra_bits_for_huffman_length_code(code: u8) -> u8 {
        match code {
            16...17 => 3,
            18 => 7,
            _ => 0,
        }
    }

    /// Calculate how many bits the huffman-encoded huffman lengths will use.
    fn calculate_huffman_length(frequencies: &[FrequencyType], code_lengths: &[u8]) -> u64 {
        frequencies.iter().zip(code_lengths).enumerate().fold(
            0,
            |acc, (n, (&f, &l))| {
                acc +
                    (u64::from(f) *
                         (u64::from(l) + u64::from(extra_bits_for_huffman_length_code(n as u8))))
            },
        )
    }

    /// Calculate how many bits data with the given frequencies will use when compressed with dynamic
    /// code lengths (first return value) and static code lengths (second return value).
    ///
    /// Parameters:
    /// Frequencies, length of dynamic codes, and a function to get how many extra bits in addition
    /// to the length of the huffman code the symbol will use.
    fn calculate_block_length<F>(
        frequencies: &[FrequencyType],
        dyn_code_lengths: &[u8],
        get_num_extra_bits: &F,
    ) -> (u64, u64)
    where
        F: Fn(usize) -> u64,
    {
        // Length of data represented by dynamic codes.
        let mut d_ll_length = 0u64;
        // length of data represented by static codes.
        let mut s_ll_length = 0u64;

        let iter = frequencies
            .iter()
            .zip(dyn_code_lengths.iter().zip(FIXED_CODE_LENGTHS.iter()))
            .enumerate();

        // This could maybe be optimised a bit by splitting the iteration of codes using extra bits and
        // codes not using extra bits, but the extra complexity may not be worth it.
        for (c, (&f, (&l, &fl))) in iter {
            // Frequency
            let f = u64::from(f);
            // How many extra bits the current code number needs.
            let extra_bits_for_code = get_num_extra_bits(c);

            d_ll_length += f * (u64::from(l) + extra_bits_for_code);
            s_ll_length += f * (u64::from(fl) + extra_bits_for_code);

        }

        (d_ll_length, s_ll_length)
    }

    /// Get how extra padding bits after a block start header a stored block would use.
    ///
    /// # Panics
    /// Panics if `pending_bits > 8`
    fn stored_padding(pending_bits: u8) -> u64 {
        assert!(pending_bits <= 8);
        let free_space = 8 - pending_bits;
        if free_space >= BLOCK_MARKER_LENGTH {
            // There is space in the current byte for the header.
            free_space - BLOCK_MARKER_LENGTH
        } else {
            // The header will require an extra byte.
            8 - (BLOCK_MARKER_LENGTH - free_space)
        }.into()
    }

    /// Calculate the number of bits storing the data in stored blocks will take up, excluding the
    /// first block start code and potential padding bits. As stored blocks have a maximum length,
    /// (as opposed to fixed and dynamic ones), multiple blocks may have to be utilised.
    ///
    /// # Panics
    /// Panics if `input_bytes` is 0.
    fn stored_length(input_bytes: u64) -> u64 {
        // Check how many stored blocks these bytes would take up.
        // (Integer divison rounding up.)
        let num_blocks = (input_bytes
                              .checked_sub(1)
                              .expect("Underflow calculating stored block length!") /
                              MAX_STORED_BLOCK_LENGTH as u64) + 1;
        // The length will be the input length and the headers for each block. (Excluding the start
        // of block code for the first one)
        (input_bytes + (STORED_BLOCK_HEADER_LENGTH as u64 * num_blocks) + (num_blocks - 1)) * 8
    }

    pub enum BlockType {
        Stored,
        Fixed,
        Dynamic(DynamicBlockHeader),
    }

    /// A struct containing the different data needed to write the header for a dynamic block.
    ///
    /// The code lengths are stored directly in the `HuffmanTable` struct.
    /// TODO: Do the same for other things here.
    pub struct DynamicBlockHeader {
        /// Length of the run-length encoding symbols.
        pub huffman_table_lengths: Vec<u8>,
        /// Number of lengths for values describing the huffman table that encodes the length values
        /// of the main huffman tables.
        pub used_hclens: usize,
    }

    /// Generate the lengths of the huffman codes we will be using, using the
    /// frequency of the different symbols/lengths/distances, and determine what block type will give
    /// the shortest representation.
    /// TODO: This needs a test
    pub fn gen_huffman_lengths(
        l_freqs: &[FrequencyType],
        d_freqs: &[FrequencyType],
        num_input_bytes: u64,
        pending_bits: u8,
        l_lengths: &mut [u8; 288],
        d_lengths: &mut [u8; 32],
        length_buffers: &mut LengthBuffers,
    ) -> BlockType {
        // Avoid corner cases and issues if this is called for an empty block.
        // For blocks this short, a fixed block will be the shortest.
        // TODO: Find the minimum value it's worth doing calculations for.
        if num_input_bytes <= 4 {
            return BlockType::Fixed;
        };

        let l_freqs = remove_trailing_zeroes(l_freqs, MIN_NUM_LITERALS_AND_LENGTHS);
        let d_freqs = remove_trailing_zeroes(d_freqs, MIN_NUM_DISTANCES);

        // The huffman spec allows us to exclude zeroes at the end of the
        // table of huffman lengths.
        // Since a frequency of 0 will give an huffman
        // length of 0. We strip off the trailing zeroes before even
        // generating the lengths to save some work.
        // There is however a minimum number of values we have to keep
        // according to the deflate spec.
        // TODO: We could probably compute some of this in parallel.
        huffman_lengths_from_frequency_m(
            l_freqs,
            MAX_CODE_LENGTH,
            &mut length_buffers.leaf_buf,
            l_lengths,
        );
        huffman_lengths_from_frequency_m(
            d_freqs,
            MAX_CODE_LENGTH,
            &mut length_buffers.leaf_buf,
            d_lengths,
        );

        let used_lengths = l_freqs.len();
        let used_distances = d_freqs.len();

        // Encode length values
        let mut freqs = [0u16; 19];
        encode_lengths_m(
            l_lengths[..used_lengths]
                .iter()
                .chain(&d_lengths[..used_distances]),
            &mut length_buffers.length_buf,
            &mut freqs,
        );

        // Create huffman lengths for the length/distance code lengths
        let mut huffman_table_lengths = vec![0; freqs.len()];
        huffman_lengths_from_frequency_m(
            &freqs,
            MAX_HUFFMAN_CODE_LENGTH,
            &mut length_buffers.leaf_buf,
            huffman_table_lengths.as_mut_slice(),
        );

        // Count how many of these lengths we use.
        let used_hclens = HUFFMAN_LENGTH_ORDER.len() -
            HUFFMAN_LENGTH_ORDER
                .iter()
                .rev()
                .take_while(|&&n| huffman_table_lengths[n as usize] == 0)
                .count();

        // There has to be at least 4 hclens, so if there isn't, something went wrong.
        debug_assert!(used_hclens >= 4);

        // Calculate how many bytes of space this block will take up with the different block types
        // (excluding the 3-bit block header since it's used in all block types).

        // Total length of the compressed literals/lengths.
        let (d_ll_length, s_ll_length) = calculate_block_length(l_freqs, l_lengths, &|c| {
            num_extra_bits_for_length_code(c.saturating_sub(LENGTH_BITS_START as usize) as u8).into()
        });

        // Total length of the compressed distances.
        let (d_dist_length, s_dist_length) = calculate_block_length(d_freqs, d_lengths, &|c| {
            num_extra_bits_for_distance_code(c as u8).into()
        });

        // Total length of the compressed huffman code lengths.
        let huff_table_length = calculate_huffman_length(&freqs, &huffman_table_lengths);

        // For dynamic blocks the huffman tables takes up some extra space.
        let dynamic_length = d_ll_length + d_dist_length + huff_table_length +
            (used_hclens as u64 * 3) + u64::from(HLIT_BITS) +
            u64::from(HDIST_BITS) + u64::from(HCLEN_BITS);

        // Static blocks don't have any extra header data.
        let static_length = s_ll_length + s_dist_length;

        // Calculate how many bits it will take to store the data in uncompressed (stored) block(s).
        let stored_length = stored_length(num_input_bytes) + stored_padding(pending_bits % 8);

        let used_length = cmp::min(cmp::min(dynamic_length, static_length), stored_length);

        // Check if the block is actually compressed. If using a dynamic block
        // increases the length of the block (for instance if the input data is mostly random or
        // already compressed), we want to output a stored(uncompressed) block instead to avoid wasting
        // space.
        if used_length == static_length {
            BlockType::Fixed
        } else if used_length == stored_length {
            BlockType::Stored
        } else {
            BlockType::Dynamic(DynamicBlockHeader {
                huffman_table_lengths: huffman_table_lengths,
                used_hclens: used_hclens,
            })
        }
    }

    /// Write the specified huffman lengths to the bit writer
    pub fn write_huffman_lengths(
        header: &DynamicBlockHeader,
        huffman_table: &HuffmanTable,
        encoded_lengths: &[EncodedLength],
        writer: &mut LsbWriter,
    ) {
        // Ignore trailing zero lengths as allowed by the deflate spec.
        let (literal_len_lengths, distance_lengths) = huffman_table.get_lengths();
        let literal_len_lengths =
            remove_trailing_zeroes(literal_len_lengths, MIN_NUM_LITERALS_AND_LENGTHS);
        let distance_lengths = remove_trailing_zeroes(distance_lengths, MIN_NUM_DISTANCES);
        let huffman_table_lengths = &header.huffman_table_lengths;
        let used_hclens = header.used_hclens;

        assert!(literal_len_lengths.len() <= NUM_LITERALS_AND_LENGTHS);
        assert!(literal_len_lengths.len() >= MIN_NUM_LITERALS_AND_LENGTHS);
        assert!(distance_lengths.len() <= NUM_DISTANCE_CODES);
        assert!(distance_lengths.len() >= MIN_NUM_DISTANCES);

        // Number of length codes - 257.
        let hlit = (literal_len_lengths.len() - MIN_NUM_LITERALS_AND_LENGTHS) as u16;
        writer.write_bits(hlit, HLIT_BITS);
        // Number of distance codes - 1.
        let hdist = (distance_lengths.len() - MIN_NUM_DISTANCES) as u16;
        writer.write_bits(hdist, HDIST_BITS);

        // Number of huffman table lengths - 4.
        let hclen = used_hclens.saturating_sub(4);

        // Write HCLEN.
        // Casting to u16 is safe since the length can never be more than the length of
        // `HUFFMAN_LENGTH_ORDER` anyhow.
        writer.write_bits(hclen as u16, HCLEN_BITS);

        // Write the lengths for the huffman table describing the huffman table
        // Each length is 3 bits
        for n in &HUFFMAN_LENGTH_ORDER[..used_hclens] {
            writer.write_bits(huffman_table_lengths[usize::from(*n)] as u16, 3);
        }

        // Generate codes for the main huffman table using the lengths we just wrote
        let mut codes = [0u16; NUM_HUFFMAN_LENGTHS];
        create_codes_in_place(&mut codes[..], huffman_table_lengths);

        // Write the actual huffman lengths
        for v in encoded_lengths {
            match *v {
                EncodedLength::Length(n) => {
                    let (c, l) = (codes[usize::from(n)], huffman_table_lengths[usize::from(n)]);
                    writer.write_bits(c, l);
                }
                EncodedLength::CopyPrevious(n) => {
                    let (c, l) = (codes[COPY_PREVIOUS], huffman_table_lengths[COPY_PREVIOUS]);
                    writer.write_bits(c, l);
                    debug_assert!(n >= 3);
                    debug_assert!(n <= 6);
                    writer.write_bits((n - 3).into(), 2);
                }
                EncodedLength::RepeatZero3Bits(n) => {
                    let (c, l) = (
                        codes[REPEAT_ZERO_3_BITS],
                        huffman_table_lengths[REPEAT_ZERO_3_BITS],
                    );
                    writer.write_bits(c, l);
                    debug_assert!(n >= 3);
                    writer.write_bits((n - 3).into(), 3);
                }
                EncodedLength::RepeatZero7Bits(n) => {
                    let (c, l) = (
                        codes[REPEAT_ZERO_7_BITS],
                        huffman_table_lengths[REPEAT_ZERO_7_BITS],
                    );
                    writer.write_bits(c, l);
                    debug_assert!(n >= 11);
                    debug_assert!(n <= 138);
                    writer.write_bits((n - 11).into(), 7);
                }
            }
        }
    }

    #[cfg(test)]
    mod test {
        use super::stored_padding;
        #[test]
        fn padding() {
            assert_eq!(stored_padding(0), 5);
            assert_eq!(stored_padding(1), 4);
            assert_eq!(stored_padding(2), 3);
            assert_eq!(stored_padding(3), 2);
            assert_eq!(stored_padding(4), 1);
            assert_eq!(stored_padding(5), 0);
            assert_eq!(stored_padding(6), 7);
            assert_eq!(stored_padding(7), 6);
        }
    }

# Help

## Keyboard Shortcuts

`?`
  Show this help dialog
`S`
  Focus the search field
`↑`
  Move up in search results
`↓`
  Move down in search results
`↹`
  Switch tab
`⏎`
  Go to active search result
`+`
  Expand all sections
`-`
  Collapse all sections

## Search Tricks

Prefix searches with a type followed by a colon (e.g., `fn:`) to restrict the search to a given type.

Accepted types are: `fn`, `mod`, `struct`, `enum`, `trait`, `type`, `macro`, and `const`.

Search functions by type signature (e.g., `vec -> usize` or `* -> vec`)

Search multiple things at once by splitting your query with comma (e.g., `str,u8` or `String,struct:Vec,test`)