"Fossies" - the Fresh Open Source Software Archive

Member "lzip-1.22-rc2/encoder_base.cc" (30 Apr 2020, 6271 Bytes) of package /linux/misc/lzip-1.22-rc2.tar.lz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "encoder_base.cc" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.21_vs_1.22-rc1.

    1 /* Lzip - LZMA lossless data compressor
    2    Copyright (C) 2008-2020 Antonio Diaz Diaz.
    3 
    4    This program is free software: you can redistribute it and/or modify
    5    it under the terms of the GNU General Public License as published by
    6    the Free Software Foundation, either version 2 of the License, or
    7    (at your option) any later version.
    8 
    9    This program is distributed in the hope that it will be useful,
   10    but WITHOUT ANY WARRANTY; without even the implied warranty of
   11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12    GNU General Public License for more details.
   13 
   14    You should have received a copy of the GNU General Public License
   15    along with this program.  If not, see <http://www.gnu.org/licenses/>.
   16 */
   17 
   18 #define _FILE_OFFSET_BITS 64
   19 
   20 #include <algorithm>
   21 #include <cerrno>
   22 #include <cstdlib>
   23 #include <cstring>
   24 #include <new>
   25 #include <string>
   26 #include <vector>
   27 #include <stdint.h>
   28 
   29 #include "lzip.h"
   30 #include "encoder_base.h"
   31 
   32 
   33 Dis_slots dis_slots;
   34 Prob_prices prob_prices;
   35 
   36 
   37 bool Matchfinder_base::read_block()
   38   {
   39   if( !at_stream_end && stream_pos < buffer_size )
   40     {
   41     const int size = buffer_size - stream_pos;
   42     const int rd = readblock( infd, buffer + stream_pos, size );
   43     stream_pos += rd;
   44     if( rd != size && errno ) throw Error( "Read error" );
   45     if( rd < size ) { at_stream_end = true; pos_limit = buffer_size; }
   46     }
   47   return pos < stream_pos;
   48   }
   49 
   50 
   51 void Matchfinder_base::normalize_pos()
   52   {
   53   if( pos > stream_pos )
   54     internal_error( "pos > stream_pos in normalize_pos." );
   55   if( !at_stream_end )
   56     {
   57     // offset is int32_t for the std::min below
   58     const int32_t offset = pos - before_size - dictionary_size;
   59     const int size = stream_pos - offset;
   60     std::memmove( buffer, buffer + offset, size );
   61     partial_data_pos += offset;
   62     pos -= offset;      // pos = before_size + dictionary_size
   63     stream_pos -= offset;
   64     for( int i = 0; i < num_prev_positions; ++i )
   65       prev_positions[i] -= std::min( prev_positions[i], offset );
   66     for( int i = 0; i < pos_array_size; ++i )
   67       pos_array[i] -= std::min( pos_array[i], offset );
   68     read_block();
   69     }
   70   }
   71 
   72 
   73 Matchfinder_base::Matchfinder_base( const int before_size_,
   74                     const int dict_size, const int after_size,
   75                     const int dict_factor, const int num_prev_positions23_,
   76                     const int pos_array_factor, const int ifd )
   77   :
   78   partial_data_pos( 0 ),
   79   before_size( before_size_ ),
   80   pos( 0 ),
   81   cyclic_pos( 0 ),
   82   stream_pos( 0 ),
   83   num_prev_positions23( num_prev_positions23_ ),
   84   infd( ifd ),
   85   at_stream_end( false )
   86   {
   87   const int buffer_size_limit =
   88     ( dict_factor * dict_size ) + before_size + after_size;
   89   buffer_size = std::max( 65536, dict_size );
   90   buffer = (uint8_t *)std::malloc( buffer_size );
   91   if( !buffer ) throw std::bad_alloc();
   92   if( read_block() && !at_stream_end && buffer_size < buffer_size_limit )
   93     {
   94     uint8_t * const tmp = (uint8_t *)std::realloc( buffer, buffer_size_limit );
   95     if( !tmp ) { std::free( buffer ); throw std::bad_alloc(); }
   96     buffer = tmp;
   97     buffer_size = buffer_size_limit;
   98     read_block();
   99     }
  100   if( at_stream_end && stream_pos < dict_size )
  101     dictionary_size = std::max( (int)min_dictionary_size, stream_pos );
  102   else
  103     dictionary_size = dict_size;
  104   pos_limit = buffer_size;
  105   if( !at_stream_end ) pos_limit -= after_size;
  106   unsigned size = 1 << std::max( 16, real_bits( dictionary_size - 1 ) - 2 );
  107   if( dictionary_size > 1 << 26 )       // 64 MiB
  108     size >>= 1;
  109   key4_mask = size - 1;         // increases with dictionary size
  110   size += num_prev_positions23;
  111   num_prev_positions = size;
  112 
  113   pos_array_size = pos_array_factor * ( dictionary_size + 1 );
  114   size += pos_array_size;
  115   if( size * sizeof prev_positions[0] <= size ) prev_positions = 0;
  116   else prev_positions = new( std::nothrow ) int32_t[size];
  117   if( !prev_positions ) { std::free( buffer ); throw std::bad_alloc(); }
  118   pos_array = prev_positions + num_prev_positions;
  119   for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = 0;
  120   }
  121 
  122 
  123 void Matchfinder_base::reset()
  124   {
  125   if( stream_pos > pos )
  126     std::memmove( buffer, buffer + pos, stream_pos - pos );
  127   partial_data_pos = 0;
  128   stream_pos -= pos;
  129   pos = 0;
  130   cyclic_pos = 0;
  131   read_block();
  132   if( at_stream_end && stream_pos < dictionary_size )
  133     {
  134     dictionary_size = std::max( (int)min_dictionary_size, stream_pos );
  135     int size = 1 << std::max( 16, real_bits( dictionary_size - 1 ) - 2 );
  136     if( dictionary_size > 1 << 26 )     // 64 MiB
  137       size >>= 1;
  138     key4_mask = size - 1;
  139     size += num_prev_positions23;
  140     num_prev_positions = size;
  141     pos_array = prev_positions + num_prev_positions;
  142     }
  143   for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = 0;
  144   }
  145 
  146 
  147 void Range_encoder::flush_data()
  148   {
  149   if( pos > 0 )
  150     {
  151     if( outfd >= 0 && writeblock( outfd, buffer, pos ) != pos )
  152       throw Error( "Write error" );
  153     partial_member_pos += pos;
  154     pos = 0;
  155     show_cprogress();
  156     }
  157   }
  158 
  159 
  160 // End Of Stream marker => (dis == 0xFFFFFFFFU, len == min_match_len)
  161 void LZ_encoder_base::full_flush( const State state )
  162   {
  163   const int pos_state = data_position() & pos_state_mask;
  164   renc.encode_bit( bm_match[state()][pos_state], 1 );
  165   renc.encode_bit( bm_rep[state()], 0 );
  166   encode_pair( 0xFFFFFFFFU, min_match_len, pos_state );
  167   renc.flush();
  168   Lzip_trailer trailer;
  169   trailer.data_crc( crc() );
  170   trailer.data_size( data_position() );
  171   trailer.member_size( renc.member_position() + Lzip_trailer::size );
  172   for( int i = 0; i < Lzip_trailer::size; ++i )
  173     renc.put_byte( trailer.data[i] );
  174   renc.flush_data();
  175   }
  176 
  177 
  178 void LZ_encoder_base::reset()
  179   {
  180   Matchfinder_base::reset();
  181   crc_ = 0xFFFFFFFFU;
  182   bm_literal[0][0].reset( (1 << literal_context_bits) * 0x300 );
  183   bm_match[0][0].reset( State::states * pos_states );
  184   bm_rep[0].reset( State::states );
  185   bm_rep0[0].reset( State::states );
  186   bm_rep1[0].reset( State::states );
  187   bm_rep2[0].reset( State::states );
  188   bm_len[0][0].reset( State::states * pos_states );
  189   bm_dis_slot[0][0].reset( len_states * (1 << dis_slot_bits) );
  190   bm_dis[0].reset( modeled_distances - end_dis_model + 1 );
  191   bm_align[0].reset( dis_align_size );
  192   match_len_model.reset();
  193   rep_len_model.reset();
  194   renc.reset( dictionary_size );
  195   }