"Fossies" - the Fresh Open Source Software Archive

Member "ffe-0.3.9/src/execute.c" (26 Aug 2018, 76808 Bytes) of package /linux/privat/ffe-0.3.9.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "execute.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 0.3.8_vs_0.3.9.

    1 /* 
    2  *    ffe - flat file extractor
    3  *
    4  *    Copyright (C) 2006 Timo Savinen
    5  *    This file is part of ffe.
    6  * 
    7  *    ffe is free software; you can redistribute it and/or modify
    8  *    it under the terms of the GNU General Public License as published by
    9  *    the Free Software Foundation; either version 2 of the License, or
   10  *    (at your option) any later version.
   11  *
   12  *    ffe is distributed in the hope that it will be useful,
   13  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
   14  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15  *    GNU General Public License for more details.
   16  *
   17  *    You should have received a copy of the GNU General Public License
   18  *    along with ffe; if not, write to the Free Software
   19  *    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
   20  *
   21  */
   22 
   23 /* $Id: execute.c,v 1.118 2011-04-10 10:12:09 timo Exp $ */
   24 
   25 #include "ffe.h"
   26 #include <stdlib.h>
   27 #include <string.h>
   28 #include <ctype.h>
   29 #include <wchar.h>
   30 #ifdef HAVE_PRINTF_H
   31 #include <printf.h>
   32 #endif
   33 
   34 
   35 #ifdef PACKAGE
   36 static char *program = PACKAGE;
   37 #else
   38 static char *program = "ffe";
   39 #endif
   40 
   41 #define READ_LINE_LEN 33554432
   42 #define READ_LINE_LEN_HIGH (READ_LINE_LEN - 524288)
   43 
   44 
   45 
   46 #define GUESS_LINES 1000
   47 #define GUESS_BUFFER 524288
   48 #define FIELD_SIZE (128 * 1024)
   49 #define WRITE_BUFFER (2 * 1024 * 1024)
   50 #define JUSTIFY_STRING 128
   51 
   52 extern int update_anon_info(struct structure *,char *);
   53 extern void init_libgcrypt();
   54 
   55 
   56 extern struct replace *replace;
   57 
   58 struct input_file *files = NULL;
   59 static struct input_file *current_file = NULL;
   60 static FILE *input_fp = NULL;
   61 static int ungetchar = -1;
   62 static char *default_output_file = NULL;
   63 static FILE *default_output_fp = NULL;
   64 
   65 static char *output_file = NULL;
   66 static FILE *output_fp = NULL;
   67 
   68 static uint8_t *read_buffer = NULL;
   69 static uint8_t *read_buffer_start = NULL;
   70 static uint8_t *read_buffer_high_water = NULL;
   71 
   72 static size_t last_consumed = 0;  /* for binary reads */
   73 static uint8_t *field_buffer = NULL;
   74 static int field_buffer_size = FIELD_SIZE;
   75 static int guess_lines = 0;
   76 
   77 static int eocf = 0;
   78 static int ccount = -1;
   79 static int orig_ccount = -1;
   80 
   81 static uint8_t justify_string[JUSTIFY_STRING];
   82 
   83 /* write buffer definitions */
   84 static uint8_t *write_buffer = NULL;
   85 static int write_buffer_size = WRITE_BUFFER;
   86 static uint8_t *write_pos;
   87 static uint8_t *write_buffer_end;
   88 
   89 /* file number counters */
   90 static long int current_file_lineno;
   91 static long int current_total_lineno;
   92 static long long int current_offset = 0;
   93 static long long int current_file_offset = 0;
   94 
   95 
   96 /* Pipe management */
   97 #define PIPE_OUTPUT_LEN 1048576
   98 static uint8_t pipe_output[PIPE_OUTPUT_LEN];
   99 
  100 /* examples of non matching lines */
  101 #define NO_MATCH_LINES 1
  102 static int no_matching_lines = 0;
  103 
  104 /* header definition from structure */
  105 static int headers;
  106 
  107 char *current_file_name = NULL;
  108 
  109 static char debug_file[128];
  110 static FILE *debug_fp = NULL;
  111 static long int debug_lineno = 0;
  112 
  113 static uint8_t bcd_to_ascii_cap[]={'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','\000'};
  114 static uint8_t hex_to_ascii_cap[]={'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
  115 
  116 static uint8_t bcd_to_ascii_low[]={'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','\000'};
  117 static uint8_t hex_to_ascii_low[]={'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
  118 
  119 uint8_t *bcd_to_ascii;
  120 uint8_t *hex_to_ascii;
  121 
  122 
  123 static void print_binary_field(uint8_t,struct field *,uint8_t *);
  124 static void print_fixed_field(uint8_t,struct field *,uint8_t *);
  125 
  126 inline uint8_t
  127 htocl(uint8_t hex)
  128 {
  129         return hex_to_ascii[hex & 0x0f];
  130 }
  131 
  132 inline uint8_t
  133 htocb(uint8_t hex)
  134 {
  135         return hex_to_ascii[(hex >> 4) & 0x0f];
  136 }
  137 
  138 inline uint8_t
  139 bcdtocl(uint8_t bcd)
  140 {
  141         return bcd_to_ascii[bcd & 0x0f];
  142 }
  143 
  144 inline uint8_t
  145 bcdtocb(uint8_t bcd)
  146 {
  147         return bcd_to_ascii[(bcd >> 4) & 0x0f];
  148 }
  149 
  150 
  151 void
  152 set_output_file(char *name)
  153 {
  154     if(name == NULL)
  155     {
  156         default_output_fp = stdout;
  157         default_output_file = "(stdout)";
  158     } else
  159     {
  160         default_output_fp = xfopen(name,"w");
  161         default_output_file = name;
  162     }
  163     output_fp = default_output_fp;
  164     output_file = default_output_file;
  165 }
  166 
  167 void 
  168 close_output_file()
  169 {
  170     struct output *o = output;
  171     int stdoutclosed = 0;
  172 
  173     if(default_output_fp == stdout) stdoutclosed = 1;
  174     if(fclose(default_output_fp) != 0)
  175     {
  176         panic("Error closing file",default_output_file,strerror(errno));
  177     }
  178 
  179     while(o != NULL)
  180     {
  181         if(o->ofp != NULL)
  182         {
  183             if((o->ofp == stdout && !stdoutclosed) || o->ofp != stdout)
  184             {
  185                 if(o->ofp == stdout) stdoutclosed = 1;
  186                 if(fclose(o->ofp) != 0)
  187                 {
  188                     panic("Error closing file",o->output_file,strerror(errno));
  189                 }
  190             }
  191         }
  192         o = o->next;
  193     }
  194 }
  195 
  196 void
  197 set_input_file(char *name)
  198 {
  199     register struct input_file *f = files;
  200     
  201     if(files == NULL)
  202     {
  203         files = xmalloc(sizeof(struct input_file));
  204         f = files;
  205     } else
  206     {
  207         while(f->next != NULL) f = f->next;
  208         f->next = xmalloc(sizeof(struct input_file));
  209         f = f->next;
  210     }
  211 
  212     f->next = NULL;
  213     f->name = xstrdup(name);
  214     f->lineno = 0;
  215 }
  216 
  217 static FILE *
  218 open_input_stream(char *file,char type)
  219 {
  220     int fds[2];
  221     pid_t pid;
  222     FILE *ret = NULL;
  223     char command[1024];
  224 
  225     if(ffe_open != NULL && ffe_open[0] != '\000')                // use preprocessor
  226     {
  227 #if defined(HAVE_WORKING_FORK) && defined(HAVE_DUP2) && defined(HAVE_PIPE)
  228        sprintf(command,ffe_open,file);
  229        if (pipe(fds) != 0) panic("Cannot create pipe",strerror(errno),NULL);
  230        pid = fork();
  231        if(pid == (pid_t) 0) /* Child */
  232        {
  233           close(fds[0]);
  234           if(dup2(fds[1],STDOUT_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
  235           if(execl(SHELL_CMD, "sh", "-c", command, NULL) == -1) panic("Starting a shell with execl failed",command,strerror(errno));
  236           close(fds[1]);
  237           _exit(EXIT_SUCCESS);
  238        } else if(pid > (pid_t) 0)
  239        {
  240           close(fds[1]);
  241           ret = fdopen(fds[0],"r");
  242           if(ret == NULL) panic("Cannot read from command",command,strerror(errno));
  243           
  244           ungetchar = fgetc(ret);
  245 
  246           if(ungetchar == EOF)       // check if pipe returns something, if not open file normally
  247           {
  248               ungetchar = -1;
  249               fclose(ret);
  250               ret = NULL;
  251           }
  252        } else
  253        {
  254           panic("Cannot fork",strerror(errno),NULL);
  255        }
  256 #else
  257        panic("Input preprocessing is not supported in this system",NULL,NULL);
  258 #endif
  259     }
  260 
  261     if(ret == NULL)
  262     {
  263         if(type == BINARY)
  264         {
  265             ret = xfopenb(file,"r");
  266         } else
  267         {
  268             ret = xfopen(file,"r");
  269         }
  270     }
  271 
  272     read_buffer = read_buffer_start;
  273 
  274     return ret;
  275 }
  276 
  277 
  278 void
  279 open_input_file(int stype)
  280 {
  281     read_buffer_start = xmalloc(READ_LINE_LEN);
  282     read_buffer = read_buffer_start;
  283     read_buffer_high_water = read_buffer_start + READ_LINE_LEN_HIGH;
  284 
  285     field_buffer = xmalloc(field_buffer_size);
  286 
  287     if(files->name[0] == '-' && !files->name[1])
  288     {
  289         input_fp = stdin;
  290         files->name = "(stdin)";
  291     } else
  292     {
  293         input_fp = open_input_stream(files->name,stype);
  294     }
  295     current_file = files;
  296     current_file->lineno = 0;
  297     current_file_name = current_file->name;
  298 }
  299 
  300 /* read input file until next newline is found
  301  */
  302 void
  303 complete_line_in_bin_buffer(int *ccount)
  304 {
  305     int c;
  306 
  307     if(read_buffer[*ccount-1] != '\n') // check if have newline as last char
  308     {
  309         do
  310         {
  311             c = fgetc(input_fp);
  312             if(*ccount >= READ_LINE_LEN) panic("Input file cannot be guessed, use -s option",NULL,NULL);
  313             if(c != EOF) read_buffer[(*ccount)++] = (uint8_t) c;
  314         } while(c != EOF && c != '\n');
  315     }
  316 }
  317 
  318 
  319 /* read from input stream. 
  320    Check if ungetchar contains a valid char and write it to buffer then read nmenb - 1 chars
  321    NOTE! it is assumed that size == 1...
  322 */
  323 static size_t
  324 uc_fread(uint8_t *ptr, size_t size, size_t nmemb, FILE *stream)
  325 {
  326     size_t ret = 0;
  327 
  328     if(ungetchar != -1)  // there is peeked char in ungetchar, write it to buffer and read the rest
  329     {
  330         *ptr = (uint8_t) ungetchar;
  331         ungetchar = -1;
  332         ptr++;
  333         nmemb--;
  334         ret = 1;
  335     }
  336 
  337     ret += fread(ptr,size,nmemb,stream);
  338     return ret;
  339 }
  340 
  341 /* read line from input stream. 
  342    Check if ungetchar contains a valid char and write it to buffer and then read the rest
  343 */
  344 static int 
  345 uc_fgets(uint8_t *s, int size, FILE *stream)
  346 {
  347     int ret;
  348     char *orig_s = s;
  349 
  350     if(ungetchar != -1)  // there is peeked char in ungetchar, write it to buffer and read the rest
  351     {
  352         *s = (uint8_t) ungetchar;
  353         ungetchar = -1;
  354         if(*s == '\n')
  355         {
  356             s++;
  357             *s = '\000';
  358             return 1;
  359         }
  360         s++;
  361         size--;
  362     } 
  363 
  364     if(fgets(s,size,stream) != NULL)
  365     {
  366         ret = strlen(orig_s);
  367     } else
  368     {
  369         ret = -1;
  370     }
  371 
  372     return ret;
  373 }
  374 
  375 
  376 /* find next lineend and return line length
  377 */
  378 static int
  379 find_next_LF(uint8_t *start,int length)
  380 {
  381     register uint8_t *p;
  382 
  383     p = memchr(start,'\n',length);
  384       
  385     if(p == NULL) 
  386     {
  387         start[length] = '\n';      // add  missing LF
  388         return length;
  389     }
  390     return p - start;
  391 }
  392 
  393 
  394 
  395 /* reads one file from input */
  396 /* returns the line length */
  397 /* return -1 on EOF */
  398 int
  399 read_input_line(int stype)
  400 {
  401     int retval;
  402     size_t unused;
  403 
  404     do
  405     {
  406         if(stype == BINARY) 
  407         {
  408             current_offset += (long long) last_consumed;
  409             current_file_offset += (long long) last_consumed;
  410         }
  411 
  412         if(ccount <= 0)
  413         {
  414              ccount = uc_fread(read_buffer_start,1,READ_LINE_LEN,input_fp);
  415              if(ccount < READ_LINE_LEN) eocf = 1;
  416              read_buffer = read_buffer_start;
  417         } else
  418         {
  419             if(read_buffer + last_consumed >= read_buffer_high_water && !eocf)
  420             {
  421                 unused = READ_LINE_LEN-(read_buffer-read_buffer_start)-last_consumed;
  422 
  423                 memmove(read_buffer_start,read_buffer+last_consumed,unused);
  424                 ccount = uc_fread(read_buffer_start+unused,1,READ_LINE_LEN - unused,input_fp);
  425                 if(ccount < READ_LINE_LEN - unused) eocf = 1;
  426                 read_buffer = read_buffer_start;
  427                 ccount += unused;
  428             } else
  429             {
  430                 read_buffer += last_consumed;
  431                 ccount -= last_consumed;
  432             }
  433         } 
  434 
  435         retval = ccount;
  436 
  437         if(ccount > 0 && stype != BINARY)
  438         {
  439             retval = find_next_LF(read_buffer,ccount);
  440             last_consumed = retval + (ccount > retval ? 1 : 0);   // add lf
  441 #ifdef WIN32
  442             if(retval && read_buffer[retval - 1]  == '\r') {
  443                 retval--;
  444                 if(retval && read_buffer[retval - 1]  == '\r') retval--; /* There might be two CRs? */
  445             }
  446 #endif
  447         } else
  448         {
  449             last_consumed = 0;
  450         }
  451 
  452 
  453         if(ccount == 0)
  454         {
  455             if(fclose(input_fp))
  456             {
  457                 panic("Error closing file",files->name,strerror(errno));
  458             }
  459             current_file = current_file->next;
  460             if(current_file != NULL)
  461             {
  462                 if(current_file->name[0] == '-' && !current_file->name[1]) 
  463                 {
  464                     input_fp = stdin;
  465                     current_file->name = "(stdin)";
  466                 } else
  467                 {
  468                     input_fp = open_input_stream(current_file->name,stype);
  469                     if(stype == BINARY)
  470                     {
  471                         current_file_offset = 0;
  472                     } 
  473                 }
  474                 eocf = 0;
  475                 last_consumed = 0;
  476                 current_file_name = current_file->name;
  477                 current_file->lineno = 0;
  478                 current_file_offset = 0;
  479             } else
  480             {
  481                 retval = -1;
  482             }
  483         } else
  484         {
  485             if(stype != BINARY) current_file->lineno++;
  486         }
  487     } while(ccount == 0 && current_file != NULL);
  488 
  489     if(ccount > 0)
  490     {
  491         current_file_lineno = current_file->lineno;
  492     }
  493     return retval;
  494 }
  495 
  496 /* calculate field count from line containing separated fields */
  497 int
  498 get_field_count(uint8_t quote, uint8_t *type, uint8_t *line)
  499 {
  500     int inside_quote = 0;
  501     int fields = 0;
  502     register uint8_t *p = line;
  503 
  504     if(type[0] != SEPARATED) return 0;
  505 
  506 #ifdef WIN32
  507     if (*p != '\n' || *p != '\r') fields++; /* at least one */
  508 #else
  509     if (*p != '\n') fields++; /* at least one */
  510 #endif
  511 
  512 
  513 #ifdef WIN32
  514     while(*p != '\n' && *p != '\r')
  515 #else
  516     while(*p != '\n')
  517 #endif
  518     {
  519         if(*p == type[1] && !inside_quote)
  520         {
  521             fields++;
  522             if(type[2] == '*') while(*p == type[1]) p++;
  523         }
  524         
  525         if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
  526         {
  527             p++;
  528         } else if(*p == quote && quote)
  529         {
  530             inside_quote = !inside_quote;
  531         }
  532 #ifdef WIN32
  533         if(*p != '\n' && *p != '\r') p++;
  534 #else
  535         if(*p != '\n') p++;
  536 #endif
  537     }
  538     return fields;
  539 }
  540 
  541 /* returns a pointer for one field of the fixed length record */
  542 /* note first position is 1 */
  543 uint8_t *
  544 get_fixed_field(int position, int length, int line_length,uint8_t *line)
  545 {
  546     register uint8_t *t,*s;
  547 
  548     if(length >= field_buffer_size - 1) 
  549     {
  550         field_buffer_size = length * 2;
  551         field_buffer = xrealloc(field_buffer,field_buffer_size);
  552     }
  553 
  554     if(position <= line_length && position)
  555     {
  556         position--;            /* to get first as zero */
  557         s = line + position;
  558         t = field_buffer;
  559         memcpy(t,s,length);
  560         t[length] = 0;
  561     } else
  562     {
  563         field_buffer[0] = 0;
  564     }
  565     return field_buffer;
  566 }
  567 
  568 /* returns pointer to field in separated record */
  569 uint8_t *
  570 get_separated_field(int position, uint8_t quote,char *type, uint8_t *line)
  571 {
  572     register uint8_t *p = line;
  573     int fieldno = 1;
  574     int inside_quote = 0;
  575     register int i = 0;
  576 
  577 #ifdef WIN32
  578     while(*p != '\n' && *p != '\r' && fieldno <= position)
  579 #else
  580     while(*p != '\n' && fieldno <= position)
  581 #endif
  582     {
  583         if(*p == type[1] && !inside_quote)
  584         {
  585             fieldno++;
  586             if(type[2] == '*') while(*p == type[1]) p++;
  587         }
  588 
  589         if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
  590         {
  591             p++;
  592         } else if(*p == quote && quote)
  593         {
  594             inside_quote = !inside_quote;
  595             if(inside_quote && p[1]) p++;
  596         }
  597 
  598         if(fieldno == position)
  599         {
  600             if(*p == type[1] || (*p == quote && quote))
  601             {
  602                 if(inside_quote)
  603                 {
  604                     field_buffer[i] = *p;
  605                     i++;
  606                 }
  607             } else
  608             {
  609                 field_buffer[i] = *p;
  610                 i++;
  611             }
  612 
  613             if(i >= field_buffer_size - 1) 
  614             {
  615                 field_buffer_size = i * 2;
  616                 field_buffer = xrealloc(field_buffer,field_buffer_size);
  617             }
  618         }
  619 #ifdef WIN32
  620         if(*p != '\n' && *p != '\r') p++;
  621 #else
  622         if(*p != '\n') p++;
  623 #endif
  624     }
  625     field_buffer[i] = 0;
  626     return field_buffer;
  627 }
  628     
  629 
  630 
  631 /* read input stream once to read_buffer
  632    if allready read (orig_ccount > -1)
  633    reset pointrs and ccount
  634  */
  635 int
  636 init_guessing()
  637 {            
  638     if(orig_ccount == -1)
  639     {
  640         ccount = uc_fread(read_buffer_start,1,READ_LINE_LEN,input_fp);
  641         orig_ccount = ccount;
  642         if(ccount < READ_LINE_LEN) eocf = 1;
  643     } else
  644     {
  645         ccount = orig_ccount;
  646     }
  647     read_buffer = read_buffer_start;
  648     last_consumed = 0;
  649 
  650     return ccount;
  651 }
  652 
  653 /* read one input line from buffer for guessing, 
  654    return line length
  655    -1 if buffer consumed
  656  */
  657 static int
  658 read_guess_line()
  659 {
  660     int retval = -1;
  661 
  662     read_buffer += last_consumed;
  663     ccount -= last_consumed;
  664 
  665     if(ccount > 0)
  666     {
  667         retval = find_next_LF(read_buffer,ccount);
  668         last_consumed = retval + (ccount > retval ? 1 : 0);
  669         current_file->lineno++;
  670         current_file_lineno = current_file->lineno;
  671 #ifdef WIN32
  672         if(retval && read_buffer[retval - 1]  == '\r') {
  673             retval--;
  674             if(retval && read_buffer[retval - 1]  == '\r') retval--; /* There might be two CRs? */
  675         }
  676 #endif
  677     }
  678     return retval;
  679 }
  680 
  681 /* guessing done, reset ccount and pointer
  682  */
  683 static void
  684 reset_guessing()
  685 {
  686     ccount = orig_ccount;
  687     read_buffer = read_buffer_start;
  688     last_consumed = 0;
  689     current_file_lineno = 0;
  690     current_file->lineno = 0;
  691     current_file_offset = 0;
  692 }
  693 
  694 /* calculates votes for record, length has the buffer len and buffer 
  695    contains the line to be examined
  696  */
  697     int
  698 vote_record(uint8_t quote,char *type,int header,struct record *record,int length)
  699 {
  700     register struct id *i = record->i;
  701     int vote = 0,len;
  702     int ids = 0;
  703 
  704     while(i != NULL)
  705     {
  706         ids++;
  707         switch(type[0])
  708         {
  709             case FIXED_LENGTH:
  710 #ifdef HAVE_REGEX
  711                 if(i->regexp)
  712                 {
  713                     uint8_t *field;
  714                     field = get_fixed_field(i->position,length - i->position + 1,length,read_buffer);
  715                     if(length >= i->position && regexec(&i->reg,field,(size_t) 0, NULL, 0) == 0) vote++;
  716                 } else
  717 #endif
  718                 {
  719                     if(strncmp(i->key,&read_buffer[i->position -1],i->length) == 0) vote++;
  720                 }
  721                 break;
  722             case SEPARATED:
  723                 if(header && current_file_lineno == 1) // forgive header lines
  724                 {
  725                     vote++;
  726                 } else
  727                 {
  728 #ifdef HAVE_REGEX
  729                     if(i->regexp)
  730                     {
  731                         if(regexec(&i->reg,get_separated_field(i->position,quote,type,read_buffer),(size_t) 0, NULL, 0) == 0) vote++;
  732                     } else
  733 #endif
  734                     {
  735                         if(strcmp(i->key,get_separated_field(i->position,quote,type,read_buffer)) == 0) vote++;
  736                     }
  737                 }
  738                 break;
  739             case BINARY:
  740 #ifdef HAVE_REGEX
  741                 if(i->regexp)
  742                 {
  743                     uint8_t *field;
  744                     int flen = 64;
  745 
  746                     flen = length - i->position + 1 < flen ? length - i->position + 1 : flen;
  747                     field = get_fixed_field(i->position,flen,length,read_buffer);
  748 
  749                     if(length >= i->position && regexec(&i->reg,field,(size_t) 0, NULL, 0) == 0) vote++;
  750                 } else
  751 #endif
  752                 {
  753                     if(memcmp(i->key,&read_buffer[i->position - 1],i->length) == 0) vote++;
  754                 }
  755                 break;
  756         }
  757         i = i->next;
  758     }
  759     if(vote || record->i == NULL)    /* if keys are ok or missing, then check line length */
  760     {
  761         switch(type[0])
  762         {
  763             case FIXED_LENGTH:
  764                 if(((record->arb_length == RL_MIN || record->length_field != NULL) && record->length <= length) ||
  765                         (record->arb_length == RL_STRICT && record->length == length)) vote++;
  766                 break;
  767             case SEPARATED:
  768                 len = get_field_count(quote,type,read_buffer);
  769                 if((record->arb_length == RL_STRICT && record->length == len)  ||
  770                         (record->arb_length == RL_MIN && record->length <= len)) vote++;
  771                 break;
  772             case BINARY:
  773                 if(((vote == ids && ids) || !ids) && record->length <= length) vote++; /* exact binary length cannot be checked */
  774                 break;
  775         }
  776     }
  777 
  778     if(vote == ids + 1)  /* every id and line length must match to get a vote */
  779     {
  780         return 1;
  781     } else
  782     {
  783         return 0;
  784     }
  785 }
  786 
  787 
  788 /* calculates votes for one input line */
  789     void
  790 vote(int bindex,int line_length)
  791 {
  792     struct structure *s = structure;
  793     struct record *r;
  794     int votes,total_votes = 0;
  795 
  796     while(s != NULL)
  797     {
  798         r = s->r;
  799         votes = 0;
  800         if(s->vote == bindex && s->type[0] != BINARY)   // check only structures having all records matched so far
  801         {
  802             while(r != NULL && !votes)
  803             {
  804                 votes = vote_record(s->quote,s->type,s->header,r,line_length);
  805                 s->vote += votes;             /* only one vote per line */
  806                 r = r->next;
  807             }
  808         }
  809         total_votes += votes;
  810         s = s->next;
  811     }
  812     if(!total_votes && no_matching_lines < NO_MATCH_LINES)
  813     {
  814         no_matching_lines++;
  815         fprintf(stderr,"%s: Line %ld in \'%s\' does not match, line length = %d\n",program,current_file->lineno,current_file->name,line_length);
  816     }
  817 }
  818 
  819     void
  820 vote_binary(int buffer_size)
  821 {
  822     struct structure *s = structure;
  823     struct record *r;
  824 
  825     while(s != NULL)
  826     {
  827         r = s->r;
  828         if(s->type[0] == BINARY)   // check only structures having all records matched so far
  829         {
  830             while(r != NULL && !s->vote)
  831             {
  832                 if(r->i != NULL)
  833                 {
  834                     if(vote_record(s->quote,s->type,s->header,r,buffer_size)) s->vote = 1;
  835                 }
  836                 r = r->next;
  837             }
  838         }
  839         s = s->next;
  840     }
  841 }
  842 
  843 
  844 /* calculates votes for structures */
  845 /* returns pointer for structure name having all lines/blocks matched, in other case NULL */
  846 /* votes has the vote count hat must mach */
  847     char *
  848 check_votes(int votes)
  849 {
  850     struct structure *s = structure;
  851     char *winner = NULL;
  852     int errors = 0;
  853 
  854     while(s != NULL && votes)
  855     {
  856         if(s->vote == votes)
  857         {
  858             if(winner == NULL)
  859             {
  860                 winner = s->name;
  861             } else
  862             {
  863                 if(!errors) 
  864                 {
  865                     fprintf(stderr,"%s: Input data matches several structures: \'%s\'",program,winner);
  866                 } 
  867                 fprintf(stderr," \'%s\'",s->name);
  868                 errors++;
  869             }
  870         }
  871         s->vote = 0;
  872         s = s->next;
  873     }
  874     if(errors) 
  875     {
  876         fprintf(stderr,"\n");
  877         winner = NULL;
  878     }
  879     return winner;
  880 }
  881 
  882 
  883 /* guesses using binary only
  884  */
  885     char *
  886 guess_binary_structure()
  887 {
  888     int buffer_size;
  889     char *ret = NULL;
  890 
  891     if(!max_binary_record_length)  // no binary structs
  892     {
  893         //file_to_text(input_fp);
  894         return NULL;
  895     }
  896 
  897     buffer_size = init_guessing();
  898 
  899     if(buffer_size > 0)
  900     {
  901         vote_binary(buffer_size);
  902         ret = check_votes(1);
  903         reset_guessing();
  904     }
  905     
  906     return ret;
  907 }
  908 
  909 
  910 
  911 /* tries guess the input file structure */
  912 /* returns pointer to the name of guessed structure */
  913 
  914 char *
  915 guess_structure()
  916 {
  917     int memory_used = 0;
  918     int len;
  919         
  920     len = init_guessing();
  921 
  922     if(len <= 0) return NULL;
  923 
  924     do
  925     {
  926         len = read_guess_line();
  927         if(len != -1)
  928         {
  929             memory_used += last_consumed;
  930             vote(guess_lines,len);
  931             guess_lines++;
  932         }
  933     } while(len != -1 && guess_lines < GUESS_LINES && memory_used < GUESS_BUFFER);
  934 
  935     reset_guessing();
  936 
  937     return check_votes(guess_lines);
  938 }
  939 
  940 /* start write to write buffer */
  941 void
  942 start_write()
  943 {
  944     write_pos = write_buffer;
  945 }
  946 
  947 /* write uint8_t to write buffer */
  948 inline void
  949 writec(uint8_t c)
  950 {
  951     *write_pos = c;
  952 
  953     if(write_pos == write_buffer_end)
  954     {
  955         int written = write_buffer_end - write_buffer;
  956 
  957         write_buffer_size = write_buffer_size * 2;
  958         write_buffer = xrealloc(write_buffer,write_buffer_size);
  959         write_pos = write_buffer + written;
  960         write_buffer_end = write_buffer + (write_buffer_size - 1);
  961     }
  962 
  963     write_pos++;
  964 }
  965 
  966 
  967 /* write string to write buffer */
  968 inline void
  969 writes(uint8_t *string)
  970 {
  971     register uint8_t *s = string;
  972 
  973     if(s != NULL)
  974     {
  975         while(*s)
  976         {
  977             writec(*s);
  978             s++;
  979         }
  980     }
  981 }
  982 
  983 void
  984 flush_write()
  985 {
  986     size_t bytes;
  987 
  988     bytes = write_pos - write_buffer;
  989 
  990     if(fwrite(write_buffer,1,bytes,output_fp) != bytes)
  991     {
  992         panic("Error writing to",output_file,NULL);
  993     }
  994 }
  995 
  996 void
  997 print_raw(int size, uint8_t *buffer,int stype)
  998 {
  999     if(fwrite(buffer,1,size,default_output_fp) != size)
 1000     {
 1001         panic("Error writing to",default_output_file,NULL);
 1002     }
 1003     if(stype != BINARY) fputc('\n',default_output_fp);
 1004 }
 1005     
 1006 
 1007 
 1008 /* prints arbitrary text */
 1009 /* text can contain %-directives (no %d,%D, or %n) */
 1010 void
 1011 print_text(struct structure *s, struct record *r,uint8_t *buffer)
 1012 {
 1013     register uint8_t *text = buffer;
 1014     char num[64];
 1015 
 1016     if(text == NULL) return;
 1017     if(r != NULL && (r->o == no_output || r->o == raw)) return;
 1018     if(s->o == no_output || s->o == raw) return;
 1019 
 1020     start_write();
 1021 
 1022     while(*text)
 1023     {
 1024         if(*text == '%' && text[1]) 
 1025         {
 1026             text++;
 1027             switch(*text)
 1028             {
 1029                 case 'f':
 1030                     writes(current_file_name);
 1031                     break;
 1032                 case 's':
 1033                     writes(s->name);
 1034                     break;
 1035                 case 'r':
 1036                     if(r != NULL) writes(r->name);
 1037                     break;
 1038                 case 'o':
 1039                     sprintf(num,"%ld",current_file_lineno);
 1040                     writes(num);
 1041                     break;
 1042                 case 'O':
 1043                     sprintf(num,"%ld",current_total_lineno);
 1044                     writes(num);
 1045                     break;
 1046                 case 'I':
 1047                     sprintf(num,"%lld",current_offset);
 1048                     writes(num);
 1049                     break;
 1050                 case 'i':
 1051                     sprintf(num,"%lld",current_file_offset);
 1052                     writes(num);
 1053                     break;
 1054                 case 'g':
 1055                     if(r->level && r->level->group_name) writes(r->level->group_name);
 1056                     break;
 1057                 case 'n':
 1058                     if(r->level && r->level->element_name) writes(r->level->element_name);
 1059                     break;
 1060                 case '%':
 1061                     writec('%');
 1062                     break;
 1063                 default:
 1064                     writec('%');
 1065                     writec(*text);
 1066                     break;
 1067             }
 1068         } else
 1069         {
 1070             writec(*text);
 1071         }
 1072         text++;
 1073     }
 1074     flush_write();
 1075 }
 1076 
 1077 
 1078 /* prints a header text */
 1079 /* text can contain only one %-directive %n */
 1080 /* return values: */
 1081 /* 1 - Header is printed or header should not be printed */
 1082 /* 0 - Header is not printed and should be printed */
 1083 int
 1084 print_header(struct structure *s, struct record *r)
 1085 {
 1086     struct print_field *pf = r->pf;
 1087 
 1088     if(r->o == no_output || r->o == raw ||  r->o->header == NULL) return 1; /* no header for this run */
 1089 
 1090     if(pf == NULL) return 0;   /* no printable fields for this record */
 1091 
 1092     start_write();
 1093 
 1094     while(pf != NULL)
 1095     {
 1096         char *text = r->o->header;
 1097         while(*text)
 1098         {
 1099             if(*text == '%' && text[1]) 
 1100             {
 1101                 text++;
 1102                 switch(*text)
 1103                 {
 1104                     case 'n':
 1105                         writes(pf->f->name);
 1106                         break;
 1107                     default:
 1108                         writec('%');
 1109                         writec(*text);
 1110                         break;
 1111                 }
 1112             } else
 1113             {
 1114                 writec(*text);
 1115             }
 1116             text++;
 1117         }
 1118         if(pf->next != NULL && r->o->separator != NULL) writes(r->o->separator);
 1119         pf = pf->next;
 1120     }
 1121     writes(r->o->record_trailer);
 1122     flush_write();
 1123 
 1124     return 1;  /* header printed */
 1125 }
 1126 
 1127 /* returns pointer to next input line
 1128 
 1129    returns NULL if no more lines
 1130    length will be written to len
 1131    */
 1132 uint8_t *
 1133 get_input_line(int *len,int stype)
 1134 {
 1135     uint8_t *ret = NULL;
 1136 
 1137     *len = -1;
 1138 
 1139     do 
 1140     {
 1141         if(current_file != NULL)
 1142         {
 1143             *len = read_input_line(stype);
 1144             ret = read_buffer;
 1145         }
 1146         if(*len == -1) 
 1147         { 
 1148             ret = NULL;
 1149         } else
 1150         {
 1151             current_total_lineno++;
 1152         }
 1153     } while(current_file_lineno == 1 && headers == HEADER_ALL && current_total_lineno > 1);
 1154 
 1155     return ret;
 1156 }
 1157 
 1158 /* bpositions will be updated according current input buffer 
 1159  * the real length of bytes consumed returned
 1160    */
 1161  
 1162 size_t update_field_positions(char *type,uint8_t quote,struct record *r,int len,uint8_t *buffer)
 1163 {
 1164     register uint8_t *p = buffer;
 1165     uint8_t *field_start;
 1166     register int inside_quote = 0;
 1167     struct field *f = r->f;
 1168     int var_record_length;
 1169     int var_field_length = 0;
 1170     int cur_pos,var_field_passed;
 1171     size_t retval;
 1172 
 1173     retval = last_consumed;
 1174 
 1175     if(r->length_field)    // If dynamic length
 1176     {
 1177         start_write();
 1178         field_start = write_pos;
 1179         switch(type[0])
 1180         {
 1181              case BINARY:
 1182                  print_binary_field('d',r->length_field,buffer);
 1183              break;
 1184              case FIXED_LENGTH:
 1185                  print_fixed_field('d',r->length_field,buffer);
 1186              break;
 1187          }
 1188          writec(0);  // end of string
 1189          sscanf(field_start,"%d",&var_record_length);
 1190          var_record_length += r->var_length_adjust;
 1191          if(var_record_length >= len) var_record_length = len - 1;
 1192          var_field_length = var_record_length - r->length;
 1193          if(var_field_length < 0) var_field_length = 0;
 1194     }     
 1195 
 1196     switch(type[0])
 1197     {
 1198         case FIXED_LENGTH:
 1199             if(r->arb_length == RL_MIN)
 1200             {
 1201                 while(f != NULL)
 1202                 {
 1203                     if(f->next == NULL) // check last field existense
 1204                     {
 1205                         if(len > f->position)
 1206                         {
 1207                             f->bposition = f->position;
 1208                         } else
 1209                         {
 1210                             f->bposition = -1;
 1211                         }
 1212                     }
 1213                     f = f->next;
 1214                 }
 1215             }
 1216         case BINARY:           // no break here
 1217             if(type[0] == BINARY) 
 1218             {
 1219                 current_file->lineno++;
 1220                 current_file_lineno = current_file->lineno;
 1221             }
 1222 
 1223             if(r->length_field)    // If dynamic length
 1224             {
 1225                 if (type[0] == BINARY) retval = var_record_length;
 1226                 cur_pos = 0;
 1227                 var_field_passed = 0;
 1228                 while(f != NULL)
 1229                 { 
 1230                     if(var_field_passed && f->const_data == NULL)
 1231                     {
 1232                         f->bposition = cur_pos;
 1233                         cur_pos+=f->length;
 1234                     }
 1235                   
 1236                     if(f->var_length)
 1237                     {
 1238                         f->length = var_field_length;
 1239                         cur_pos = f->bposition + f->length;
 1240                         var_field_passed = 1;
 1241                     }
 1242                     f = f->next;
 1243                 }
 1244             } else
 1245             {
 1246                 if (type[0] == BINARY) retval = r->length;
 1247             }
 1248             break;
 1249         case SEPARATED:
 1250             while(f != NULL) 
 1251             {
 1252                 f->bposition = -1;
 1253                 f = f->next;
 1254             }
 1255 
 1256             f = r->f;
 1257 
 1258 #ifdef WIN32
 1259             while(*p != '\n' && *p != '\r' && f != NULL)
 1260 #else
 1261             while(*p != '\n' && f != NULL)
 1262 #endif
 1263             {
 1264                 if(p == buffer)  // first
 1265                 {
 1266                     if(*p != type[1]) f->bposition = 0;
 1267                     f = f->next;
 1268                 } 
 1269                 if((*p == type[1] && !inside_quote))
 1270                 {
 1271                     p++;
 1272                     if(type[2] == '*') while(*p == type[1]) p++;
 1273                     if(*p != type[1]) 
 1274                     {
 1275                         f->bposition = (int) (p - buffer);
 1276                     } else
 1277                     {
 1278                         p--;
 1279                     }
 1280                     f = f->next;
 1281                 }
 1282                 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && inside_quote && quote)
 1283                 {
 1284                     p++;
 1285                 } else if(*p == quote && quote)
 1286                 {
 1287                     inside_quote = !inside_quote;
 1288                 }
 1289 #ifdef WIN32
 1290                 if(*p != '\n' && *p != '\r') p++;
 1291 #else
 1292                 if(*p != '\n') p++;
 1293 #endif
 1294             }
 1295             break;
 1296     }
 1297     return retval;
 1298 }
 1299                        
 1300 /* check which record applies to current line */
 1301 struct record *
 1302 select_record(struct structure *s,int length,uint8_t *buffer)
 1303 {
 1304     register struct record *r;
 1305 
 1306     r = s->r;
 1307 
 1308     while(r != NULL)
 1309     {
 1310         if(vote_record(s->quote,s->type,s->header,r,length)) return r;
 1311         r = r->next;
 1312     }
 1313 
 1314     return NULL;
 1315 }
 1316 
 1317 /* write field content to pipe and read the output, returns bytes read. Output is written to pipe_output  */
 1318 int execute_pipe(uint8_t *input,int input_length,struct pipe *p)
 1319 {
 1320      int in_fds[2];
 1321      int out_fds[2];
 1322      pid_t pid;
 1323      FILE *rfd,*wfd;
 1324      int ret=0;
 1325 
 1326      if(!input_length) return 0; // dont pipe with no data
 1327 
 1328 #if defined(HAVE_WORKING_FORK) && defined(HAVE_DUP2) && defined(HAVE_PIPE)
 1329        if (pipe(in_fds) != 0 || pipe(out_fds) != 0) panic("Cannot create pipe",strerror(errno),NULL);
 1330        pid = fork();
 1331        if(pid == (pid_t) 0) /* Child */
 1332        {
 1333           close(in_fds[1]);
 1334           close(out_fds[0]);
 1335           if(dup2(in_fds[0],STDIN_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
 1336           close(in_fds[0]);
 1337           if(dup2(out_fds[1],STDOUT_FILENO) == -1) panic("dup2 error",strerror(errno),NULL);
 1338           close(out_fds[1]);
 1339           if(execl(SHELL_CMD, "sh", "-c", p->command, NULL) == -1) panic("Starting a shell with execl failed",p->command,strerror(errno));
 1340           _exit(EXIT_SUCCESS);
 1341        } else if(pid > (pid_t) 0)
 1342        {
 1343           close(in_fds[0]);
 1344           close(out_fds[1]);
 1345           wfd = fdopen(in_fds[1],"w");
 1346           if(wfd == NULL) panic("Cannot write to command",p->command,strerror(errno));
 1347           rfd = fdopen(out_fds[0],"r");
 1348           if(rfd == NULL) panic("Cannot read from command",p->command,strerror(errno));
 1349 
 1350           if(fwrite(input,input_length,1,wfd) != 1) panic("Cannot write to command",p->command,strerror(errno));
 1351           fclose(wfd);
 1352           ret = (int) fread(pipe_output,1,PIPE_OUTPUT_LEN,rfd);
 1353           fclose(rfd);
 1354           if(ret && pipe_output[ret - 1] == '\n')  pipe_output[ret - 1] = '\000';    // remove last linefeed
 1355        } else
 1356        {
 1357           panic("Cannot fork",strerror(errno),NULL);
 1358        }
 1359 #else
 1360 panic("pipe is not supported in this system",NULL,NULL);
 1361 #endif
 1362 return ret;
 1363 }
 1364 
 1365 
 1366 /* print a single fixed field */
 1367 void
 1368 print_fixed_field(uint8_t format,struct field *f,uint8_t *buffer)
 1369 {
 1370     register int i = 0;
 1371     register uint8_t *data;
 1372     uint8_t *start = write_pos;
 1373     int len = f->length;
 1374 
 1375     if(!f->length && f->var_length) return;
 1376 
 1377     if(f->const_data != NULL)
 1378     {
 1379         data = f->const_data;
 1380         if (format == 't') format = 'd';  // Dont't trim consts
 1381     } else
 1382     {
 1383         if(f->bposition < 0) return;  /* last variable length field is missing */
 1384         data = &buffer[f->bposition];
 1385 
 1386         if(f->p != NULL)
 1387         {
 1388             len = execute_pipe(data,f->length,f->p);
 1389             data = pipe_output;
 1390         }
 1391     }
 1392 
 1393     switch(format)
 1394     {
 1395         case 'd':
 1396         case 'D':
 1397         case 'C':
 1398         case 'e':
 1399         case 'x':
 1400             if(data == pipe_output)
 1401             {
 1402                 while(i < len) writec(data[i++]);
 1403             } else
 1404             {
 1405                 if(f->length)
 1406                 {
 1407 #ifdef WIN32
 1408                     while(i < len && data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
 1409 #else
 1410                     while(i < len && data[i] != '\n' && data[i]) writec(data[i++]);
 1411 #endif
 1412                 } else
 1413                 {
 1414 #ifdef WIN32
 1415                     while(data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
 1416 #else
 1417                     while(data[i] != '\n' && data[i]) writec(data[i++]);
 1418 #endif
 1419                 }
 1420             }
 1421             break;
 1422         case 't':
 1423             if(data == pipe_output)
 1424             {
 1425                 while(isblank(data[i])) i++;
 1426                 while(i < len) writec(data[i++]);
 1427             } else
 1428             {  
 1429                 while(isblank(data[i])) i++;
 1430                 if(f->length)
 1431                 {
 1432 #ifdef WIN32
 1433                     while(i < len && data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
 1434 #else
 1435                     while(i < len && data[i] != '\n' && data[i]) writec(data[i++]);
 1436 #endif
 1437                 } else
 1438                 {
 1439 #ifdef WIN32
 1440                     while(data[i] != '\n' && data[i] != '\r' && data[i]) writec(data[i++]);
 1441 #else
 1442                     while(data[i] != '\n' && data[i]) writec(data[i++]);
 1443 #endif
 1444                 }
 1445             }
 1446             
 1447             if(write_pos > start && isspace(write_pos[-1]))
 1448             {
 1449                 write_pos--;
 1450                 while(write_pos > start && (isspace(*write_pos))) write_pos--;
 1451                 write_pos++;
 1452             }
 1453             break;
 1454     }
 1455 }
 1456 
 1457 /* print a single binary field 
 1458    if field type is ASC, fixed field printing is used 
 1459 */
 1460 void
 1461 print_binary_field(uint8_t format,struct field *f,uint8_t *buffer)
 1462 {
 1463     register uint8_t *p,*data_end;
 1464     uint8_t *data,c;
 1465     char *pf;
 1466     static uint8_t pb[2 * FIELD_SIZE];
 1467     
 1468     if(!f->length && f->var_length) return;
 1469     
 1470     if(f->const_data != NULL || (f->type == F_ASC && format != 'h'))
 1471     {
 1472         print_fixed_field(format,f,buffer);
 1473         return;
 1474     }
 1475 
 1476     if(f->p != NULL)
 1477     {
 1478         int i=0,len;
 1479         len = execute_pipe(&buffer[f->bposition],f->length,f->p);
 1480         data = pipe_output;
 1481         while(i < len && data[i]) writec(data[i++]);
 1482         return;
 1483     }
 1484 
 1485 
 1486     switch(f->type)
 1487     {
 1488         case F_INT:
 1489         case F_UINT:
 1490         case F_FLOAT:
 1491         case F_DOUBLE:
 1492             data = endian_and_align(&buffer[f->bposition],system_endianess,f->endianess,f->length);
 1493             break;
 1494         default:
 1495             data = &buffer[f->bposition];
 1496             break;
 1497     }
 1498 
 1499     pb[0] = 0;
 1500 
 1501     switch(format)
 1502     {
 1503         case 'h':
 1504             p = &buffer[f->bposition];  // original data before any endian things
 1505             data_end = p + f->length;
 1506             while(p < data_end)
 1507             {
 1508                 writec('x');
 1509                 writec(htocb(*p));
 1510                 writec(htocl(*p));
 1511                 p++;
 1512             }
 1513             break;
 1514         case 'd':
 1515         case 't':
 1516         case 'D':
 1517         case 'C':
 1518         case 'x':
 1519             switch(f->type)
 1520             {
 1521                 case F_CHAR:
 1522                     writec(*data);
 1523                     break;
 1524                 case F_INT:
 1525                     switch(f->length)
 1526                     {
 1527                         case 1:
 1528                             pf = format == 'x' ? "%x" : "%i";
 1529                             sprintf(pb,pf,(int) *(int8_t *) data);
 1530                             break;
 1531                         case 2:
 1532                             pf = format == 'x' ? "%x" : "%i";
 1533                             sprintf(pb,pf,(int) *(int16_t *) data);
 1534                             break;
 1535                         case 4:
 1536                             pf = format == 'x' ? "%lx" : "%li";
 1537                             sprintf(pb,pf,(long int) *(int32_t *) data);
 1538                             break;
 1539                         case 8:
 1540                             pf = format == 'x' ? "%llx" : "%lli";
 1541                             sprintf(pb,pf,(long long int) *(int64_t *) data);
 1542                             break;
 1543                     }
 1544                     writes(pb);
 1545                     break;
 1546                 case F_UINT:
 1547                     switch(f->length)
 1548                     {
 1549                         case 1:
 1550                             pf = format == 'x' ? "%x" : "%u";
 1551                             sprintf(pb,pf,(unsigned int) *(uint8_t *) data);
 1552                             break;
 1553                         case 2:
 1554                             pf = format == 'x' ? "%x" : "%u";
 1555                             sprintf(pb,pf,(unsigned int) *(uint16_t *) data);
 1556                             break;
 1557                         case 4:
 1558                             pf = format == 'x' ? "%lx" : "%lu";
 1559                             sprintf(pb,pf,(unsigned long int) *(uint32_t *) data);
 1560                             break;
 1561                         case 8:
 1562                             pf = format == 'x' ? "%llx" : "%llu";
 1563                             sprintf(pb,pf,(unsigned long long int) *(uint64_t *) data);
 1564                             break;
 1565                     }
 1566                     writes(pb);
 1567                     break;
 1568                 case F_FLOAT:
 1569                     sprintf(pb,"%f",(double) *(float *) data);
 1570                     writes(pb);
 1571                     break;
 1572                 case F_DOUBLE:
 1573                     sprintf(pb,"%f",(double) *(double *) data);
 1574                     writes(pb);
 1575                     break;
 1576                 case F_BCD:
 1577                     p = data;
 1578                     data_end = p + f->length;
 1579                     switch(f->endianess)
 1580                     {
 1581                         case F_BIG_ENDIAN:
 1582                             do
 1583                             {
 1584                                 c = bcdtocb(*p);
 1585                                 if(c)
 1586                                 {
 1587                                     writec(c);
 1588                                     c = bcdtocl(*p);
 1589                                     if(c) writec(c);
 1590                                 }
 1591                                 p++;
 1592                             } while(p < data_end && c);
 1593                             break;
 1594                         case F_LITTLE_ENDIAN:
 1595                             do
 1596                             {
 1597                                 c = bcdtocl(*p);
 1598                                 if(c)
 1599                                 {
 1600                                     writec(c);
 1601                                     c = bcdtocb(*p);
 1602                                     if(c) writec(c);
 1603                                 }
 1604                                 p++;
 1605                             } while(p < data_end && c);
 1606                             break;
 1607                     }
 1608                     break;
 1609                 case F_HEX:
 1610                     p = data;
 1611                     data_end = p + f->length;
 1612                     switch(f->endianess)
 1613                     {
 1614                         case F_BIG_ENDIAN:
 1615                             while(p < data_end)
 1616                             {
 1617                                 writec(htocb(*p));
 1618                                 writec(htocl(*p));
 1619                                 p++;
 1620                             }
 1621                             break;
 1622                         case F_LITTLE_ENDIAN:
 1623                             p += f->length - 1;
 1624                             while(p >= data)
 1625                             {
 1626                                 writec(htocb(*p));
 1627                                 writec(htocl(*p));
 1628                                 p--;
 1629                             }
 1630                             break;
 1631                     }
 1632                     break;
 1633             }
 1634     }
 1635 }
 1636 
 1637 
 1638 
 1639 /* print a single separated field */
 1640 void
 1641 print_separated_field(uint8_t format,uint8_t quote,uint8_t separator,struct field *f,uint8_t *buffer)
 1642 {
 1643     register uint8_t *p;
 1644     uint8_t *start;
 1645     int inside_quote = 0;
 1646 
 1647     start = write_pos;
 1648 
 1649     if(f->const_data != NULL)   // use fixed field printing for const data
 1650     {
 1651         print_fixed_field(format,f,buffer);
 1652     } else
 1653     {
 1654         if(f->bposition < 0)
 1655         {
 1656             if(format == 'D' || format == 'C') while(write_pos - start < f->length) writec(' ');
 1657             return;
 1658         }
 1659 
 1660         if(f->p != NULL)
 1661         {
 1662             p = &buffer[f->bposition];
 1663             if(*p == quote && quote) {
 1664                 p++;
 1665                 inside_quote = 1;
 1666             }
 1667 #ifdef WIN32
 1668             while((*p != separator || inside_quote) && *p != '\n' && *p != '\r')
 1669 #else
 1670             while((*p != separator || inside_quote) && *p != '\n')
 1671 #endif
 1672             {
 1673                 if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && quote) 
 1674                 {
 1675                     p++;
 1676                 } else if(*p == quote)
 1677                 {
 1678                     if(inside_quote) inside_quote=0;
 1679                 }
 1680 #ifdef WIN32
 1681                 if(*p != '\n' && *p != '\r') p++;
 1682 #else
 1683                 if(*p != '\n') p++;
 1684 #endif
 1685             }            
 1686             int i=0,len;
 1687             len = execute_pipe(&buffer[f->bposition],p - &buffer[f->bposition],f->p);
 1688             while(i < len && pipe_output[i]) writec(pipe_output[i++]);
 1689             return;
 1690         }
 1691 
 1692 
 1693         switch(format)
 1694         {
 1695             case 'd':
 1696             case 't':
 1697             case 'D':
 1698             case 'C':
 1699             case 'e':
 1700             case 'x':
 1701                 p = &buffer[f->bposition];
 1702                 if(quote || format == 't') while(*p != separator && isblank(*p)) p++;
 1703                 if(*p == quote && quote) 
 1704                 {
 1705                     p++;
 1706                     inside_quote = 1;
 1707                     if(format == 't') while(isblank(*p)) p++;
 1708                 }
 1709 #ifdef WIN32
 1710                 while((*p != separator || inside_quote) && *p != '\n' && *p != '\r')
 1711 #else
 1712                 while((*p != separator || inside_quote) && *p != '\n')
 1713 #endif
 1714                 {
 1715                     if(((*p == quote && p[1] == quote) || (*p == '\\' && p[1] == quote)) && quote) 
 1716                     {
 1717                         p++;
 1718                     } else if(*p == quote)
 1719                     {
 1720                         if(inside_quote)
 1721                         {
 1722                             if(format == 't' && isblank(write_pos[-1]))
 1723                             {
 1724                                 write_pos--;     
 1725                                 while(write_pos > start && isblank(*write_pos)) write_pos--;
 1726                                 write_pos++;
 1727                             }
 1728                             if(format == 'D' || format == 'C') while(write_pos - start == f->length) writec(' ');
 1729                             return;
 1730                         }
 1731                     }
 1732 
 1733                     if(format == 'C' && (write_pos - start) == f->length) return;
 1734 #ifdef WIN32
 1735                     if(*p != '\n' && *p != '\r') {
 1736                        writec(*p);
 1737                        p++;
 1738                     }
 1739 #else
 1740                     if(*p != '\n') {
 1741                        writec(*p);
 1742                        p++;
 1743                     }
 1744 #endif
 1745 
 1746 
 1747                 }            
 1748                 if(format == 't' && isspace(write_pos[-1]))
 1749                 {
 1750                     write_pos--;     
 1751                     while(write_pos > start && isspace(*write_pos)) write_pos--;
 1752                     write_pos++;
 1753                 }
 1754                 if(format == 'D' || format == 'C') while(write_pos - start < f->length) writec(' ');
 1755                 break;
 1756         }
 1757     }
 1758 }
 1759 
 1760 /* search the lookup table, return the found value */
 1761 uint8_t *
 1762 make_lookup(struct lookup *l,uint8_t *search)
 1763 {
 1764     register uint8_t *k,*s;
 1765     int search_len = strlen(search);
 1766     uint8_t *ret_val = NULL;
 1767     struct lookup_data *d;
 1768 
 1769     switch(l->type)
 1770     {
 1771         case EXACT:
 1772 
 1773             d = l->data[hash(search,0)];
 1774 
 1775         while(d != NULL && ret_val == NULL)
 1776         {
 1777                 k = d->key;
 1778         s = search;
 1779         while(*k == *s && *k) 
 1780         {
 1781            k++;
 1782            s++;
 1783         }
 1784         if(!*s && !*k) ret_val = d->value;
 1785         d = d->next;
 1786         }
 1787         break;
 1788     case LONGEST:
 1789             while(search_len && ret_val == NULL)
 1790         {
 1791                 d = l->data[hash(search,0)];
 1792 
 1793             while(d != NULL && ret_val == NULL)
 1794             {
 1795             k = d->key;
 1796             s = search;
 1797             while(*k == *s && *k) 
 1798             {
 1799                 k++;
 1800                 s++;
 1801             }
 1802             if(!*s && !*k) ret_val = d->value;
 1803             d = d->next;
 1804              }
 1805                  search_len--;
 1806                  search[search_len] = 0;
 1807         }
 1808         break;
 1809     }
 1810 
 1811     if(ret_val == NULL) ret_val = l->default_value;
 1812 
 1813     return ret_val;
 1814 }
 1815 
 1816 void
 1817 print_indent(uint8_t *buffer,int times)
 1818 {
 1819     start_write();
 1820     while(times--) writes(buffer);
 1821     flush_write();
 1822 }
 1823 
 1824 /* format field with printf
 1825  */
 1826 #define CONV_BUF_SIZE 1048576
 1827 void
 1828 make_conversion(struct format *f,uint8_t *start)
 1829 {
 1830     static uint8_t conv_buffer[CONV_BUF_SIZE];
 1831 
 1832     *write_pos = 0;
 1833 
 1834     conv_buffer[0] = 0;
 1835 
 1836 #ifdef HAVE_PRINTF_H
 1837     switch(f->type)
 1838     {
 1839         case PA_INT:
 1840         case PA_INT|PA_FLAG_SHORT:
 1841         case PA_CHAR:
 1842         case PA_WCHAR:
 1843             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atoi(start));
 1844             break;
 1845         case PA_INT|PA_FLAG_LONG:
 1846             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atol(start));
 1847             break;
 1848         case PA_INT|PA_FLAG_LONG_LONG:
 1849             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atoll(start));
 1850             break;
 1851         case PA_FLOAT:
 1852         case PA_DOUBLE:
 1853             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,atof(start));
 1854             break;
 1855         case PA_DOUBLE|PA_FLAG_LONG_DOUBLE:
 1856             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,strtold(start,NULL));
 1857             break;
 1858         case PA_WSTRING:
 1859         case PA_STRING:
 1860         case PA_POINTER:
 1861             snprintf(conv_buffer,CONV_BUF_SIZE,f->conversion,start);
 1862             break;
 1863         default:
 1864             return;
 1865     }
 1866     write_pos = start;
 1867     writes(conv_buffer);
 1868 #endif
 1869 }
 1870 
 1871 
 1872 
 1873 /* print fields */
 1874 /* returns the count of fields actually printed */
 1875 int
 1876 print_fields(struct structure *s, struct record *r,uint8_t *buffer)
 1877 {
 1878     uint8_t num[64];
 1879     int max_justify_len = 0;
 1880     register uint8_t *d;
 1881     uint8_t *f;
 1882     int i;
 1883     uint8_t justify = LEFT_JUSTIFY;
 1884     uint8_t *indent,*separator;
 1885     uint8_t *data_start,*rep_start = NULL,*rep_pos = NULL;
 1886     uint8_t *field_start;
 1887     uint8_t *lookup_value;
 1888     int retval = 0;
 1889     int replacing,just_replaced;
 1890     int lookup_len;
 1891     struct print_field *pf = r->pf;
 1892     struct output *o;
 1893 
 1894     while(pf != NULL) {
 1895         pf->justify_length = -1;
 1896         pf = pf->next;
 1897     }
 1898 
 1899     pf = r->pf;
 1900 
 1901     start_write();
 1902     
 1903     while(pf != NULL)
 1904     {
 1905         o = pf->f->o ? pf->f->o : r->o;
 1906 
 1907         if(o != no_output)
 1908         {
 1909             justify = o->justify;
 1910             d = o->data;
 1911             data_start = write_pos;
 1912             pf->data = data_start;
 1913             pf->empty = 1;
 1914             replacing = 0;
 1915             just_replaced = 0;
 1916             lookup_value = NULL;
 1917 
 1918             if(pf->f->lookup != NULL) d = o->lookup;
 1919 
 1920             if(o->hex_cap)
 1921             {
 1922                 bcd_to_ascii = bcd_to_ascii_cap;
 1923                 hex_to_ascii = hex_to_ascii_cap;
 1924             } else
 1925             {
 1926                 bcd_to_ascii = bcd_to_ascii_low;
 1927                 hex_to_ascii = hex_to_ascii_low;
 1928             }
 1929 
 1930             while(*d)
 1931             {
 1932                 if(justify == *d)
 1933                 {
 1934                     if(justify != LEFT_JUSTIFY && justify != RIGHT_JUSTIFY && pf->justify_length == -1 && !replacing)
 1935                     {
 1936                         pf->justify_length = (int) (write_pos - data_start);
 1937                         if(pf->justify_length > max_justify_len)
 1938                         {
 1939                             max_justify_len = pf->justify_length;
 1940                         }
 1941                     } 
 1942                 }
 1943 
 1944                 if(*d == '%')
 1945                 {
 1946                     d++;
 1947                     switch(*d)
 1948                     {
 1949                         case 'f':
 1950                             writes(current_file_name);
 1951                             break;
 1952                         case 's':
 1953                             writes(s->name);
 1954                             break;
 1955                         case 'r':
 1956                             if(r != NULL) writes(r->name);
 1957                             break;
 1958                         case 'o':
 1959                             sprintf(num,"%ld",current_file_lineno);
 1960                             writes(num);
 1961                             break;
 1962                         case 'O':
 1963                             sprintf(num,"%ld",current_total_lineno);
 1964                             writes(num);
 1965                             break;
 1966                         case 'I':
 1967                             sprintf(num,"%lld",current_offset);
 1968                             writes(num);
 1969                             break;
 1970                         case 'i':
 1971                             sprintf(num,"%lld",current_file_offset);
 1972                             writes(num);
 1973                             break;
 1974                         case 'p':
 1975                             if(pf->f->const_data == NULL)
 1976                             {
 1977                                 sprintf(num,"%d",pf->f->position + (s->type[0] != SEPARATED ? 1 : 0));
 1978                                 writes(num);
 1979                             }
 1980                             break;
 1981                         case '%':
 1982                             writec('%');
 1983                             break;
 1984                         case 'n':
 1985                             writes(pf->f->name);
 1986                             break;
 1987                         case 'l':
 1988                         case 'L':
 1989                             if(pf->f->lookup != NULL)
 1990                             {
 1991                                 if(lookup_value == NULL)
 1992                                 {
 1993                                     field_start = write_pos;  // misuse write buffer for temp space for field value
 1994                                     switch(s->type[0])        // write trimmed data for search key
 1995                                     {
 1996                                         case FIXED_LENGTH:
 1997                                             print_fixed_field('t',pf->f,buffer);
 1998                                             break;
 1999                                         case SEPARATED:
 2000                                             print_separated_field('t',s->quote,s->type[1],pf->f,buffer);
 2001                                             break;
 2002                                         case BINARY:
 2003                                             print_binary_field('t',pf->f,buffer);
 2004                                             break;
 2005                                     }
 2006                                     writec(0);
 2007                                     lookup_value = make_lookup(pf->f->lookup,field_start);
 2008                                     write_pos = field_start;  // restore write buffer
 2009                                 }
 2010                             } else if(lookup_value ==  NULL)
 2011                             {
 2012                                 lookup_value = "";
 2013                             }
 2014 
 2015                         writes(lookup_value);
 2016 
 2017                         if(*d == 'L')
 2018                         {
 2019                             lookup_len = strlen(lookup_value);
 2020                             while(lookup_len++ < pf->f->length) writec(' ');
 2021                         }
 2022                         break;
 2023                     case 'h':
 2024                         if(s->type[0] == BINARY) print_binary_field(*d,pf->f,buffer);
 2025                         break;
 2026                     case 'd':
 2027                     case 't':
 2028                     case 'D':
 2029                     case 'C':
 2030                     case 'e':
 2031                     case 'x':
 2032                         field_start = write_pos;
 2033 
 2034                         if(pf->f->rep != NULL && !replacing) // start possible replacing
 2035                         {
 2036                             replacing = 1;
 2037                             just_replaced = 1;
 2038                             rep_pos = d;
 2039                             rep_start = write_pos;
 2040                             d = pf->f->rep->value;
 2041                             break; // end case
 2042                         } 
 2043                         
 2044                         switch(s->type[0])
 2045                         {
 2046                             case FIXED_LENGTH:
 2047                                 print_fixed_field(*d,pf->f,buffer);
 2048                                 break;
 2049                             case SEPARATED:
 2050                                 print_separated_field(*d,s->quote,s->type[1],pf->f,buffer);
 2051                                 break;
 2052                             case BINARY:
 2053                                 print_binary_field(*d,pf->f,buffer);
 2054                                 break;
 2055                         }
 2056 
 2057                         if(pf->f->f != NULL) make_conversion(pf->f->f,field_start);
 2058 
 2059                         if(!o->print_empty)
 2060                         {
 2061                             f = field_start;
 2062                             while(f < write_pos)
 2063                             {
 2064                                 if(strchr(o->empty_chars,*f) == NULL)
 2065                                 {
 2066                                     pf->empty = 0;
 2067                                     f = write_pos; // to stop while loop
 2068                                 }
 2069                                 f++;
 2070                             }
 2071                         }
 2072                         if(*d == 'e') write_pos = field_start;
 2073                         break;
 2074                     default:
 2075                         writec('%');
 2076                         writec(*d);
 2077                         break;
 2078                     }
 2079                     if(!just_replaced) d++;
 2080                 } else 
 2081                 {
 2082                     writec(*d);
 2083                     d++;
 2084                 }
 2085 
 2086 
 2087                 if(replacing && !*d)
 2088                 {
 2089                     d = rep_pos;
 2090                     d++;
 2091                     replacing = 0;
 2092                     if(*rep_pos == 'D' || *rep_pos == 'C')
 2093                     {
 2094                         if(write_pos - rep_start < pf->f->length)
 2095                         {
 2096                             while(write_pos - rep_start < pf->f->length) writec(' ');
 2097                         } else if(write_pos - rep_start > pf->f->length)
 2098                         {
 2099                             write_pos = rep_start + pf->f->length;
 2100                         }
 2101                     }
 2102                 }
 2103 
 2104                 if(just_replaced) just_replaced = 0;
 2105             }
 2106 
 2107             if(justify == RIGHT_JUSTIFY)
 2108             {
 2109                 pf->justify_length = (int) (write_pos - data_start);
 2110                 if (pf->justify_length > max_justify_len)
 2111                 {
 2112                     max_justify_len = pf->justify_length;
 2113                 }
 2114             }
 2115             writec(0);    // end of data marker
 2116         }
 2117         pf = pf->next;
 2118     }
 2119 
 2120     pf = r->pf;
 2121 
 2122     /* count the number of fields to be printed */
 2123     /* we need this before hand, because we must know if the is att least */
 2124     /* one field to be printed, then all separators must be printed */
 2125     while(pf != NULL) 
 2126     {
 2127         o = pf->f->o ? pf->f->o : r->o;
 2128 
 2129         if(o != no_output)
 2130         {
 2131             if(o->print_empty || !pf->empty)
 2132             {
 2133                 retval++;
 2134             }
 2135         }
 2136         pf = pf->next;
 2137     }
 2138 
 2139     pf = r->pf;
 2140 
 2141     while(pf != NULL && retval)
 2142     {
 2143         o = pf->f->o ? pf->f->o : r->o;
 2144 
 2145         if(o != no_output)
 2146         {
 2147             separator = o->separator;
 2148             indent = o->indent;
 2149             if(o->print_empty || !pf->empty)
 2150             {
 2151                 if(indent != NULL)
 2152                 {
 2153                     if(r->level)
 2154                     {
 2155                         int i;
 2156                         i = get_indent_depth(r->level->level);
 2157                         while(i--) fputs(indent,output_fp);
 2158                     } else
 2159                     {
 2160                         fputs(indent,output_fp);
 2161                         fputs(indent,output_fp);
 2162                     }
 2163                 }
 2164                 if((justify != LEFT_JUSTIFY && justify != RIGHT_JUSTIFY && max_justify_len) || justify == RIGHT_JUSTIFY)
 2165                 {
 2166                     i = max_justify_len - pf->justify_length;
 2167                     if(pf->justify_length > -1 && i)
 2168                     {
 2169                         while(i > JUSTIFY_STRING - 1)
 2170                         {
 2171                             justify_string[JUSTIFY_STRING - 1] = (uint8_t) 0;
 2172                             fputs(justify_string,output_fp);
 2173                             i -= JUSTIFY_STRING - 1;
 2174                         }
 2175                         justify_string[i] = (uint8_t) 0;
 2176                         fputs(justify_string,output_fp);
 2177                         justify_string[i] = ' ';
 2178                     }
 2179                 }
 2180                 fputs(pf->data,output_fp);
 2181             }
 2182             if(pf->next != NULL && separator != NULL) fputs(separator,output_fp);
 2183         }
 2184         pf = pf->next;
 2185     }
 2186     return retval;
 2187 }
 2188                 
 2189         
 2190 
 2191 /* make a list of printable fields 
 2192    if include list is non empty use names from it 
 2193    else use the whole field list in f */
 2194 struct print_field *
 2195 make_print_list(struct include_field *fl,struct field *f)
 2196 {
 2197     struct print_field *ret = NULL,*c = NULL;
 2198     struct field *n;
 2199 
 2200     if(fl == NULL)
 2201     {
 2202         while(f != NULL)
 2203         {
 2204             if(strcmp(f->name,"FILLER") != 0)
 2205             {
 2206                 if(ret == NULL)
 2207                 {
 2208                     ret = xmalloc(sizeof(struct print_field));
 2209                     c = ret;
 2210                 } else
 2211                 {
 2212                     c->next = xmalloc(sizeof(struct print_field));
 2213                     c = c->next;
 2214                 }
 2215                 c->f = f;
 2216                 c->next = NULL;
 2217             }
 2218             f = f->next;
 2219         }
 2220     } else
 2221     {
 2222         while(fl != NULL)
 2223         {
 2224             n = const_field;        // first check const list
 2225 
 2226             while(n != NULL && (strcasecmp(fl->name,n->name) != 0))
 2227             {
 2228                 n = n->next;
 2229             }
 2230 
 2231             if(n == NULL)           // if not found in const list check input fields
 2232             {
 2233                 n = f;
 2234                 while(n != NULL && (strcasecmp(fl->name,n->name) != 0))
 2235                 {
 2236                     n = n->next;
 2237                 }
 2238             }
 2239 
 2240             if(n != NULL) 
 2241             {
 2242                 if(ret == NULL)
 2243                 {
 2244                     ret = xmalloc(sizeof(struct print_field));
 2245                     c = ret;
 2246                 } else
 2247                 {
 2248                     c->next = xmalloc(sizeof(struct print_field));
 2249                     c = c->next;
 2250                 }
 2251                 c->f = n;
 2252                 c->next = NULL;
 2253                 fl->found = 1;
 2254             }
 2255             fl = fl->next;
 2256         }
 2257     }
 2258     return ret;
 2259 }
 2260 
 2261 /* check that all fields we found from current structure */
 2262 /* return the count of unmatched fields */
 2263 int
 2264 check_field_list(struct include_field *fl)
 2265 {
 2266     int ret = 0;
 2267 
 2268     while(fl != NULL)
 2269     {
 2270         if(!fl->found) 
 2271         {
 2272             if(!fl->reported) fprintf(stderr,"%s: Field '%s' was not found in the current structure\n",program,fl->name);
 2273             fl->reported = 1;
 2274             ret++;
 2275         }
 2276         fl = fl->next;
 2277     }
 2278     return ret;
 2279 }
 2280 
 2281 void
 2282 init_structure(struct structure *s,struct record *current_record,int length, uint8_t *buffer)
 2283 {
 2284     struct record *r;
 2285     struct field *f;
 2286     struct replace *rep;
 2287     struct expression *e;
 2288     int list_errors = 0;
 2289 
 2290     r = s->r;
 2291 
 2292     while(r != NULL)
 2293     {
 2294         f = r->f;
 2295         while(f != NULL)
 2296         {
 2297             if(s->header && f->name == NULL) 
 2298                 f->name = xstrdup(get_separated_field(f->position,s->quote,s->type,buffer));
 2299 
 2300             rep = replace;
 2301             while(rep != NULL)
 2302             {
 2303                 if(strcasecmp(f->name,rep->field) == 0)
 2304                 {
 2305                     f->rep = rep;
 2306                     rep->found = 1;
 2307                 }
 2308                 rep = rep->next;
 2309             }
 2310 
 2311             e = expression;
 2312             while(e != NULL)
 2313             {
 2314                 if(strcasecmp(f->name,e->field) == 0)
 2315                 {
 2316                     e->found = 1;
 2317                 }
 2318                 e = e->next;
 2319             }
 2320 
 2321             f = f->next;
 2322         }
 2323         r = r->next;
 2324     }
 2325 
 2326     r = s->r;
 2327 
 2328     while(r != NULL)
 2329     {
 2330         if(r->o != no_output) r->pf = make_print_list(r->o->fl,r->f);
 2331         r = r->next;
 2332     }
 2333 
 2334     r = s->r;
 2335 
 2336     while(r != NULL)
 2337     {
 2338         if(check_field_list(r->o->fl)) list_errors++;
 2339         r = r->next;
 2340     }
 2341     if(list_errors) panic("Some fields from field list were not found in the current structure or constant values",s->name,NULL);
 2342 
 2343     rep = replace;
 2344     while(rep != NULL)
 2345     {
 2346         if(!rep->found) panic("Field to be replaced was not found in the current structure",rep->field,NULL);
 2347         rep = rep->next;
 2348     }
 2349 
 2350     e = expression;
 2351     while(e != NULL)
 2352     {
 2353         if(!e->found) panic("Field in expression was not found in the current structure",e->field,NULL);
 2354         e = e->next;
 2355     }
 2356 }
 2357 
 2358 void
 2359 invalid_input(char *file, long int lineno,int strict,int length,int stype)
 2360 {
 2361     if(stype == BINARY)
 2362     {
 2363         if(strict)
 2364         {
 2365             fprintf(stderr,"%s: Invalid input block in file \'%s\', offset %lld",program,file,current_file_offset);
 2366             if(debug_lineno) fprintf(stderr,". Block was written to debug file");
 2367             fprintf(stderr,"\n"); 
 2368         }
 2369     } else
 2370     {
 2371         fprintf(stderr,"%s: Invalid input line in file \'%s\', line %ld, line length = %d",program,file,lineno,length);
 2372         if(debug_lineno) fprintf(stderr,", line %ld in debug file",debug_lineno);
 2373         fprintf(stderr,"\n");
 2374     }
 2375     if(strict) panic("Using option -l does not cause program to abort in case of invalid input",NULL,NULL);
 2376 }
 2377 
 2378 
 2379 static int
 2380 hash_scan_expression(struct expr_list **list,char *value,int casecmp)
 2381 {
 2382     register struct expr_list *l = list[hash(value,0)];
 2383 
 2384     while(l != NULL)
 2385     {
 2386         if(casecmp)
 2387         {
 2388             if(strcasecmp(l->value,value) == 0) return 1;
 2389         } else
 2390         {
 2391             if(strcmp(l->value,value) == 0) return 1;
 2392         }
 2393                 l = l->next;
 2394     }
 2395         return 0;
 2396 } 
 2397     
 2398 static int
 2399 hash_scan_expression_len(struct expr_list **list,char *value,int casecmp,size_t comp_len)
 2400 {
 2401     register struct expr_list *l = list[hash(value,comp_len)];
 2402 
 2403     while(l != NULL)
 2404     {
 2405         if(casecmp)
 2406         {
 2407             if(strncasecmp(l->value,value,l->value_len) == 0) return 1;
 2408         } else
 2409         {
 2410             if(strncmp(l->value,value,l->value_len) == 0) return 1;
 2411         }
 2412                 l = l->next;
 2413     }
 2414         return 0;
 2415 } 
 2416 
 2417 static inline int
 2418 scan_expression_list(struct expr_list *l,char *value,char op,int casecmp)
 2419 {
 2420     while(l != NULL)
 2421     {
 2422         switch(op)
 2423         {
 2424             case OP_START:
 2425                 if(casecmp)
 2426                 {
 2427                     if(strncasecmp(l->value,value,l->value_len) == 0) return 1;
 2428                 } else
 2429                 {
 2430                     if(strncmp(l->value,value,l->value_len) == 0) return 1;
 2431                 }
 2432                 break;
 2433             case OP_EQUAL:
 2434             case OP_NOT_EQUAL:
 2435                 if(casecmp)
 2436                 {
 2437                     if(strcasecmp(l->value,value) == 0) return 1;
 2438                 } else
 2439                 {
 2440                     if(strcmp(l->value,value) == 0) return 1;
 2441                 }
 2442                 break;
 2443             case OP_CONTAINS:
 2444                 if(casecmp)
 2445                 {
 2446                     if(strcasestr(value,l->value) != NULL) return 1;
 2447                 } else
 2448                 {
 2449                     if(strstr(value,l->value) != NULL) return 1;
 2450                 }
 2451                 break;
 2452 #ifdef HAVE_REGEX
 2453             case OP_REQEXP:
 2454                 if(regexec(&l->reg,value,(size_t) 0, NULL, 0) == 0) return 1;
 2455                 break;
 2456 #endif
 2457         }
 2458         l = l->next;
 2459     }
 2460     return 0;
 2461 }
 2462 
 2463 static int
 2464 full_scan_expression(struct expression *e,char *value,int casecmp)
 2465 {
 2466     register int i = 0;
 2467 
 2468 
 2469     if(e->fast_entries)
 2470     {
 2471         while(i < e->fast_entries)
 2472         {
 2473             if(scan_expression_list(e->expr_hash[e->fast_expr_hash[i]],value,e->op,casecmp)) return 1;
 2474             i++;
 2475         }
 2476     } else
 2477     {
 2478         while(i < MAX_EXPR_HASH)
 2479         {
 2480             if(scan_expression_list(e->expr_hash[i],value,e->op,casecmp)) return 1;
 2481             i++;
 2482         }
 2483     }
 2484     return 0;
 2485 } 
 2486 
 2487 /* returns true if and = 0 and attleast one expression is true 
 2488  * or and = 1 and all expressions are true
 2489  */
 2490 int
 2491 eval_expression(struct structure *s,struct record *r, int and,int invert, int casecmp, uint8_t *buffer)
 2492 {
 2493     struct expression *e = expression;
 2494     int retval = 0;
 2495     int prev_retval;
 2496     int loop_break = 0;
 2497     int expression_count = 0;
 2498     struct output *o;
 2499     size_t len;
 2500 
 2501     if(e == NULL) return 0;
 2502 
 2503     while(e != NULL && !loop_break)
 2504     {
 2505         if(e->f != NULL)
 2506         {
 2507 
 2508             o = e->f->o ? e->f->o : r->o;
 2509             if(o != no_output && o->hex_cap)
 2510             {
 2511                 bcd_to_ascii = bcd_to_ascii_cap;
 2512                 hex_to_ascii = hex_to_ascii_cap;
 2513             } else
 2514             {
 2515                 bcd_to_ascii = bcd_to_ascii_low;
 2516                 hex_to_ascii = hex_to_ascii_low;
 2517             }
 2518 
 2519 
 2520             start_write();
 2521             switch(s->type[0])
 2522             {
 2523                 case FIXED_LENGTH:
 2524                     print_fixed_field('d',e->f,buffer);
 2525                     break;
 2526                 case SEPARATED:
 2527                     print_separated_field('d',s->quote,s->type[1],e->f,buffer);
 2528                     break;
 2529                 case BINARY:
 2530                     print_binary_field('d',e->f,buffer);
 2531                     break;
 2532             }
 2533             writec(0);  // end of string
 2534             prev_retval = retval;
 2535 
 2536             switch(e->op)
 2537             {
 2538                 case OP_START:
 2539                     len = e->exp_max_len;
 2540                     prev_retval = retval;
 2541                     while(len >= e->exp_min_len && prev_retval == retval)
 2542                     {
 2543                         retval += hash_scan_expression_len(e->expr_hash,write_buffer,casecmp,len);
 2544                         len--;
 2545                     }
 2546                     break;
 2547                 case OP_CONTAINS:
 2548                 case OP_REQEXP:
 2549                     retval += full_scan_expression(e,write_buffer,casecmp);
 2550                     break;
 2551                 case OP_EQUAL:
 2552                     retval += hash_scan_expression(e->expr_hash,write_buffer,casecmp);
 2553                     break;
 2554                 case OP_NOT_EQUAL:
 2555                     if(hash_scan_expression(e->expr_hash,write_buffer,casecmp) == 0) retval++;
 2556                     break;
 2557             } 
 2558         }
 2559         if(!and && retval)
 2560         {
 2561             loop_break = 1;
 2562         }
 2563         expression_count++;
 2564         if(and && retval != expression_count)
 2565         {
 2566             loop_break = 1;
 2567         }
 2568         e = e->next;
 2569     }
 2570     if(and)
 2571     {
 2572         if (retval == expression_count)
 2573         {
 2574             retval = 1;
 2575         } else
 2576         {
 2577             retval = 0;
 2578         }
 2579     }
 2580     if(invert) retval = !retval;
 2581     return retval;
 2582 }
 2583 /* initialize expressions field pointers */
 2584 /* pointers are preinitialized for faster execution */
 2585     void
 2586 init_expression_list(struct record *r)
 2587 {
 2588     struct expression *e;
 2589     struct field *f = r->f;
 2590     int exp_count = 0;
 2591 
 2592     e = expression;
 2593 
 2594     while(e != NULL)
 2595     {
 2596         e->f = NULL;
 2597         e = e->next;
 2598         exp_count++;
 2599     }
 2600 
 2601     e = expression;
 2602 
 2603     while(f != NULL && exp_count)
 2604     {
 2605         e = expression;
 2606         while(e != NULL && exp_count)
 2607         {
 2608             if(e->f == NULL && strcasecmp(f->name,e->field) == 0)
 2609             {
 2610                 e->f = f;
 2611                 exp_count--;
 2612             }
 2613             e = e->next;
 2614         }
 2615         f = f->next;
 2616     }
 2617 }
 2618 
 2619     void
 2620 open_debug_file(int stype)
 2621 {
 2622     sprintf(debug_file,"ffe_error_%d.log",(int) getpid());
 2623     if(stype == BINARY)
 2624     {
 2625         debug_fp = xfopenb(debug_file,"w");
 2626     } else
 2627     {
 2628         debug_fp = xfopen(debug_file,"w");
 2629     }
 2630 }
 2631 
 2632 
 2633 void
 2634 write_debug_file(uint8_t *line,int len,int stype)
 2635 {
 2636     if(debug_fp == NULL) open_debug_file(stype);
 2637 
 2638     if(stype == BINARY)
 2639     {
 2640         fwrite(line,len,1,debug_fp);
 2641     } else if(line != NULL)
 2642     {
 2643         while(*line != '\n') fputc(*line++,debug_fp);
 2644         fputs("\n",debug_fp);
 2645     }
 2646     debug_lineno++;
 2647 }
 2648 
 2649 static void
 2650 select_output(struct output *o)
 2651 {
 2652     if(o->ofp != NULL)
 2653     {
 2654         output_fp = o->ofp;
 2655         output_file = o->output_file;
 2656     } else
 2657     {
 2658         output_fp = default_output_fp;
 2659         output_file = default_output_file;
 2660     }
 2661 }
 2662 
 2663 
 2664 /* main loop for execution */
 2665 void 
 2666 execute(struct structure *s,int strict, int expression_and,int expression_invert,int expression_case, int debug,char *anon_to_use)
 2667 {
 2668     uint8_t *input_line;
 2669     struct record *r = NULL;
 2670     struct record *prev_record = NULL;
 2671     int length;
 2672     int header_printed = 0;
 2673     int fields_printed;
 2674     int first_line = 1;
 2675     int i;
 2676     int anon_field_count=0;
 2677 
 2678     current_file_lineno = 0;
 2679     current_total_lineno = 0;
 2680     current_file_name = files->name;
 2681     headers = s->header;
 2682 
 2683     i = 0;
 2684     while(i < JUSTIFY_STRING) justify_string[i++] = ' ';
 2685 
 2686     reset_levels(0,MAXLEVEL);
 2687 
 2688     write_buffer = xmalloc(write_buffer_size);
 2689     write_buffer_end = write_buffer + (write_buffer_size - 1);
 2690 
 2691     select_output(s->o);
 2692     print_text(s,NULL,s->o->file_header);
 2693     while((input_line = get_input_line(&length,s->type[0])) != NULL)
 2694     {
 2695         prev_record = r;
 2696         r = select_record(s,length,input_line);
 2697         if(r == NULL) 
 2698         {
 2699             if(debug) write_debug_file(input_line,length,s->type[0]);
 2700             invalid_input(current_file_name,current_file_lineno,strict,length,s->type[0]);
 2701             if(s->type[0] == BINARY)
 2702             {
 2703                 last_consumed = 1; /* advance one byte and check next block, this is a rather expensive way to read ahead... */
 2704             }
 2705 
 2706         } else
 2707         {
 2708             if(first_line)
 2709             {
 2710                 init_structure(s,r,length,input_line);
 2711                 anon_field_count = update_anon_info(s,anon_to_use);
 2712                 if(anon_field_count) init_libgcrypt();
 2713 
 2714             }
 2715 
 2716             if(expression != NULL && (prev_record != r || prev_record == NULL))
 2717             {
 2718                 init_expression_list(r);
 2719             }
 2720             
 2721             last_consumed = update_field_positions(s->type,s->quote,r,length,input_line);
 2722 
 2723             if((!first_line || !headers) && r->o != no_output)
 2724             {
 2725                 if((r->pf == NULL && r->o->no_data == 1) || r->pf != NULL || r->o == raw)
 2726                 {
 2727                     if(expression == NULL || (eval_expression(s,r,expression_and,expression_invert,expression_case,input_line)))
 2728                     {
 2729                         if(anon_field_count) anonymize_fields(s->type,s->quote,r,length,input_line);  // anonymize after exp. evaluation
 2730                         if(r->o == raw)
 2731                         {
 2732                             print_raw(s->type[0] == BINARY ? last_consumed : length,input_line,s->type[0]);
 2733                         } else
 2734                         {
 2735                             select_output(r->o);
 2736                             print_level_before(prev_record,r);
 2737                             if(r->o->header != NULL && !header_printed) 
 2738                             {
 2739                                 header_printed = print_header(s,r);
 2740                             }
 2741                             if(r->o->indent != NULL && r->o->record_header != NULL) 
 2742                                 print_indent(r->o->indent,r->level != NULL ? r->level->level : 1);
 2743                             print_text(s,r,r->o->record_header);
 2744                             fields_printed = print_fields(s,r,input_line);
 2745                             if(fields_printed || r->o->print_empty)
 2746                             {
 2747                                 if(r->o->indent != NULL && r->o->record_trailer != NULL) 
 2748                                     print_indent(r->o->indent,r->level != NULL ? r->level->level : 1);
 2749                                 print_text(s,r,r->o->record_trailer);
 2750                             }
 2751                         }
 2752                     }
 2753                 }
 2754             } 
 2755 
 2756             if(first_line) {
 2757                 first_line = 0;
 2758                 if(headers) r = NULL;
 2759             }
 2760         }
 2761     }
 2762     print_level_end(r);
 2763     select_output(s->o);
 2764     print_text(s,r,s->o->file_trailer);
 2765     free(write_buffer);
 2766     if(debug_fp != NULL) fclose(debug_fp);
 2767 }
 2768 
 2769 
 2770