"Fossies" - the Fresh Open Source Software Archive

Member "tin-2.4.3/src/art.c" (18 Dec 2018, 89022 Bytes) of package /linux/misc/tin-2.4.3.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "art.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.4.2_vs_2.4.3.

    1 /*
    2  *  Project   : tin - a Usenet reader
    3  *  Module    : art.c
    4  *  Author    : I.Lea & R.Skrenta
    5  *  Created   : 1991-04-01
    6  *  Updated   : 2018-12-18
    7  *  Notes     :
    8  *
    9  * Copyright (c) 1991-2019 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
   10  * All rights reserved.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. The name of the author may not be used to endorse or promote
   21  *    products derived from this software without specific prior written
   22  *    permission.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
   25  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
   28  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
   30  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
   32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
   33  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
   34  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   35  */
   36 
   37 
   38 #ifndef TIN_H
   39 #   include "tin.h"
   40 #endif /* !TIN_H */
   41 #ifndef TCURSES_H
   42 #   include "tcurses.h"
   43 #endif /* !TCURSES_H */
   44 #ifndef NEWSRC_H
   45 #   include "newsrc.h"
   46 #endif /* !NEWSRC_H */
   47 
   48 #ifndef STPWATCH_H
   49 #   include "stpwatch.h"
   50 #endif /* !STPWATCH_H */
   51 
   52 /*
   53  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
   54  */
   55 #define SortBy(func)    tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
   56 
   57 int top_art = 0;                /* # of articles in arts[] */
   58 
   59 /*
   60  * Local prototypes
   61  */
   62 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
   63 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
   64 static char *find_nov_file(struct t_group *group, int mode);
   65 static char *print_date(time_t secs);
   66 static char *print_from(struct t_group *group, struct t_article *article, int charset);
   67 static int artnum_comp(t_comptype p1, t_comptype p2);
   68 static int base_comp(t_comptype p1, t_comptype p2);
   69 static int date_comp_asc(t_comptype p1, t_comptype p2);
   70 static int date_comp_desc(t_comptype p1, t_comptype p2);
   71 static int from_comp_asc(t_comptype p1, t_comptype p2);
   72 static int from_comp_desc(t_comptype p1, t_comptype p2);
   73 static int global_get_multiparts(int aindex, MultiPartInfo **malloc_and_setme_info);
   74 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
   75 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
   76 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
   77 static int lines_comp_asc(t_comptype p1, t_comptype p2);
   78 static int lines_comp_desc(t_comptype p1, t_comptype p2);
   79 static int read_art_headers(struct t_group *group, int total, t_artnum top);
   80 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
   81 static int score_comp_asc(t_comptype p1, t_comptype p2);
   82 static int score_comp_desc(t_comptype p1, t_comptype p2);
   83 static int score_comp_base(t_comptype p1, t_comptype p2);
   84 static int subj_comp_asc(t_comptype p1, t_comptype p2);
   85 static int subj_comp_desc(t_comptype p1, t_comptype p2);
   86 static int valid_artnum(t_artnum art);
   87 static t_artnum find_first_unread(struct t_group *group);
   88 static t_artnum setup_hard_base(struct t_group *group);
   89 static t_bool parse_headers(FILE *fp, struct t_article *h);
   90 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
   91 static time_t get_last_posting_date(long n);
   92 static void sort_base(unsigned int sort_threads_type);
   93 static void thread_by_multipart(void);
   94 static void thread_by_percentage(struct t_group *group);
   95 static void thread_by_subject(void);
   96 static void write_overview(struct t_group *group);
   97 #ifdef NNTP_ABLE
   98     static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
   99     static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
  100 #endif /* NNTP_ABLE */
  101 
  102 
  103 /*
  104  * Display a suitable 'entering group' message if screen needs redrawing
  105  * Allow for the non-printing %s, and the %-age counter
  106  */
  107 void
  108 show_art_msg(
  109     const char *group)
  110 {
  111     wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
  112 }
  113 
  114 
  115 /*
  116  * Construct the pointers to the first (base) article in each thread.
  117  * If we are showing only unread, then point to the first unread. I have
  118  * no idea why this should be so, it causes problems elsewhere [which_response]
  119  */
  120 void
  121 find_base(
  122     struct t_group *group)
  123 {
  124     int i, j;
  125 
  126     grpmenu.max = 0;
  127 
  128 #ifdef DEBUG
  129     debug_print_arts();
  130 #endif /* DEBUG */
  131 
  132     for_each_art(i) {
  133         /*
  134          * .prev will be set on each article that is after the first article in
  135          * the thread. Invalid articles which have been expired will have
  136          * .thread set to ART_EXPIRED
  137          */
  138         if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
  139             continue;
  140 
  141         if (grpmenu.max >= max_base)
  142             expand_base();
  143 
  144         if (group->attribute->show_only_unread_arts) {
  145             if (arts[i].status != ART_READ || arts[i].keep_in_base)
  146                 base[grpmenu.max++] = i;
  147             else {
  148                 /* Find 1st unread art in thread */
  149                 for (j = i; j >= 0; j = arts[j].thread) {
  150                     if (arts[j].status != ART_READ || arts[j].keep_in_base) {
  151                         base[grpmenu.max++] = i;
  152                         break;
  153                     }
  154                 }
  155             }
  156         } else
  157             base[grpmenu.max++] = i;
  158     }
  159     /* sort base[] */
  160     if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
  161         sort_base(group->attribute->sort_threads_type);
  162 }
  163 
  164 
  165 /*
  166  * Longword comparison routine for the tin_sort()
  167  */
  168 static int
  169 base_comp(
  170     t_comptype p1,
  171     t_comptype p2)
  172 {
  173     const t_artnum *a = (const t_artnum *) p1;
  174     const t_artnum *b = (const t_artnum *) p2;
  175 
  176     if (*a < *b)
  177         return -1;
  178 
  179     if (*a > *b)
  180         return 1;
  181 
  182     return 0;
  183 }
  184 
  185 
  186 /*
  187  * via NNTP:
  188  *   Issue a LISTGROUP command
  189  *   Read the article numbers existing in the group into base[]
  190  *   If the LISTGROUP failed, issue a GROUP command. Use the results to
  191  *   create a less accurate version of base[]
  192  *   This data will already be sorted
  193  *
  194  * on local spool:
  195  *   Read the spool dir to populate base[] as above. Sort it.
  196  *
  197  * Grow the arts[] and bitmaps as needed.
  198  * NB: the output will be sorted on artnum
  199  *
  200  * grpmenu.max is one past top.
  201  * Returns total number of articles in group, or -1 on error
  202  */
  203 static t_artnum
  204 setup_hard_base(
  205     struct t_group *group)
  206 {
  207     t_artnum total = 0;
  208 
  209     grpmenu.max = 0;
  210 
  211     /*
  212      * If reading with NNTP, issue a LISTGROUP
  213      */
  214     if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
  215 #ifdef NNTP_ABLE
  216         char buf[NNTP_STRLEN];
  217         char line[NNTP_STRLEN];
  218         int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  219         FILE *fp;
  220         t_artnum last, start, count = 0, j = 0;
  221         static t_bool skip_listgroup = FALSE;
  222 
  223         /*
  224          * Some nntp servers are broken and need an extra GROUP command
  225          * (reported by reorx@irc.pl). This affects (old?) versions of
  226          * nntpcache, leafnode and SurgeNews. Usually this should not be
  227          * needed.
  228          *
  229          * For getart_limit recheck lowwatermark as at least giganews gives
  230          * very different results for LIST ACTIVE (3 year retention for all)
  231          * and GROUP (based on the clients contract).
  232          * Calculate range and prepare base[] not to lose unread arts.
  233          */
  234         if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
  235             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  236             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  237                 return -1;
  238 
  239             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
  240                 return -1;
  241 
  242             if (getart_limit > 0) {
  243                 j = group->xmax - getart_limit;
  244                 count = MAX(find_first_unread(group), start);
  245             }
  246             if (getart_limit < 0) {
  247                 j = getart_limit + find_first_unread(group);
  248                 count = group->xmin;
  249             }
  250             if (j < group->xmin)
  251                 j = group->xmin;
  252 
  253             for (; count < j; count++) {
  254                 if (grpmenu.max >= max_base)
  255                     expand_base();
  256                 base[grpmenu.max++] = count;
  257             }
  258         }
  259 
  260         /*
  261          * See if LISTGROUP works
  262          */
  263         if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
  264             if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
  265                 /* RFC 3977 allows ranges in LISTGROUP */
  266                 if (getart_limit > 0)
  267                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
  268                 else /* getart_limit < 0; fetch till newest art */
  269                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
  270 
  271             } else /* for RFC 977 just use GROUP */
  272                 skip_listgroup = TRUE;
  273 
  274         } else /* get all article numbers */
  275             snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
  276 
  277         if (!skip_listgroup) {
  278             if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
  279                 char *ptr;
  280 
  281 #   ifdef DEBUG
  282                 if ((debug & DEBUG_NNTP) && verbose > 1)
  283                     debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  284 #   endif /* DEBUG */
  285 
  286                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
  287                     if (grpmenu.max >= max_base)
  288                         expand_base();
  289                     base[grpmenu.max++] = atoartnum(ptr);
  290                     total++;
  291                 }
  292 
  293                 if (tin_errno)
  294                     return -1;
  295             } else
  296                 skip_listgroup = TRUE;
  297         }
  298 
  299         if (skip_listgroup) { /* LISTGROUP was skipped or failed */
  300             /*
  301              * Handle the obscure case that the user aborted before the LISTGROUP
  302              * had a chance to respond
  303              */
  304             if (tin_errno)
  305                 return -1;
  306 
  307             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  308             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  309                 return -1;
  310 
  311             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
  312                 return -1;
  313 
  314 #   ifdef DEBUG
  315             if ((debug & DEBUG_NNTP) && verbose > 1)
  316                 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  317 #   endif /* DEBUG */
  318             total = count;
  319             grpmenu.max = 0;
  320             if (getart_limit > 0) {
  321                 if ((j = find_first_unread(group)) > start) {
  322                     if (group->xmax > getart_limit) {
  323                         start = MIN(j, group->xmax - getart_limit);
  324                         total = getart_limit;
  325                     } else
  326                         start = j;
  327                 }
  328             }
  329             if (getart_limit < 0) {
  330                 if ((j = (getart_limit + find_first_unread(group))) > start)
  331                     start = j;
  332             }
  333             while (start <= last) {
  334                 if (grpmenu.max >= max_base)
  335                     expand_base();
  336                 base[grpmenu.max++] = start++;
  337             }
  338         }
  339 #endif /* NNTP_ABLE */
  340     /*
  341      * Reading off local spool, read the directory files
  342      */
  343     } else {
  344         DIR *d;
  345         DIR_BUF *e;
  346         char group_path[PATH_LEN];
  347         t_artnum art;
  348 
  349         make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
  350 
  351         if ((d = opendir(group_path)) != NULL) {
  352             while ((e = readdir(d)) != NULL) {
  353                 art = atoartnum(e->d_name);
  354                 if (art >= 1) {
  355                     total++;
  356                     if (grpmenu.max >= max_base)
  357                         expand_base();
  358                     base[grpmenu.max++] = art;
  359                 }
  360             }
  361             CLOSEDIR(d);
  362             tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
  363         } else {
  364             perror_message(_(txt_cannot_open), group_path);
  365 #if 0
  366             if (access(group_path, R_OK) != 0)
  367                 error_message(2, _(txt_not_exist));
  368 #endif /* 0 */
  369             return -1;
  370         }
  371     }
  372 
  373     if (grpmenu.max) {
  374         if (base[grpmenu.max - 1] > group->xmax)
  375             group->xmax = base[grpmenu.max - 1];
  376         expand_bitmap(group, base[0]);
  377     }
  378 
  379     return total;
  380 }
  381 
  382 
  383 /*
  384  * Main group indexing routine.
  385  *
  386  * Will read any existing index, create or incrementally update
  387  * the index by looking at the articles in the spool directory,
  388  * and attempt to write a new index if necessary.
  389  *
  390  * Returns FALSE if the user aborted the indexing, otherwise TRUE
  391  */
  392 t_bool
  393 index_group(
  394     struct t_group *group)
  395 {
  396     int i;
  397     int changed;                /* Count of articles whose overview has changed */
  398     int getart_limit;
  399     int respnum;
  400     int total;
  401     t_artnum last_read_article;
  402     t_artnum min, new_min, max;
  403     t_bool caching_xover;
  404     t_bool filtered;
  405     t_bool path_in_nov = FALSE;
  406     t_bool rebuild_cache = FALSE;
  407 
  408     if (group == NULL)
  409         return TRUE;
  410 
  411     if (!batch_mode)
  412         show_art_msg(group->name);
  413 
  414     signal_context = cArt;          /* Set this only once curr_group is valid */
  415 
  416     hash_reclaim();
  417     free_art_array();
  418     free_msgids();
  419 
  420     BegStopWatch("setup_hard_base()");
  421 
  422     /*
  423      * Get list of valid article numbers
  424      */
  425     if (setup_hard_base(group) < 0)
  426         return FALSE;
  427 
  428     EndStopWatch();
  429     PrintStopWatch();
  430 
  431 #ifdef DEBUG
  432     if (debug & DEBUG_NEWSRC) {
  433         debug_print_comment("Before read_overview");
  434         debug_print_bitmap(group, NULL);
  435     }
  436 #endif /* DEBUG */
  437 
  438     min = grpmenu.max ? base[0] : group->xmin;
  439     max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
  440 
  441     getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  442 
  443     if (getart_limit > 0) {
  444         if (grpmenu.max && (grpmenu.max > getart_limit))
  445             min = base[grpmenu.max - getart_limit];
  446         else
  447             getart_limit = 0;
  448     } else if (getart_limit < 0) {
  449         t_artnum first_unread = find_first_unread(group);
  450 
  451         if (min - first_unread < getart_limit)
  452             min = first_unread + getart_limit;
  453         else
  454             getart_limit = 0;
  455     }
  456 
  457     /*
  458      * Quit now if no articles
  459      */
  460     if (max < 0)
  461         return FALSE;
  462 
  463     top_art = 0;
  464     last_read_article = T_ARTNUM_CONST(0);
  465 
  466     /*
  467      * Read in the existing overview data for min..max
  468      * This read has local=TRUE set if locally caching XOVER records to ensure
  469      * we pull in any private overview caches in preference to using OVER
  470      *
  471      * When reading local spool, this will pull in the system wide overview
  472      * cache (if found) otherwise the private overview cache will be read
  473      */
  474     caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
  475     if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
  476         return FALSE;   /* user aborted indexing */
  477 
  478     /*
  479      * Fill in the range last_read_article...max using XOVER
  480      * Only do this if the previous read_overview() was against private cache
  481      */
  482     if ((last_read_article < max) && caching_xover) {
  483         new_min = (last_read_article >= min) ? last_read_article + 1 : min;
  484 
  485         if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
  486             return FALSE;   /* user aborted indexing */
  487         else
  488             changed += i;
  489     } else
  490         caching_xover = FALSE;
  491 
  492     /*
  493      * Mark as UNTHREADED all articles that have been verified as valid
  494      * Get num of new arts to index so the user will have an idea of index time
  495      */
  496     for (i = 0, total = 0; i < grpmenu.max; i++) {
  497         if ((respnum = valid_artnum(base[i])) >= 0) {
  498             arts[respnum].thread = ART_UNTHREADED;
  499             continue;
  500         }
  501         if (base[i] <= last_read_article)       /* It is vital this test be done last */
  502             continue;
  503         total++;
  504     }
  505 
  506     /*
  507      * Add any articles to arts[] that are new or were killed
  508      */
  509     if (total > 0) {
  510         new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
  511 
  512         if ((i = read_art_headers(group, total, new_min)) == -1)
  513             return FALSE;       /* user aborted indexing */
  514         else
  515             changed += i;
  516     }
  517 
  518 #ifdef DEBUG
  519     if (debug & DEBUG_NEWSRC) {
  520         debug_print_comment("Before parse_unread_arts()");
  521         debug_print_bitmap(group, NULL);
  522     }
  523 #endif /* DEBUG */
  524     /*
  525      * Do this before calling art_mark(,, ART_READ) if you want
  526      * the unread count to be correct.
  527      */
  528     min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
  529     parse_unread_arts(group, min);
  530 #ifdef DEBUG
  531     if (debug & DEBUG_NEWSRC) {
  532         debug_print_comment("After parse_unread_arts()");
  533         debug_print_bitmap(group, NULL);
  534     }
  535 #endif /* DEBUG */
  536 
  537     /*
  538      * Stat all articles to see if any have expired
  539      */
  540     for_each_art(i) {
  541         if (arts[i].thread == ART_EXPIRED) {
  542             changed++;
  543 #ifdef DEBUG
  544             if (debug & DEBUG_NEWSRC)
  545                 debug_print_comment("art.c: index_group() purging...");
  546 #endif /* DEBUG */
  547             art_mark(group, &arts[i], ART_READ);
  548             if (group->attribute->show_only_unread_arts)
  549                 arts[i].keep_in_base = FALSE;
  550         }
  551         if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
  552             path_in_nov = TRUE;
  553     }
  554 
  555     /*
  556      * Only rewrite the index if it has changed
  557      * TODO review the exact logic behind "|| caching_xover"
  558      */
  559     if (changed || caching_xover || rebuild_cache)
  560         write_overview(group);
  561 
  562     /*
  563      * Create the reference tree. The msgid and ref ptrs will
  564      * be free()d now that the NovFile has been written.
  565      */
  566     build_references(group);
  567 
  568     /*
  569      * Needs access to the reference tree
  570      */
  571     filtered = filter_articles(group);
  572 
  573     BegStopWatch("make_threads()");
  574 
  575     /*
  576      * Thread the group
  577      */
  578     make_threads(group, FALSE);
  579 
  580     EndStopWatch();
  581     PrintStopWatch();
  582 
  583     if ((changed > 0 || filtered) && !batch_mode)
  584         clear_message();
  585 
  586     return TRUE;
  587 }
  588 
  589 
  590 /*
  591  * Returns number of first unread article
  592  */
  593 static t_artnum
  594 find_first_unread(
  595     struct t_group *group)
  596 {
  597     unsigned char *p;
  598     unsigned char *end = group->newsrc.xbitmap;
  599     t_artnum first = group->newsrc.xmin; /* initial value */
  600 
  601     if ((p = group->newsrc.xbitmap)) {
  602         end += group->newsrc.xbitlen / NBITS;
  603         for (; *p == '\0' && p < end; p++, first += NBITS)
  604             ;
  605     }
  606     return first;
  607 }
  608 
  609 
  610 /*
  611  * Open an article for reading just the header
  612  * 'next' is used/updated with the next article number
  613  * to optimise the number of 'HEAD' commands issued on
  614  * groups with holes.
  615  */
  616 static FILE *
  617 open_art_header(
  618     char *groupname,
  619     t_artnum art,
  620     t_artnum *next)
  621 {
  622     char buf[NNTP_STRLEN];
  623 #ifdef NNTP_ABLE
  624     FILE *fp;
  625     int i;
  626 
  627     if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
  628         /*
  629          * Don't bother requesting if we have not got there yet.
  630          * This is a big win if the group has got holes in it (ie. if 000's
  631          * of articles have expired between active files min & max values).
  632          */
  633         if (art < *next)
  634             return NULL;
  635 
  636         snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  637         if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  638             return fp;
  639 
  640         /*
  641          * TODO:
  642          * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
  643          * "503 Access denied" instead of 480 but NEXT still works,
  644          * so tin loops over all articles without getting useful data
  645          */
  646 
  647         /*
  648          * HEAD failed, try to find NEXT
  649          * Should return "223 artno message-id more text...."
  650          */
  651         i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
  652         switch (i) {
  653             case OK_NOTEXT:
  654                 *next = atoartnum(buf);     /* Set next art number */
  655                 break;
  656 
  657 #   ifndef BROKEN_LISTGROUP
  658             /*
  659              * might happen if LISTGROUP doesn't select group, but
  660              * we are not -DBROKEN_LISTGROUP
  661              */
  662             case ERR_NCING:
  663                 nntp_caps.broken_listgroup = TRUE;
  664                 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
  665                 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
  666                     return NULL;
  667                 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  668                 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  669                     return fp;
  670                 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
  671                     *next = atoartnum(buf);
  672                 break;
  673 #   endif /* !BROKEN_LISTGROUP */
  674 
  675             default:
  676                 /*
  677                  * TODO: abort loop over all arts on ERR_NONEXT
  678                  */
  679 #   ifndef BROKEN_LISTGROUP
  680                 /*
  681                  * to avoid out of sync responses
  682                  * (listgroup seems to work, but didn't select new group,
  683                  *  so xover seems to work but returns old data)
  684                  * we set listgroup_broken = TRUE; once we saw a
  685                  * ERR_NOARTIG / ERR_NONEXT or the like - even if
  686                  * ERR_NOARTIG may occur on servers where listgroup
  687                  * isn't broken...
  688                  */
  689                 nntp_caps.broken_listgroup = TRUE;
  690 #   endif /* !BROKEN_LISTGROUP */
  691                 break;
  692         }
  693 
  694         return NULL;
  695     }
  696 #endif /* NNTP_ABLE */
  697 
  698     snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
  699     return (fopen(buf, "r"));
  700 }
  701 
  702 
  703 /*
  704  * Called after XOVER/local/private overview databases have been loaded
  705  * Read and parse in headers for any arts not already found (usually
  706  * new articles that have not been indexed yet)
  707  * Any new articles that are added have ->thread set to ART_UNTHREADED
  708  * 'top' is the current highest artnum read
  709  *
  710  * Return values are:
  711  *   >0   Number of additional articles read in
  712  *    0   No additional (new) articles were found
  713  *   -1   user aborted during read
  714  */
  715 static int
  716 read_art_headers(
  717     struct t_group *group,
  718     int total,
  719     t_artnum top)
  720 {
  721     FILE *fp;
  722     char dir[PATH_LEN];
  723     char group_msg[LEN];
  724     int i;
  725     int modified = 0;
  726     t_artnum art;
  727     t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
  728     t_bool res;
  729 
  730     /*
  731      * Change to groups spooldir to optimize fopen()'s on local articles
  732      * NB open_art_header() requires this
  733      */
  734     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
  735         char buf[PATH_LEN];
  736 
  737         get_cwd(dir);
  738         make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
  739         if (chdir(buf) != 0) {
  740 #ifdef DEBUG
  741             if (debug & DEBUG_MISC)
  742                 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
  743 #endif /* DEBUG */
  744             return -1;
  745         }
  746     }
  747 
  748     snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
  749 
  750     for (i = 0; i < grpmenu.max; i++) { /* for each article number */
  751         art = base[i];
  752 
  753         /*
  754          * Skip articles that are below the low water mark or are
  755          * already present
  756          */
  757         if (valid_artnum(art) >= 0)
  758             continue;
  759         if (art <= top)
  760             continue;
  761 
  762         /*
  763          * Try and open the article
  764          */
  765         if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
  766             continue;
  767 
  768         /*
  769          * Add article to arts[]
  770          */
  771         if (top_art >= max_art)
  772             expand_art();
  773 
  774         set_article(&arts[top_art]);
  775         arts[top_art].artnum = art;
  776         arts[top_art].thread = ART_UNTHREADED;
  777 
  778         res = parse_headers(fp, &arts[top_art]);
  779 
  780         TIN_FCLOSE(fp);
  781         if (tin_errno) {
  782             modified = -1;
  783             break;
  784         }
  785 
  786         if (!res) {
  787 #ifdef DEBUG
  788             if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
  789                 char buf[PATH_LEN];
  790 
  791                 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
  792                 debug_print_file("ARTS", "read_art_headers() %s", buf);
  793             }
  794 #endif /* DEBUG */
  795             arts[top_art].artnum = T_ARTNUM_CONST(0);
  796             arts[top_art].date = (time_t) 0;
  797             FreeAndNull(arts[top_art].xref);
  798             FreeAndNull(arts[top_art].path);
  799             FreeAndNull(arts[top_art].refs);
  800             FreeAndNull(arts[top_art].msgid);
  801             if (arts[top_art].archive) {
  802                 FreeAndNull(arts[top_art].archive->partnum);
  803                 FreeAndNull(arts[top_art].archive);
  804             }
  805             arts[top_art].tagged = 0;
  806             arts[top_art].thread = ART_EXPIRED;
  807             arts[top_art].prev = ART_NORMAL;
  808             arts[top_art].status = ART_UNREAD;
  809             arts[top_art].killed = ART_NOTKILLED;
  810             arts[top_art].selected = FALSE;
  811             continue;
  812         }
  813 
  814         top = arts[top_art].artnum; /* used if arts are killed */
  815         top_art++;
  816 
  817         if (++modified % MODULO_COUNT_NUM == 0)
  818             show_progress(group_msg, modified, total);
  819     }
  820 
  821     /*
  822      * Change back to previous dir before indexing started
  823      */
  824     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
  825         chdir(dir);
  826 
  827     return modified;
  828 }
  829 
  830 
  831 /*
  832  * The algorithm is elegant, using the fact that identical Subject lines
  833  * are hashed to the same node in table[] (see hashstr.c)
  834  *
  835  * Mark i as being in j's thread list if
  836  * . The article is _not_ being ignored
  837  * . The article is not already threaded
  838  * . One of the following is true:
  839  *    1) The subject lines are the same
  840  *    2) Both are part of the same archive (name's match and arch bit set)
  841  * IMHO the tests for archive name are redundant and have been for years
  842  */
  843 static void
  844 thread_by_subject(
  845     void)
  846 {
  847     int i, j;
  848     struct t_hashnode *h;
  849 
  850     for_each_art(i) {
  851         if (IGNORE_ART_THREAD(i))
  852             continue;
  853 
  854         /*
  855          * Get the contents of the magic marker in the hashnode
  856          */
  857         h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
  858 
  859         j = h->aptr;
  860 
  861         if (j != -1 && j < i) {
  862 #if 1
  863             if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
  864 #else
  865             /* see also refs.c:collate_subjects() */
  866             if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
  867 #endif /* 1 */
  868             {
  869                 arts[j].thread = i;
  870                 arts[i].prev = j;
  871             }
  872         }
  873 
  874         /*
  875          * Update the magic marker with the highest numbered mesg in
  876          * arts[] that has been used in this thread so far
  877          */
  878         h->aptr = i;
  879     }
  880 
  881 #if 0
  882     fprintf(stderr, "Subj dump\n");
  883     fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
  884     for_each_art(i) {
  885         fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
  886             (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
  887             (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
  888             (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
  889             arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
  890     }
  891 #endif /* 0 */
  892 }
  893 
  894 
  895 /*
  896  * This Threading algorithm threads articles into 'buckets' where each bucket
  897  * contains all the articles which match the root article's subject line to
  898  * the configured percentage. Eg, if the root article had the subject "asdf"
  899  * and the match percentage was configured to be 75% then any article would
  900  * match if its subject was no different in more than a single character.
  901  */
  902 static void
  903 thread_by_percentage(
  904     struct t_group *group)
  905 {
  906     int i, j, k;
  907     int root_num = 0; /* The index number of the root we are currently working on. */
  908     unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
  909     unsigned int percentage = 100 - group->attribute->thread_perc;
  910     size_t slen;
  911 
  912     /* First we need to sort art[] to simplify and speed up the matching. */
  913     SortBy(subj_comp_asc);
  914 
  915     /*
  916      * Now we put all the articles which match enough into the thread. If
  917      * an article doesn't match enough we create a new thread and then add
  918      * to that and so on.
  919      */
  920     base[0] = 0;
  921     arts[0].prev = ART_NORMAL;
  922     for_each_art(i) {
  923         if (i == 0)
  924             continue;
  925 
  926         /* Check each character to see if it matched enough */
  927         k = 0;
  928         unmatched = 0;
  929         for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
  930             if (arts[base[root_num]].subject[j] != arts[i].subject[k])
  931                 unmatched++;
  932         }
  933 
  934         /*
  935          * By getting here we have a number of unmatched characters
  936          * between the two strings. We also have the length of the
  937          * strings available to us easily.
  938          * All we need to do is see if the match is good enough, but
  939          * we count differences in the length of the strings against
  940          * them matching.
  941          */
  942         if (!(slen = strlen(arts[base[root_num]].subject)))
  943             slen++;
  944         unmatched += slen - strlen(arts[i].subject);
  945         if (unmatched * 100 / slen > percentage) {
  946             /*
  947              * If there is less greater than percentage% different start a
  948              * new thread.
  949              */
  950             base[++root_num] = i;
  951             arts[i].prev = ART_NORMAL;
  952             continue;
  953         } else {
  954             /*
  955              * The subject lines match enough to consider them part of a single
  956              * thread, so add the current article to the thread.
  957              */
  958             if (arts[base[root_num]].thread < 0)
  959                 arts[base[root_num]].thread = i;
  960             arts[i].prev = i - 1;
  961             arts[i - 1].thread = i;
  962             continue;
  963         }
  964     }
  965 }
  966 
  967 
  968 /*
  969  * This was brought over from tags.c, however this version doesn't
  970  * operate on base_index
  971  *
  972  * Parses a subject header of the type "multipart message subject (01/42)"
  973  * into a MultiPartInfo struct, or fails if the message subject isn't in the
  974  * right form.
  975  *
  976  * @return nonzero on success
  977  */
  978 int
  979 global_get_multipart_info(
  980     int aindex,
  981     MultiPartInfo *setme)
  982 {
  983     int i, j, offi, offj;
  984     MultiPartInfo setmei, setmej;
  985 
  986     i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
  987     j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
  988 
  989     /* Ok i hits first */
  990     if (offi > offj) {
  991         *setme = setmei;
  992         return i;
  993     }
  994 
  995     /* Its j or they are both the same (which must be zero!) so we don't care */
  996     *setme = setmej;
  997     return j;
  998 }
  999 
 1000 
 1001 /*
 1002  * Again this was taken from tags.c, but works on global indices into arts
 1003  * rather than on base_index.
 1004  */
 1005 static int
 1006 global_look_for_multipart_info(
 1007     int aindex,
 1008     MultiPartInfo* setme,
 1009     char start,
 1010     char stop,
 1011     int *offset)
 1012 {
 1013     char *subj;
 1014     char *pch;
 1015     MultiPartInfo tmp;
 1016 
 1017     *offset = 0;
 1018 
 1019     /* entry assertions */
 1020     assert(0 <= aindex && aindex < top_art && "invalid index");
 1021     assert(setme != NULL && "setme must not be NULL");
 1022 
 1023     /* parse the message */
 1024     subj = arts[aindex].subject;
 1025     pch = strrchr(subj, start);
 1026     if (!pch || !isdigit((int) pch[1]))
 1027         return 0;
 1028 
 1029     tmp.base_index = aindex; /* This could be confusing because we are actually storing the index into arts, not the base_index. */
 1030     tmp.subject_compare_len = pch - subj;
 1031     tmp.part_number = (int) strtol(pch + 1, &pch, 10);
 1032     if (*pch != '/' && *pch != '|')
 1033         return 0;
 1034 
 1035     if (!isdigit((int) pch[1]))
 1036         return 0;
 1037 
 1038     tmp.total = (int) strtol(pch + 1, &pch, 10);
 1039     if (*pch != stop)
 1040         return 0;
 1041 
 1042     tmp.subject = subj;
 1043     *setme = tmp;
 1044     *offset = pch - subj;
 1045     return 1;
 1046 }
 1047 
 1048 
 1049 /*
 1050  * Taken from tags.c but changed to use indices into arts[] instead of
 1051  * base_index. Changed so that even when we don't have all the parts we
 1052  * return a valid array.
 1053  *
 1054  * Tries to find all the parts to the multipart message pointed to by
 1055  * base_index.
 1056  *
 1057  * @return on success, the number of parts found. On failure, zero if not a
 1058  * multipart or the negative value of the first missing part.
 1059  * @param base_index index pointing to one of the messages in a multipart
 1060  * message.
 1061  * @param malloc_and_setme_info on success, set to a malloced array the
 1062  * parts found.
 1063  */
 1064 static int
 1065 global_get_multiparts(
 1066     int aindex,
 1067     MultiPartInfo **malloc_and_setme_info)
 1068 {
 1069     int i;
 1070     int part_index;
 1071     MultiPartInfo tmp, tmp2;
 1072     MultiPartInfo *info = NULL;
 1073 
 1074     /* entry assertions */
 1075     assert(0 <= aindex && aindex < top_art && "Invalid index");
 1076     assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
 1077 
 1078     /* make sure this is a multipart message... */
 1079     if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
 1080         return 0;
 1081 
 1082     /* make a temporary buffer to hold the multipart info... */
 1083     info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
 1084 
 1085     /* zero out part-number for the repost check below */
 1086     for (i = 0; i < tmp.total; ++i) {
 1087         info[i].total = tmp.total; /* Added this for thread_by_multipart */
 1088         info[i].part_number = -1;
 1089     }
 1090 
 1091     /* try to find all the multiparts... */
 1092     for_each_art(i) {
 1093         if (strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
 1094             continue;
 1095 
 1096         if (!global_get_multipart_info(i, &tmp2))
 1097             continue;
 1098 
 1099         /* 'test (1/5)' is not the same as 'test (1/15)' */
 1100         if (tmp.total != tmp2.total)
 1101             continue;
 1102 
 1103         part_index = tmp2.part_number - 1;
 1104 
 1105         /*
 1106          * skip "blah (00/102)" or "blah (103/102)" subjects
 1107          */
 1108         if (part_index < 0 || part_index >= tmp.total)
 1109             continue;
 1110 
 1111         /* repost check: do we already have this part? */
 1112         if (info[part_index].part_number != -1) {
 1113             assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
 1114             continue;
 1115         }
 1116 
 1117         /* we have a match, hooray! */
 1118         info[part_index] = tmp2;
 1119     }
 1120 
 1121     /* see if we got them all. */
 1122     for (i = 0; i < tmp.total; ++i) {
 1123         if (info[i].part_number != i + 1) {
 1124             *malloc_and_setme_info = info;
 1125             return -(i + 1); /* missing part #(i+1) */
 1126         }
 1127     }
 1128 
 1129     /* looks like a success .. */
 1130     *malloc_and_setme_info = info;
 1131     return tmp.total;
 1132 }
 1133 
 1134 
 1135 /*
 1136  *  The algorithm uses the tag multipart searches to thread articles together.
 1137  */
 1138 static void
 1139 thread_by_multipart(
 1140     void)
 1141 {
 1142     int i, j, threadNum;
 1143     MultiPartInfo *minfo = NULL;
 1144 
 1145     for_each_art(i) {
 1146         if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo))
 1147             continue;
 1148 
 1149         threadNum = -1;
 1150         for (j = minfo[0].total - 1; j >= 0; j--) {
 1151             if (minfo[j].part_number != -1) {
 1152                 if (threadNum != -1) {
 1153                     arts[minfo[j].base_index].thread = threadNum;
 1154                     arts[threadNum].prev = minfo[j].base_index;
 1155                 }
 1156 
 1157                 threadNum = minfo[j].base_index;
 1158             }
 1159         }
 1160     }
 1161     free(minfo);
 1162 }
 1163 
 1164 
 1165 /*
 1166  * Go through the articles in arts[] and create threads. There are
 1167  * 5 strategies currently defined :
 1168  *
 1169  *  THREAD_NONE     No threading
 1170  *  THREAD_SUBJ     Threads are created using like Subject lines
 1171  *  THREAD_REFS     Threads are created using the References headers
 1172  *  THREAD_BOTH     Threads created using References and then Subject
 1173  *  THREAD_MULTI    Threads created using Subject to search for Multiparts
 1174  *  THREAD_PERC     Threads based upon a char for char match of greater than x%
 1175  *
 1176  * .thread and .prev are used to hold the threading information, see tin.h for
 1177  * more information
 1178  * Only process valid (unexpired) articles we haven't visited yet
 1179  * (ie arts[].thread == ART_UNTHREADED)
 1180  *
 1181  * The rethread parameter is used to force the deletion of existing threading
 1182  * information before threading which happens anyway expect when using
 1183  * THREAD_NONE (I don't immediately see how this is useful)
 1184  */
 1185 /* TODO: rewrite that user can easily combine different 'threading'
 1186  *       methods, i.e:
 1187  *       - thread_by_multipart() + collate_subjects()
 1188  */
 1189 void
 1190 make_threads(
 1191     struct t_group *group,
 1192     t_bool rethread)
 1193 {
 1194     if (!cmd_line && !batch_mode)
 1195         info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
 1196 
 1197 #ifdef DEBUG
 1198     if (debug & DEBUG_MISC)
 1199         error_message(2, "rethread=[%d]  thread_articles=[%d]  attr_thread_articles=[%d]",
 1200                 rethread, tinrc.thread_articles, group->attribute->thread_articles);
 1201 #endif /* DEBUG */
 1202 
 1203     /*
 1204      * Sort all the articles using the preferred method
 1205      * When find_base() is called, the bases are created ordered
 1206      * on arts[] and so the base messages under all threading systems
 1207      * will be sorted in this way.
 1208      */
 1209     sort_arts(group->attribute->sort_article_type);
 1210 
 1211     /*
 1212      * Reset all the ptrs to articles following the above sort
 1213      */
 1214     clear_art_ptrs();
 1215 
 1216     /*
 1217      * The threading pointers need to be reset if re-threading
 1218      * If using ref threading, revector the links back to the articles
 1219      */
 1220     if (rethread || group->attribute->thread_articles) {
 1221         int i;
 1222 
 1223         for_each_art(i) {
 1224             if (arts[i].thread >= 0)
 1225                 arts[i].thread = ART_UNTHREADED;
 1226 
 1227             arts[i].prev = ART_NORMAL;
 1228 
 1229             /* Should never happen if tree is built properly */
 1230             if (arts[i].refptr == NULL) {
 1231 #ifdef DEBUG
 1232                 if (debug & DEBUG_REFS) {
 1233                     my_fprintf(stderr, "\nError  : art->refptr is NULL\n");
 1234                     my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
 1235                     my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
 1236                     my_fprintf(stderr, "From   : %s\n", arts[i].from);
 1237                     assert(arts[i].refptr != NULL);
 1238                 } else
 1239 #endif /* DEBUG */
 1240                     continue;
 1241             }
 1242             arts[i].refptr->article = i;
 1243         }
 1244     }
 1245 
 1246     /*
 1247      * Do the right thing according to the threading strategy
 1248      */
 1249     switch (group->attribute->thread_articles) {
 1250         case THREAD_NONE:
 1251             break;
 1252 
 1253         case THREAD_SUBJ:
 1254             thread_by_subject();
 1255             break;
 1256 
 1257         case THREAD_REFS:
 1258             thread_by_reference();
 1259             break;
 1260 
 1261         case THREAD_BOTH:
 1262             thread_by_reference();
 1263             collate_subjects();
 1264             break;
 1265 
 1266         case THREAD_MULTI:
 1267             thread_by_multipart();
 1268             break;
 1269 
 1270         case THREAD_PERC:
 1271             thread_by_percentage(group);
 1272             break;
 1273 
 1274         default: /* not reached */
 1275             break;
 1276     }
 1277 
 1278     /*
 1279      * Rebuild base[]
 1280      */
 1281     find_base(group);
 1282 }
 1283 
 1284 
 1285 static t_compfunc
 1286 eval_sort_arts_func(
 1287     unsigned int sort_art_type)
 1288 {
 1289     switch (sort_art_type) {
 1290         case SORT_ARTICLES_BY_NOTHING:      /* don't sort at all */
 1291             return artnum_comp;
 1292 
 1293         case SORT_ARTICLES_BY_SUBJ_DESCEND:
 1294             return subj_comp_desc;
 1295 
 1296         case SORT_ARTICLES_BY_SUBJ_ASCEND:
 1297             return subj_comp_asc;
 1298 
 1299         case SORT_ARTICLES_BY_FROM_DESCEND:
 1300             return from_comp_desc;
 1301 
 1302         case SORT_ARTICLES_BY_FROM_ASCEND:
 1303             return from_comp_asc;
 1304 
 1305         case SORT_ARTICLES_BY_DATE_DESCEND:
 1306             return date_comp_desc;
 1307 
 1308         case SORT_ARTICLES_BY_DATE_ASCEND:
 1309             return date_comp_asc;
 1310 
 1311         case SORT_ARTICLES_BY_SCORE_DESCEND:
 1312             return score_comp_desc;
 1313 
 1314         case SORT_ARTICLES_BY_SCORE_ASCEND:
 1315             return score_comp_asc;
 1316 
 1317         case SORT_ARTICLES_BY_LINES_DESCEND:
 1318             return lines_comp_desc;
 1319 
 1320         case SORT_ARTICLES_BY_LINES_ASCEND:
 1321             return lines_comp_asc;
 1322 
 1323         default:
 1324             break;
 1325     }
 1326     return NULL;
 1327 }
 1328 
 1329 
 1330 void
 1331 sort_arts(
 1332     unsigned int sort_art_type)
 1333 {
 1334     t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
 1335 
 1336     if (comp_func)
 1337         SortBy(comp_func);
 1338 }
 1339 
 1340 
 1341 static void
 1342 sort_base(
 1343     unsigned int sort_threads_type)
 1344 {
 1345     switch (sort_threads_type) {
 1346         case SORT_THREADS_BY_SCORE_DESCEND:
 1347         case SORT_THREADS_BY_SCORE_ASCEND:
 1348             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
 1349             break;
 1350 
 1351         case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
 1352             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
 1353             break;
 1354 
 1355         case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
 1356             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
 1357             break;
 1358     }
 1359 }
 1360 
 1361 
 1362 /*
 1363  * This is called to get header info for articles not already found in the
 1364  * overview files.
 1365  * Code reads (max_lineno) lines of article to catch headers like Archive-name:
 1366  * which are not normally included in XOVER or even the normal block of headers.
 1367  * How this is supposed to be useful when 99% of the time we'll have overview
 1368  * data I don't know...
 1369  * TODO: move Archive-name: parsing to article body parsing, remove the
 1370  * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
 1371  * TODO: currgrp->auto_save is set, otherwise it is redundant info
 1372  */
 1373 static t_bool
 1374 parse_headers(
 1375     FILE *fp,
 1376     struct t_article *h)
 1377 {
 1378     char art_from_addr[HEADER_LEN];
 1379     char art_full_name[HEADER_LEN];
 1380     char *hdr, *ptr;
 1381     unsigned int lineno = 0;
 1382     unsigned int max_lineno = 25;
 1383     t_bool got_from, got_lines, got_received;
 1384 
 1385     got_from = got_lines = got_received = FALSE;
 1386 
 1387     while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
 1388         /*
 1389          * Look for the end of information which tin wants to get.
 1390          * Applies when reading local spool and via NNTP.
 1391          */
 1392 
 1393         /*
 1394          * as Archive-name: is placed in the body it's safe to exit
 1395          * the loop if it was found. Don't mix up with Archive: from
 1396          * RFC 5536.
 1397          */
 1398         if (lineno++ > max_lineno || h->archive)
 1399             break;
 1400 
 1401         unfold_header(ptr);
 1402         switch (toupper((unsigned char) *ptr)) {
 1403             case 'A':   /* Archive-name:  optional */
 1404                 /*
 1405                  * Archive-name: {name}/{part|patch}{number}
 1406                  * eg, acorn/faq/part01
 1407                  */
 1408                 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
 1409                     char *s;
 1410 
 1411                     if ((s = strrchr(hdr, '/')) != NULL) {
 1412                         struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
 1413 
 1414                         if (STRNCASECMPEQ(s + 1, "part", 4)) {
 1415                             archptr->partnum = my_strdup(s + 5);
 1416                             archptr->ispart = TRUE;
 1417                         } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
 1418                             archptr->partnum = my_strdup(s + 6);
 1419                             archptr->ispart = FALSE;
 1420                         } else {        /* part or patch must be present */
 1421                             free(archptr);
 1422                             continue;
 1423                         }
 1424                         strtok(archptr->partnum, "\n");
 1425                         *s = '\0';
 1426                         archptr->name = hash_str(hdr);
 1427                         h->archive = archptr;
 1428                     }
 1429                 }
 1430                 break;
 1431 
 1432             case 'D':   /* Date:  mandatory */
 1433                 if (!h->date) {
 1434                     if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
 1435                         h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
 1436                 }
 1437                 break;
 1438 
 1439             case 'F':   /* From:  mandatory */
 1440                 if (!got_from) {
 1441                     if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
 1442                         h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
 1443                         h->from = hash_str(buffer_to_ascii(art_from_addr));
 1444                         if (*art_full_name)
 1445                             h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 1446                         got_from = TRUE;
 1447                     }
 1448                 }
 1449                 break;
 1450 
 1451             case 'L':   /* Lines:  optional */
 1452                 if (!got_lines) {
 1453                     if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
 1454                         h->line_count = atoi(hdr);
 1455                         got_lines = TRUE;
 1456                     }
 1457                 }
 1458                 break;
 1459 
 1460             case 'M':   /* Message-ID:  mandatory */
 1461                 if (!h->msgid) {
 1462                     if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
 1463                         h->msgid = my_strdup(hdr);
 1464                 }
 1465                 break;
 1466 
 1467             /* for Path:-filter when reading from local spool */
 1468             case 'P':   /* Path: */
 1469                 if (!h->path) {
 1470                     if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
 1471                         h->path = my_strdup(hdr);
 1472                 }
 1473                 break;
 1474 
 1475             case 'R':   /* References:  optional */
 1476                 if (!h->refs) {
 1477                     if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
 1478                         h->refs = my_strdup(hdr);
 1479                 }
 1480 
 1481                 /* Received:  If found it's probably a mail article */
 1482                 if (!got_received) {
 1483                     if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
 1484                         max_lineno <<= 1;       /* double the max number of line to read for mails */
 1485                         got_received = TRUE;
 1486                     }
 1487                 }
 1488                 break;
 1489 
 1490             case 'S':   /* Subject:  mandatory */
 1491                 if (!h->subject) {
 1492                     if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE)))
 1493                         h->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(hdr), FALSE)), FALSE));
 1494                 }
 1495                 break;
 1496 
 1497             case 'X':   /* Xref:  optional */
 1498                 if (!h->xref) {
 1499                     if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
 1500                         h->xref = my_strdup(hdr);
 1501                 }
 1502                 break;
 1503 
 1504             default:
 1505                 break;
 1506         } /* switch */
 1507 
 1508     } /* while */
 1509 
 1510 #ifdef NNTP_ABLE
 1511     if (ptr)
 1512         drain_buffer(fp);
 1513 #endif /* NNTP_ABLE */
 1514 
 1515     if (tin_errno)
 1516         return FALSE;
 1517 
 1518     /*
 1519      * The son of RFC 1036 states that the following hdrs are mandatory. It
 1520      * also states that Subject, Newsgroups and Path are too. Ho hum.
 1521      *
 1522      * What about reading mail from local spool via ~/.tin/active.mail,
 1523      * they might not have a Message-ID but got_received is very likely to
 1524      * be true.
 1525      */
 1526     if (got_from && h->date && h->msgid) {
 1527         if (!h->subject)
 1528             h->subject = hash_str("<No subject>");
 1529 
 1530 #ifdef DEBUG
 1531         debug_print_header(h);
 1532 #endif /* DEBUG */
 1533         return TRUE;
 1534     }
 1535 
 1536     return FALSE;
 1537 }
 1538 
 1539 #ifdef NNTP_ABLE
 1540 /*
 1541  * Loop over arts[] and find ranges without Path: header
 1542  * If there are any try to optimize the ranges regarding traffic consumption
 1543  * Start optimization if at least MIN_CNT ranges exist
 1544  * If there are more than MAX_CNT ranges after optimization, fetch all in one
 1545  * big range
 1546  */
 1547 #define MIN_CNT 10
 1548 #define MAX_CNT 50
 1549 static struct t_article_range *
 1550 build_range_list(
 1551     t_artnum min,
 1552     t_artnum max,
 1553     int *range_cnt)
 1554 {
 1555     int i, gap_cnt = 0;
 1556     struct t_article_range *res = NULL, *gap_list, *curr, *from;
 1557     t_artnum new_end;
 1558 
 1559     new_end = T_ARTNUM_CONST(0);
 1560     gap_list = my_malloc(sizeof(struct t_article_range));
 1561     curr = gap_list;
 1562     curr->start = min;
 1563     curr->end = max;
 1564     curr->cnt = T_ARTNUM_CONST(0);
 1565     curr->next = NULL;
 1566 
 1567     for_each_art(i) {
 1568         if (arts[i].artnum < min)
 1569             continue;
 1570         if (arts[i].artnum > max)
 1571             break;
 1572         if (arts[i].path) {
 1573             for (; i < top_art && arts[i].path; i++)
 1574                 ;
 1575             /*
 1576              * the current art has no path -> we use this one
 1577              * if we reached top_art all arts have path
 1578              * so we use max
 1579              */
 1580             curr->start = i == top_art ? max : arts[i--].artnum;
 1581         } else {
 1582             for (; i < top_art && !arts[i].path; i++)
 1583                 ;
 1584             /* the current art has path -> we use the last one */
 1585             new_end = curr->end = arts[--i].artnum;
 1586         }
 1587         if (new_end) {
 1588             curr->cnt = curr->end - curr->start + 1;
 1589             curr->next = my_malloc(sizeof(struct t_article_range));
 1590             curr = curr->next;
 1591             curr->start = new_end;
 1592             curr->end = max;
 1593             curr->cnt = T_ARTNUM_CONST(0);
 1594             curr->next = NULL;
 1595             new_end = T_ARTNUM_CONST(0);
 1596         }
 1597     }
 1598 
 1599     curr = gap_list;
 1600     while (curr && curr->cnt) {
 1601         ++gap_cnt;
 1602 #   ifdef DEBUG
 1603         if ((debug & DEBUG_NNTP) && verbose > 1)
 1604             debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1605 #   endif /* DEBUG */
 1606         curr = curr->next;
 1607     }
 1608 
 1609     /*
 1610      * Optimize only if there are at least MIN_CNT ranges
 1611      */
 1612     if (gap_cnt >= MIN_CNT) {
 1613         res = my_malloc(sizeof(struct t_article_range));
 1614         res->start = T_ARTNUM_CONST(0);
 1615         res->end = T_ARTNUM_CONST(0);
 1616         res->cnt = T_ARTNUM_CONST(0);
 1617         res->next = NULL;
 1618 
 1619         from = gap_list;
 1620         curr = res;
 1621         while (from) {
 1622             curr->start = from->start;
 1623             curr->end = from->end;
 1624             curr->cnt = from->cnt;
 1625             if ((from = from->next)) {
 1626                 /*
 1627                  * If the next range is grater then the gap between the current
 1628                  * one and the next one we build a new range including the
 1629                  * current one, the next one and the gap between
 1630                  */
 1631                 while (from && from->cnt >= from->start - curr->end - 1) {
 1632                     curr->end = from->end;
 1633                     from = from->next;
 1634                 }
 1635                 curr->cnt = curr->end - curr->start + 1;
 1636                 curr->next = my_malloc(sizeof(struct t_article_range));
 1637                 curr = curr->next;
 1638                 curr->start = T_ARTNUM_CONST(0);
 1639                 curr->end = T_ARTNUM_CONST(0);
 1640                 curr->cnt = T_ARTNUM_CONST(0);
 1641                 curr->next = NULL;
 1642             }
 1643         }
 1644     }
 1645 
 1646     /*
 1647      * If there are less then MIN_CNT ranges
 1648      * no res is build -> return the original list
 1649      */
 1650     if (res) {
 1651         while (gap_list) {
 1652             curr = gap_list;
 1653             gap_list = curr->next;
 1654             free(curr);
 1655         }
 1656     } else
 1657         res = gap_list;
 1658 
 1659     curr = res;
 1660     gap_cnt = 0;
 1661     while (curr && curr->cnt) {
 1662         ++gap_cnt;
 1663 #   ifdef DEBUG
 1664         if ((debug & DEBUG_NNTP) && verbose > 1)
 1665             debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1666 #   endif /* DEBUG */
 1667         curr = curr->next;
 1668     }
 1669 
 1670     if (gap_cnt >= MAX_CNT) {
 1671         curr = res;
 1672         while (curr->next && curr->next->cnt) {
 1673             res->end = curr->next->end;
 1674             curr->next->cnt = 0;
 1675             curr = curr->next;
 1676         }
 1677         res->cnt = res->end - res->start + 1;
 1678         gap_cnt = 1;
 1679 #   ifdef DEBUG
 1680         if ((debug & DEBUG_NNTP) && verbose > 1) {
 1681             debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
 1682         }
 1683 #   endif /* DEBUG */
 1684     }
 1685     *range_cnt = gap_cnt;
 1686 
 1687     return res;
 1688 }
 1689 
 1690 
 1691 /*
 1692  * Fetch the Path header in case we want to filter on that in the given group
 1693  *
 1694  * Try [X]HDR first, then XPAT
 1695  */
 1696 static t_bool
 1697 get_path_header(
 1698     int cur,
 1699     int cnt,
 1700     struct t_group *group,
 1701     t_artnum min,
 1702     t_artnum max)
 1703 {
 1704     FILE *fp = NULL;
 1705     char *prep_msg;
 1706     char *buf, *ptr;
 1707     char cmd[NNTP_STRLEN];
 1708     t_artnum artnum, i;
 1709     t_bool found = FALSE;
 1710     static t_bool supported = TRUE; /* assume HDR || XPAT works */
 1711 
 1712     if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
 1713         return FALSE;
 1714 
 1715 #   ifdef DEBUG
 1716     if ((debug & DEBUG_NNTP) && verbose > 1)
 1717         debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
 1718 #   endif /* DEBUG */
 1719 
 1720     if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
 1721         int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
 1722         switch (j) {
 1723             case 215:
 1724                 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 1725 #   ifdef DEBUG
 1726                     if (debug & DEBUG_NNTP)
 1727                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 1728 #   endif /* DEBUG */
 1729                     nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 1730                     strcat(nntp_caps.headers_range, ptr);
 1731                     strcat(nntp_caps.headers_range, "\n");
 1732                 }
 1733                 break;
 1734 
 1735             default:
 1736                 break;
 1737         }
 1738     }
 1739 
 1740     /* does HDR return Path? */
 1741     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
 1742         do {
 1743             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
 1744                 found = TRUE;
 1745         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 1746     }
 1747 
 1748     if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
 1749         if (min == max)
 1750             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
 1751         else
 1752             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
 1753         fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
 1754         if (!nntp_caps.hdr && fp)
 1755             nntp_caps.hdr = TRUE;
 1756     } else if (nntp_caps.xpat) {
 1757         if (min == max)
 1758             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
 1759         else
 1760             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
 1761         fp = nntp_command(cmd, OK_XPAT, NULL, 0);
 1762     }
 1763 
 1764     if (fp) {
 1765         prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
 1766         while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
 1767 #   ifdef DEBUG
 1768             if ((debug & DEBUG_NNTP) && verbose)
 1769                 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1770 #   endif /* DEBUG */
 1771             if ((ptr = tin_strtok(buf, " ")) == NULL)
 1772                 continue;
 1773             artnum = atoartnum(ptr);
 1774             if ((ptr = tin_strtok(NULL, " ")) == NULL)
 1775                 continue;
 1776             for_each_art(i) {
 1777                 if (arts[i].artnum == artnum) {
 1778                     FreeIfNeeded(arts[i].path);
 1779                     arts[i].path = my_strdup(ptr);
 1780                 }
 1781             }
 1782             if (++artnum % MODULO_COUNT_NUM == 0)
 1783                 show_progress(prep_msg, artnum - min, max - min);
 1784         }
 1785         free(prep_msg);
 1786         return supported;
 1787     }
 1788 
 1789     /* !fp */
 1790     supported = FALSE;
 1791     if (nntp_caps.xpat)
 1792         nntp_caps.xpat = FALSE;
 1793     /* as nntp_caps.hdr may work with other headers we don't disable it*/
 1794 
 1795 #ifdef DEBUG
 1796     if ((debug & DEBUG_NNTP) && verbose > 1)
 1797         debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supportet.", group->name);
 1798 #endif /* DEBUG */
 1799     return supported;
 1800 }
 1801 #endif /* NNTP_ABLE */
 1802 
 1803 
 1804 /*
 1805  * Read in an overview index file. Fields are separated by TAB.
 1806  * return the number of expired articles encountered or -1 if the user aborted
 1807  * the read
 1808  * 'top' is set to the highest artnum read
 1809  * If 'local' is set then always open local overview cache in preference to
 1810  * using NNTP XOVER
 1811  *
 1812  * Format (mandatory as far as line count [RFC2980]):
 1813  *  1. article number (ie. 183)                [mandatory]
 1814  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 1815  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 1816  *  4. Date: line     (rfc822 format)          [mandatory]
 1817  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 1818  *  6. References:    (ie. <message-id> ....)  [optional]
 1819  *  7. Byte count     (Skipped - not used)     [mandatory]
 1820  *  8. Line count     (ie. 23)                 [mandatory]
 1821  *  9. Xref: line     (ie. alt.test:389)       [optional]
 1822  */
 1823 static int
 1824 read_overview(
 1825     struct t_group *group,
 1826     t_artnum min,
 1827     t_artnum max,
 1828     t_artnum *top,
 1829     t_bool local,
 1830     t_bool *rebuild_cache)
 1831 {
 1832     FILE *fp;
 1833     char *ptr;
 1834     char *q;
 1835     char *buf;
 1836     char *group_msg;
 1837     char art_full_name[HEADER_LEN];
 1838     char art_from_addr[HEADER_LEN];
 1839     unsigned int count;
 1840     int expired = 0;
 1841     t_artnum artnum;
 1842     t_bool path_found = FALSE, path_in_ofmt = FALSE;
 1843     struct t_article *art;
 1844     size_t over_fields = 1;
 1845 
 1846     /*
 1847      * open the overview file (whether it be local or via nntp)
 1848      */
 1849     if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
 1850         return expired;
 1851 
 1852     if (group->xmax > max)
 1853         group->xmax = max;
 1854 
 1855     group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
 1856 
 1857     /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
 1858     if (ofmt) {
 1859         for (; ofmt[over_fields].name; over_fields++) {
 1860             if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
 1861                 path_in_ofmt = TRUE;
 1862         }
 1863     }
 1864 
 1865     if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
 1866         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
 1867         ofmt[0].type = OVER_T_INT;
 1868         ofmt[0].name = my_strdup("Artnum:");
 1869         ofmt[1].type = OVER_T_STRING;
 1870         ofmt[1].name = my_strdup("Subject:");
 1871         ofmt[2].type = OVER_T_STRING;
 1872         ofmt[2].name = my_strdup("From:");
 1873         ofmt[3].type = OVER_T_STRING;
 1874         ofmt[3].name = my_strdup("Date:");
 1875         ofmt[4].type = OVER_T_STRING;
 1876         ofmt[4].name = my_strdup("Message-ID:");
 1877         ofmt[5].type = OVER_T_STRING;
 1878         ofmt[5].name = my_strdup("References:");
 1879         ofmt[6].type = OVER_T_INT;
 1880         ofmt[6].name = my_strdup("Bytes:");
 1881         ofmt[7].type = OVER_T_INT;
 1882         ofmt[7].name = my_strdup("Lines:");
 1883         ofmt[8].type = OVER_T_ERROR;
 1884         ofmt[8].name = NULL;
 1885         over_fields = 7;
 1886     }
 1887 
 1888     while ((buf = tin_fgets(fp, FALSE)) != NULL) {
 1889 #ifdef DEBUG
 1890         if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
 1891             debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1892 #endif /* DEBUG */
 1893 
 1894         if (need_resize) {
 1895             handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
 1896             need_resize = cNo;
 1897         }
 1898 
 1899         /*
 1900          * Read artnum
 1901          */
 1902         if ((ptr = tin_strtok(buf, "\t")) == NULL)
 1903             continue;
 1904 
 1905         /*
 1906          * read the article number, guaranteed to be the first field
 1907          */
 1908         artnum = atoartnum(ptr);
 1909 
 1910         /*
 1911          * artnum field invalid/corrupt or is 1st line of local cached overview
 1912          * (group name)
 1913          */
 1914         if (artnum <= 0)
 1915             continue;
 1916 
 1917         /*
 1918          * skip artnums below the given minimum (getart_limit)
 1919          */
 1920         if (artnum < min)
 1921             continue;
 1922 
 1923         /*
 1924          * Check to make sure article in nov file has not expired in group
 1925          */
 1926         if (artnum < group->xmin) {
 1927             expired++;
 1928             continue;
 1929         }
 1930 
 1931         /*
 1932          * artnum in overview data higher than groups high mark
 1933          *
 1934          * TODO: - warn user about broken overviews?
 1935          *       - try to parse the Xref:-line to get the correct artnum
 1936          *       - see also parse_unread_arts()
 1937          */
 1938         if (artnum > group->xmax)
 1939             continue;
 1940 
 1941         if (top_art >= max_art)
 1942             expand_art();
 1943 
 1944         art = &arts[top_art];
 1945         set_article(art);
 1946         art->artnum = *top = artnum;
 1947 
 1948         /*
 1949          * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
 1950          *       to check for additions like we do with xref_supported
 1951          */
 1952         for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
 1953             /* skip unexpected tailing fields */
 1954             if (count > over_fields) {
 1955 #ifdef DEBUG
 1956                 if ((debug & DEBUG_NNTP) && verbose > 1)
 1957                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
 1958 #endif /* DEBUG */
 1959 
 1960                 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
 1961                 if (count == over_fields + 1) {
 1962                     if (!strncasecmp(ptr, "Xref: ", 6)) {
 1963 #ifdef DEBUG
 1964                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1965                             debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
 1966 #endif /* DEBUG */
 1967                         over_fields++;
 1968                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 1969                         ofmt[over_fields].type = OVER_T_FSTRING;
 1970                         ofmt[over_fields].name = my_strdup("Xref:");
 1971                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 1972                         ofmt[over_fields + 1].name = NULL;
 1973                         xref_supported = TRUE;
 1974                     } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
 1975 #ifdef DEBUG
 1976                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1977                             debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
 1978 #endif /* DEBUG */
 1979                         over_fields++;
 1980                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 1981                         ofmt[over_fields].type = OVER_T_FSTRING;
 1982                         ofmt[over_fields].name = my_strdup("Path:");
 1983                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 1984                         ofmt[over_fields + 1].name = NULL;
 1985                         xref_supported = TRUE;
 1986                     } else
 1987                         continue;
 1988                 } else
 1989                     continue;
 1990             }
 1991 
 1992             /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
 1993             if (expensive_over_parse) { /* strange order */
 1994                 /* madatory fields */
 1995                 if (ofmt[count].type == OVER_T_STRING) {
 1996                     if (!strcasecmp(ofmt[count].name, "Subject:")) {
 1997                         if (*ptr)
 1998                             art->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 1999                         else {
 2000                             art->subject = hash_str("");
 2001 #ifdef DEBUG
 2002                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2003                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2004 #endif /* DEBUG */
 2005                         }
 2006                         continue;
 2007                     }
 2008 
 2009                     if (!strcasecmp(ofmt[count].name, "From:")) {
 2010                         if (*ptr) {
 2011                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2012                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2013                             if (*art_full_name)
 2014                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2015                         } else {
 2016                             art->from = hash_str("");
 2017 #ifdef DEBUG
 2018                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2019                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2020 #endif /* DEBUG */
 2021                         }
 2022                         continue;
 2023                     }
 2024 
 2025                     if (!strcasecmp(ofmt[count].name, "Date:")) {
 2026                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2027 #ifdef DEBUG
 2028                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2029                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2030 #endif /* DEBUG */
 2031                         continue;
 2032                     }
 2033 
 2034                     if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
 2035                         if (*ptr) {
 2036                             FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
 2037                             art->msgid = my_strdup(ptr);
 2038                         } else {
 2039                             art->msgid = NULL;
 2040 #ifdef DEBUG
 2041                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2042                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2043 #endif /* DEBUG */
 2044                         }
 2045                         continue;
 2046                     }
 2047 
 2048                     if (!strcasecmp(ofmt[count].name, "References:")) {
 2049                         if (*ptr) {
 2050                             FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
 2051                             art->refs = my_strdup(ptr);
 2052                         } else
 2053                             art->refs = NULL;
 2054                         continue;
 2055                     }
 2056 
 2057                     /*
 2058                      * non std. fields when doing
 2059                      * expensive overview parsing (very
 2060                      * rare, just happens if RFC 3977
 2061                      * 8.4.2 is violated) go here
 2062                      */
 2063                     /* for Path:-filter */
 2064                     if (!strcasecmp(ofmt[count].name, "Path:")) {
 2065                         if (!path_found)
 2066                             path_found = TRUE;
 2067                         if (*ptr) {
 2068                             FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
 2069                             art->path = my_strdup(ptr);
 2070                         } else
 2071                             art->path = NULL;
 2072                         continue;
 2073                     }
 2074                 }
 2075                 /* metadata fields */
 2076                 if (ofmt[count].type == OVER_T_INT) {
 2077                     if (!strcasecmp(ofmt[count].name, "Bytes:")) {
 2078                         if (*ptr) {
 2079 #ifdef DEBUG
 2080                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2081                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2082 #endif /* DEBUG */
 2083                         }
 2084                         continue;
 2085                     }
 2086 
 2087                     if (!strcasecmp(ofmt[count].name, "Lines:")) {
 2088                         if (*ptr) {
 2089                             if (isdigit((unsigned char) *ptr))
 2090                                 art->line_count = atoi(ptr);
 2091                             else {
 2092                                 art->line_count = 0;
 2093 #ifdef DEBUG
 2094                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2095                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2096 #endif /* DEBUG */
 2097                             }
 2098                         } else
 2099                             art->line_count = 0;
 2100                         continue;
 2101                     }
 2102                 }
 2103             } else { /* first 7 fields are in RFC 3977 order */
 2104                 switch (count) {
 2105                     case 1: /* Subject: */
 2106                         if (*ptr)
 2107                             art->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2108                         else {
 2109                             art->subject = hash_str("");
 2110 #ifdef DEBUG
 2111                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2112                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2113 #endif /* DEBUG */
 2114                         }
 2115                         break;
 2116 
 2117                     case 2: /* From: */
 2118                         if (*ptr) {
 2119                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2120                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2121                             if (*art_full_name)
 2122                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2123                         } else {
 2124                             art->from = hash_str("");
 2125 #ifdef DEBUG
 2126                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2127                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2128 #endif /* DEBUG */
 2129                         }
 2130                         break;
 2131 
 2132                     case 3: /* Date: */
 2133                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2134 #ifdef DEBUG
 2135                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2136                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2137 #endif /* DEBUG */
 2138                         break;
 2139 
 2140                     case 4: /* Message-ID: */
 2141                         if (*ptr)
 2142                             art->msgid = my_strdup(ptr);
 2143                         else {
 2144                             art->msgid = NULL;
 2145 #ifdef DEBUG
 2146                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2147                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2148 #endif /* DEBUG */
 2149                         }
 2150                         break;
 2151 
 2152                     case 5: /* References: */
 2153                         if (*ptr)
 2154                             art->refs = my_strdup(ptr);
 2155                         else
 2156                             art->refs = NULL;
 2157                         break;
 2158 
 2159                     case 6: /* :bytes || Bytes: */
 2160                         if (*ptr) {
 2161 #ifdef DEBUG
 2162                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2163                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2164 #endif /* DEBUG */
 2165                         }
 2166                         break;
 2167 
 2168                     case 7: /* :lines || Lines: */
 2169                         if (*ptr) {
 2170                             if (isdigit((unsigned char) *ptr))
 2171                                 art->line_count = atoi(ptr);
 2172                             else {
 2173                                 art->line_count = 0;
 2174 #ifdef DEBUG
 2175                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2176                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2177 #endif /* DEBUG */
 2178                             }
 2179                         } else
 2180                             art->line_count = 0;
 2181                         break;
 2182 
 2183                     default:
 2184                         break;
 2185                 }
 2186             }
 2187 
 2188             /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
 2189             if (ofmt[count].type == OVER_T_FSTRING) {
 2190                 if (*ptr) {
 2191                     if (!strcasecmp(ofmt[count].name, "Xref:")) {
 2192                         if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
 2193                             FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
 2194                             art->xref = my_strdup(q);
 2195                         }
 2196 #ifdef DEBUG
 2197                         else {
 2198                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2199                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2200                         }
 2201 #endif /* DEBUG */
 2202                         continue;
 2203                     }
 2204                     /*
 2205                      * handling of addition overview fields
 2206                      * goes here
 2207                      */
 2208 #ifdef DEBUG
 2209                     if ((debug & DEBUG_NNTP) && verbose > 1)
 2210                         debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
 2211 #endif /* DEBUG */
 2212                     /* if we're lucky we've Path in NOV */
 2213                     /*
 2214                      * if reading locally cached overview data try
 2215                      * path regardless of the server OVERVIEW.FMT
 2216                      */
 2217                     if (local || !strcasecmp(ofmt[count].name, "Path:")) {
 2218                         if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
 2219                             if (!path_found)
 2220                                 path_found = TRUE;
 2221                             FreeIfNeeded(art->path);
 2222                             art->path = my_strdup(q);
 2223 #ifdef DEBUG
 2224                             if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
 2225                                 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
 2226 #endif /* DEBUG */
 2227 
 2228                         }
 2229                         continue;
 2230                     }
 2231                 }
 2232                 continue;
 2233             }
 2234         }
 2235 
 2236         /*
 2237          * RFC says Message-ID is mandatory in newsgroups (but not in
 2238          * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
 2239          * threading in mailgroups
 2240          */
 2241         if (!art->msgid && group->type == GROUP_TYPE_NEWS)
 2242             continue;
 2243 
 2244         /* we might lose accuracy here, but that shouldn't hurt */
 2245         if (artnum % MODULO_COUNT_NUM == 0)
 2246             show_progress(group_msg, artnum - min, max - min);
 2247 
 2248         top_art++;              /* Basically this statement commits the article */
 2249     }
 2250 
 2251     free(group_msg);
 2252     TIN_FCLOSE(fp);
 2253 
 2254     if (tin_errno)
 2255         return -1;
 2256 
 2257 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
 2258     if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
 2259         char cbuf[HEADER_LEN];
 2260         int i;
 2261         static t_bool found;
 2262         static t_bool first = TRUE;
 2263 
 2264         if (first) {
 2265             found = TRUE;
 2266             /*
 2267              * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
 2268              */
 2269             if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
 2270                 if (!*nntp_caps.headers_range) {
 2271                     i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
 2272 
 2273                     found = FALSE;
 2274                     switch (i) {
 2275                         case 215:
 2276                             while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 2277 #   ifdef DEBUG
 2278                                 if (debug & DEBUG_NNTP)
 2279                                     debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2280 #   endif /* DEBUG */
 2281                                 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
 2282                                     found = TRUE;
 2283                                 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 2284                                 strcat(nntp_caps.headers_range, ptr);
 2285                                 strcat(nntp_caps.headers_range, "\n");
 2286                             }
 2287                             break;
 2288 
 2289                         default:
 2290                             break;
 2291                     }
 2292                     first = FALSE;
 2293                 } else {
 2294                     found = FALSE;
 2295                     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
 2296                         do {
 2297                             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
 2298                                 found = TRUE;
 2299                         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 2300                     }
 2301                 }
 2302             }
 2303         }
 2304 
 2305         if (found) {
 2306             snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
 2307             group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
 2308             if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
 2309                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
 2310 #   ifdef DEBUG
 2311                     if ((debug & DEBUG_NNTP) && verbose)
 2312                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2313 #   endif /* DEBUG */
 2314 
 2315                     artnum = atoartnum(ptr);
 2316                     if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
 2317                         continue;
 2318                     art = &arts[top_art];
 2319                     set_article(art);
 2320                     if (!art->xref && !strstr(ptr, "(none)")) {
 2321                         if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
 2322                             continue;
 2323                         ptr = q;
 2324                         while (*ptr && isspace((int) *ptr))
 2325                             ptr++;
 2326                         q = strchr(ptr, '\n');
 2327                         if (q)
 2328                             *q = '\0';
 2329                         art->xref = my_strdup(ptr);
 2330                     }
 2331                     /* we might lose accuracy here, but that shouldn't hurt */
 2332                     if (artnum % MODULO_COUNT_NUM == 0)
 2333                         show_progress(group_msg, artnum - min, max - min);
 2334                 }
 2335             }
 2336             free(group_msg);
 2337         }
 2338     }
 2339 #endif /* NNTP_ABLE && XHDR_XREF */
 2340 
 2341     if (local) {
 2342 #ifdef NNTP_ABLE
 2343         if (filter_on_path(group)) {
 2344             struct t_article_range *ranges, *curr;
 2345             t_bool supported = TRUE;
 2346             int curr_range, range_cnt;
 2347 
 2348             /*
 2349              * Get the ranges without Path: header and try to fetch the
 2350              * headers
 2351              */
 2352             if ((ranges = build_range_list(min, *top, &range_cnt))) {
 2353                 curr = ranges;
 2354                 curr_range = 1;
 2355                 while (curr && supported) {
 2356                     if (curr->cnt)
 2357                         supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
 2358                     curr = curr->next;
 2359                 }
 2360                 if (!supported && path_in_ofmt) {
 2361                     /*
 2362                      * fetching Path: headers via [X]HDR or XPAT has failed
 2363                      * Path: is in the servers overview so let the next
 2364                      * read_overview() fetch them
 2365                      */
 2366                     free_art_array();
 2367                     free_msgids();
 2368                     top_art = 0;
 2369                     *top = T_ARTNUM_CONST(0);
 2370                     expired = 0;
 2371                 }
 2372                 *rebuild_cache = TRUE;
 2373                 while (ranges) {
 2374                     curr = ranges;
 2375                     ranges = curr->next;
 2376                     free(curr);
 2377                 }
 2378             }
 2379         }
 2380 #endif /* NNTP_ABLE */
 2381     } else
 2382         if (!path_found && filter_on_path(group)) {
 2383 #ifdef NNTP_ABLE
 2384             if (!get_path_header(1, 1, group, min, *top))
 2385 #endif /* NNTP_ABLE */
 2386                 wait_message(2, _(txt_cannot_filter_on_path));
 2387         }
 2388     return expired;
 2389 }
 2390 
 2391 
 2392 /*
 2393  * Write an Nov/Xover index file. Fields are separated by '\t'.
 2394  *
 2395  * Format:
 2396  *  1. article number (ie. 183)                [mandatory]
 2397  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 2398  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 2399  *  4. Date: line     (rfc822 format)          [mandatory]
 2400  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 2401  *  6. References:    (ie. <message-id> ....)  [optional]
 2402  *  7. Byte count     (Skipped - not used)     [mandatory]
 2403  *  8. Line count     (ie. 23)                 [mandatory]
 2404  *  9. Xref: line     (ie. alt.test:389)       [optional]
 2405  *
 2406  * TODO: as we don't use the original data, we currently can't store
 2407  *       the data (from/subject) in the original charset (we don't store
 2408  *       that info). this has the advantage that we can avoid raw 8bit data
 2409  *       in our overviews, but the disadvantage that we might store the data
 2410  *       with a wrong charset and thus lose information. a similar problem
 2411  *       exists with the data for the from:-line, we don't store it in the
 2412  *       original format, whenever our from-parser (partially) fails we'll
 2413  *       lose information in our overviews (but those couldn't be handled
 2414  *       by tin anyway, so this is not a real problem).
 2415  *       long-term solution: store the original data in the overview
 2416  *       (tin has to handle raw 8bit data and other ugly stuff in the
 2417  *       overviews anyway and thus we preserver as much info as possible)
 2418  *       this would require some changes in read_overview() and
 2419  *       parse_headers(): don't do the decoding/unfolding there, but in a
 2420  *       second pass right after write_overview(), or two additional fields
 2421  *       which hold the raw data for from/subject. the latter has the
 2422  *       disadvantage that it costs (much) more memory.
 2423  */
 2424 static void
 2425 write_overview(
 2426     struct t_group *group)
 2427 {
 2428     FILE *fp;
 2429     int i;
 2430     struct t_article *article;
 2431 #ifdef CHARSET_CONVERSION
 2432     int c = -1;
 2433 #endif /* CHARSET_CONVERSION */
 2434 
 2435     /*
 2436      * Can't write or caching is off or getart_limit is set
 2437      */
 2438     if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
 2439         return;
 2440 
 2441     if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
 2442         return;
 2443 
 2444     if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
 2445         SortBy(artnum_comp);
 2446 
 2447     /*
 2448      * Needed to preserve uniqueness in hashed private overview files
 2449      */
 2450     fprintf(fp, "%s\n", group->name);
 2451 
 2452 #ifdef CHARSET_CONVERSION
 2453     /* get undeclared_charset number if required */
 2454     if (group->attribute->undeclared_charset) {
 2455         for (i = 0; txt_mime_charsets[i] != NULL; i++) {
 2456             if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
 2457                 c = i;
 2458                 break;
 2459             }
 2460         }
 2461     }
 2462 #endif /* CHARSET_CONVERSION */
 2463 
 2464     for_each_art(i) {
 2465         char *p;
 2466         char *q, *ref;
 2467 
 2468         article = &arts[i];
 2469 
 2470         if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
 2471             ref = NULL;
 2472 
 2473             if (!group->attribute->post_8bit_header) { /* write encoded data */
 2474                 /*
 2475                  * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
 2476                  *       here and in print_from() in the CHARSET_CONVERSION case.
 2477                  *       note that this requires something like
 2478                  *          buffer_to_network(article->subject, "UTF-8");
 2479                  *       right before the rfc1522_encode() call.
 2480                  *
 2481                  *       if we would cache the original undecoded data, we could
 2482                  *       ignore stuff like this.
 2483                  */
 2484                 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
 2485                 /* as the subject might now be folded we have to unfold it */
 2486                 unfold_header(p);
 2487             } else { /* raw data */
 2488                 p = my_strdup(article->subject);
 2489 #ifdef CHARSET_CONVERSION
 2490                 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
 2491                     buffer_to_network(p, c);
 2492 #endif /* CHARSET_CONVERSION */
 2493             }
 2494 
 2495             /*
 2496              * replace any '\t's with ' ' in the references-data
 2497              *
 2498              * TODO: nntpext-draft might come up with a new scheme:
 2499              *       For all fields, the value is processed by first
 2500              *       removing all US-ASCII CRLF pairs and then replacing
 2501              *       each remaining US-ASCII NUL, TAB, CR, or LF character
 2502              *       with a single US-ASCII space (for example, CR LF LF TAB
 2503              *       will become two spaces).
 2504              */
 2505             if (article->refs) {
 2506                 ref = q = my_strdup(article->refs);
 2507                 while (*q) {
 2508                     if (*q == '\t')
 2509                         *q = ' ';
 2510                     q++;
 2511                 }
 2512             }
 2513 
 2514             fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
 2515                 article->artnum,
 2516                 p,
 2517 #ifdef CHARSET_CONVERSION
 2518                 print_from(group, article, c),
 2519 #else
 2520                 print_from(group, article, -1),
 2521 #endif /* CHARSET_CONVERSION */
 2522                 print_date(article->date),
 2523                 BlankIfNull(article->msgid),
 2524                 BlankIfNull(ref),
 2525                 0,  /* bytes */
 2526                 article->line_count);
 2527 
 2528             if (article->xref)
 2529                 fprintf(fp, "\tXref: %s", article->xref);
 2530 
 2531             if (article->path)
 2532                 fprintf(fp, "\tPath: %s", article->path);
 2533 
 2534             fprintf(fp, "\n");
 2535             free(p);
 2536             if (article->refs) {
 2537                 FreeIfNeeded(ref);
 2538                 q = NULL;
 2539             }
 2540         }
 2541     }
 2542     fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
 2543     fclose(fp);
 2544 }
 2545 
 2546 
 2547 /*
 2548  * A complex little function to determine the correct overview index file
 2549  * according to 'mode' (read or write)
 2550  * NULL is returned if the current setup dictates otherwise
 2551  *
 2552  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
 2553  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
 2554  *
 2555  * Both of these are hashed
 2556  *
 2557  * GROUP_TYPE_NEWS index files are a little bit more complex
 2558  *
 2559  * When hashing the index filename will be in format number.number.
 2560  * Hashing the groupname gets a number. See if that #.1 file exists;
 2561  * if so, read first line. Is this the group we want? If no, try #.2.
 2562  * Repeat until no such file or we find an existing file that matches
 2563  * our group. Return pointer to path or NULL if not found.
 2564  */
 2565 static char *
 2566 find_nov_file(
 2567     struct t_group *group,
 2568     int mode)
 2569 {
 2570     FILE *fp;
 2571     const char *dir;
 2572     char buf[PATH_LEN];
 2573     int i;
 2574     struct stat sb;
 2575     unsigned long hash;
 2576     static char nov_file[PATH_LEN];
 2577     static t_bool once_only = FALSE;    /* Trap things that are done only 1 time */
 2578 
 2579     if (group == NULL || (mode != R_OK && mode != W_OK))
 2580         return NULL;
 2581 
 2582     switch (group->type) {
 2583         case GROUP_TYPE_MAIL:
 2584             dir = index_maildir;
 2585             break;
 2586 
 2587         case GROUP_TYPE_SAVE:
 2588             dir = index_savedir;
 2589             break;
 2590 
 2591         case GROUP_TYPE_NEWS:
 2592             /*
 2593              * nntp.caps.over_cmd is not an issue here, any gripes and warnings
 2594              * about [X]OVER are handled in nntp_open()
 2595              */
 2596 
 2597             /*
 2598              * When reading via NNTP, system wide overviews are irrelevant, of
 2599              * course, and the private overview filename will be the same for
 2600              * both reading and writing.
 2601              *
 2602              * When working locally, we only use a private cache for reading
 2603              * if requested and when system wide overviews don't already exist.
 2604              * When writing then only private overviews can be used since
 2605              * updating system wide overviews is not safe wrt locking etc.
 2606              *
 2607              * See if local overview file $SPOOLDIR/<groupname>/.overview exists
 2608              */
 2609 #ifndef NNTP_ONLY
 2610             if (!read_news_via_nntp) {
 2611                 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
 2612                 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
 2613                 if (access(nov_file, R_OK) == 0) {
 2614                     if (mode == R_OK)
 2615                         return nov_file;        /* Use system wide overviews */
 2616                     else
 2617                         return NULL;            /* Don't write cache in this case */
 2618                 }
 2619             }
 2620 #endif /* !NNTP_ONLY */
 2621 
 2622             /*
 2623              * We only get here when private overviews are going to be used
 2624              * Go no further if they are explicitly turned off
 2625              */
 2626             if (!tinrc.cache_overview_files)
 2627                 return NULL;
 2628 
 2629             /*
 2630              * Append -<nntpserver> to private cache dir
 2631              */
 2632             if (!once_only && nntp_server) {
 2633                 size_t sp, ln = strlen(index_newsdir);
 2634 
 2635                 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
 2636                     char *srv = my_strdup(nntp_server);
 2637 
 2638                     strcat(index_newsdir, "-");
 2639                     sp--;
 2640                     ln++;
 2641                     str_lwr(srv);
 2642                     my_strncpy(index_newsdir + ln, srv, sp);
 2643                     free(srv);
 2644                 }
 2645                 once_only = TRUE;
 2646             }
 2647 
 2648             /*
 2649              * Only try to set up the private cache when writing. If it
 2650              * doesn't exist yet, then ergo we can't read from it.
 2651              * The cache will be checked/created on every write; a previous
 2652              * bug report complained that this was not the case
 2653              */
 2654             if (stat(index_newsdir, &sb) == -1) {           /* Private cache doesn't exist */
 2655                 if (mode == R_OK)
 2656                     return NULL;
 2657                 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
 2658                     return NULL;
 2659             } else {
 2660                 if (!S_ISDIR(sb.st_mode))
 2661                     return NULL;
 2662             }
 2663 
 2664             /*
 2665              * Update the newsgroups cache to point to the new location
 2666              * now that we know it is valid
 2667              */
 2668             if (!once_only)
 2669                 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
 2670 
 2671             dir = index_newsdir;
 2672             break;
 2673 
 2674         default: /* not reached */
 2675             return NULL;
 2676     }
 2677 
 2678     /*
 2679      * We only get here if writing to a private overview.
 2680      * These always have hashed filenames.
 2681      * Try <hash>.<seqno> and check the group name tagline until
 2682      * matching index file is found. If not found return next unused
 2683      * filename
 2684      */
 2685     hash = hash_groupname(group->name);
 2686 
 2687     for (i = 1; ; i++) {
 2688         char *ptr;
 2689 
 2690         snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
 2691         joinpath(nov_file, sizeof(nov_file), dir, buf);
 2692 
 2693         if ((fp = fopen(nov_file, "r")) == NULL)
 2694             break;
 2695 
 2696         /*
 2697          * No group name header, so not a valid index file => overwrite it
 2698          */
 2699         if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
 2700             fclose(fp);
 2701             break;
 2702         }
 2703         fclose(fp);
 2704 
 2705         if ((ptr = strrchr(buf, '\n')) != NULL)
 2706             *ptr = '\0';
 2707 
 2708         if (strcmp(buf, group->name) == 0)
 2709             break;
 2710     }
 2711 
 2712     return nov_file;
 2713 }
 2714 
 2715 
 2716 /*
 2717  * Run the index file updater only for the groups we've loaded.
 2718  */
 2719 void
 2720 do_update(
 2721     t_bool catchup)
 2722 {
 2723     int i, j, k = 0;
 2724     time_t beg_epoch = 0;
 2725     struct t_article *art;
 2726     struct t_group *group;
 2727 
 2728     if (verbose)
 2729         (void) time(&beg_epoch);
 2730 
 2731     /*
 2732      * loop through groups and update any required index files
 2733      */
 2734     for (i = 0; i < selmenu.max; i++) {
 2735         group = &active[my_group[i]];
 2736         /*
 2737          * FIXME: workaround to get a valid CURR_GROUP
 2738          * it also points to the currently processed group so that
 2739          * the correct attributes are used
 2740          * The correct fix is to get rid of CURR_GROUP
 2741          */
 2742         selmenu.curr = i;
 2743 
 2744         if (group->bogus || !group->subscribed)
 2745             continue;
 2746 
 2747         if (!index_group(group)) {
 2748             for_each_art(j) {
 2749                 art = &arts[j];
 2750                 FreeAndNull(art->refs);
 2751                 FreeAndNull(art->msgid);
 2752             }
 2753             continue;
 2754         }
 2755 
 2756         k++;
 2757 
 2758         if (verbose) {
 2759             my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
 2760             my_flush();
 2761         }
 2762 
 2763         if (catchup) {
 2764             for_each_art(j)
 2765                 art_mark(group, &arts[j], ART_READ);
 2766         }
 2767     }
 2768 
 2769     if (verbose) {
 2770         wait_message(0, _(txt_catchup_update_info),
 2771             (catchup ? _(txt_caughtup) : _(txt_updated)), k,
 2772             PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
 2773     }
 2774 }
 2775 
 2776 
 2777 static int
 2778 artnum_comp(
 2779     t_comptype p1,
 2780     t_comptype p2)
 2781 {
 2782     const struct t_article *s1 = (const struct t_article *) p1;
 2783     const struct t_article *s2 = (const struct t_article *) p2;
 2784 
 2785     /*
 2786      * s1->artnum less than s2->artnum
 2787      */
 2788     if (s1->artnum < s2->artnum)
 2789         return -1;
 2790 
 2791     /*
 2792      * s1->artnum greater than s2->artnum
 2793      */
 2794     if (s1->artnum > s2->artnum)
 2795         return 1;
 2796 
 2797     return 0;
 2798 }
 2799 
 2800 
 2801 /*
 2802  * return result of strcmp (reversed for descending)
 2803  */
 2804 static int
 2805 subj_comp_asc(
 2806     t_comptype p1,
 2807     t_comptype p2)
 2808 {
 2809     int retval;
 2810     const struct t_article *s1 = (const struct t_article *) p1;
 2811     const struct t_article *s2 = (const struct t_article *) p2;
 2812 
 2813     if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
 2814         return retval;
 2815 
 2816     return s1->date - s2->date > 0 ? 1 : -1;
 2817 }
 2818 
 2819 
 2820 static int
 2821 subj_comp_desc(
 2822     t_comptype p1,
 2823     t_comptype p2)
 2824 {
 2825     int retval;
 2826     const struct t_article *s1 = (const struct t_article *) p1;
 2827     const struct t_article *s2 = (const struct t_article *) p2;
 2828 
 2829     if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
 2830         return retval;
 2831 
 2832     return s1->date - s2->date > 0 ? 1 : -1;
 2833 }
 2834 
 2835 
 2836 /*
 2837  * return result of strcmp (reversed for descending)
 2838  */
 2839 static int
 2840 from_comp_asc(
 2841     t_comptype p1,
 2842     t_comptype p2)
 2843 {
 2844     int retval;
 2845     const struct t_article *s1 = (const struct t_article *) p1;
 2846     const struct t_article *s2 = (const struct t_article *) p2;
 2847 
 2848     if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
 2849         return retval;
 2850 
 2851     return s1->date - s2->date > 0 ? 1 : -1;
 2852 }
 2853 
 2854 
 2855 static int
 2856 from_comp_desc(
 2857     t_comptype p1,
 2858     t_comptype p2)
 2859 {
 2860     int retval;
 2861     const struct t_article *s1 = (const struct t_article *) p1;
 2862     const struct t_article *s2 = (const struct t_article *) p2;
 2863 
 2864     if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
 2865         return retval;
 2866 
 2867     return s1->date - s2->date > 0 ? 1 : -1;
 2868 }
 2869 
 2870 
 2871 /*
 2872  * Works like strcmp() for comparing time_t type values
 2873  * Return codes:
 2874  *  -1:     If p1 is before p2
 2875  *   0:     If they are the same time
 2876  *   1:     If p1 is after p2
 2877  * If the sort order is _not_ DATE_ASCEND then the sense of the above
 2878  * is reversed.
 2879  */
 2880 static int
 2881 date_comp_asc(
 2882     t_comptype p1,
 2883     t_comptype p2)
 2884 {
 2885     const struct t_article *s1 = (const struct t_article *) p1;
 2886     const struct t_article *s2 = (const struct t_article *) p2;
 2887 
 2888     /*
 2889      * s1->date less than s2->date
 2890      */
 2891     if (s1->date < s2->date)
 2892         return -1;
 2893 
 2894     /*
 2895      * s1->date greater than s2->date
 2896      */
 2897     if (s1->date > s2->date)
 2898         return 1;
 2899 
 2900     return 0;
 2901 }
 2902 
 2903 
 2904 static int
 2905 date_comp_desc(
 2906     t_comptype p1,
 2907     t_comptype p2)
 2908 {
 2909     const struct t_article *s1 = (const struct t_article *) p1;
 2910     const struct t_article *s2 = (const struct t_article *) p2;
 2911 
 2912     /*
 2913      * s2->date less than s1->date
 2914      */
 2915     if (s2->date < s1->date)
 2916         return -1;
 2917 
 2918     /*
 2919      * s2->date greater than s1->date
 2920      */
 2921     if (s2->date > s1->date)
 2922         return 1;
 2923 
 2924     return 0;
 2925 }
 2926 
 2927 
 2928 /*
 2929  * Same again, but for art[].score
 2930  */
 2931 static int
 2932 score_comp_asc(
 2933     t_comptype p1,
 2934     t_comptype p2)
 2935 {
 2936     const struct t_article *s1 = (const struct t_article *) p1;
 2937     const struct t_article *s2 = (const struct t_article *) p2;
 2938 
 2939     if (s1->score < s2->score)
 2940         return -1;
 2941 
 2942     if (s1->score > s2->score)
 2943         return 1;
 2944 
 2945     return s1->date - s2->date > 0 ? 1 : -1;
 2946 }
 2947 
 2948 
 2949 static int
 2950 score_comp_desc(
 2951     t_comptype p1,
 2952     t_comptype p2)
 2953 {
 2954     const struct t_article *s1 = (const struct t_article *) p1;
 2955     const struct t_article *s2 = (const struct t_article *) p2;
 2956 
 2957     if (s2->score < s1->score)
 2958         return -1;
 2959 
 2960     if (s2->score > s1->score)
 2961         return 1;
 2962 
 2963     return s1->date - s2->date > 0 ? 1 : -1;
 2964 }
 2965 
 2966 
 2967 /*
 2968  * Same again, but for art[].line_count
 2969  */
 2970 static int
 2971 lines_comp_asc(
 2972     t_comptype p1,
 2973     t_comptype p2)
 2974 {
 2975     const struct t_article *s1 = (const struct t_article *) p1;
 2976     const struct t_article *s2 = (const struct t_article *) p2;
 2977 
 2978     if (s1->line_count < s2->line_count)
 2979         return -1;
 2980 
 2981     if (s1->line_count > s2->line_count)
 2982         return 1;
 2983 
 2984     return s1->date - s2->date > 0 ? 1 : -1;
 2985 }
 2986 
 2987 
 2988 static int
 2989 lines_comp_desc(
 2990     t_comptype p1,
 2991     t_comptype p2)
 2992 {
 2993     const struct t_article *s1 = (const struct t_article *) p1;
 2994     const struct t_article *s2 = (const struct t_article *) p2;
 2995 
 2996     if (s2->line_count < s1->line_count)
 2997         return -1;
 2998 
 2999     if (s2->line_count > s1->line_count)
 3000         return 1;
 3001 
 3002     return s1->date - s2->date > 0 ? 1 : -1;
 3003 }
 3004 
 3005 
 3006 /*
 3007  * Compares the total score of two threads. Used for sorting base[].
 3008  */
 3009 static int
 3010 score_comp_base(
 3011     t_comptype p1,
 3012     t_comptype p2)
 3013 {
 3014     int a = get_score_of_thread(*(const long *) p1);
 3015     int b = get_score_of_thread(*(const long *) p2);
 3016 
 3017     /* If scores are equal, compare using the article sort order.
 3018      * This determines the order in a group of equally scored threads.
 3019      */
 3020     if (a == b) {
 3021         const struct t_article *s1 = &arts[*(const long *) p1];
 3022         const struct t_article *s2 = &arts[*(const long *) p2];
 3023         t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
 3024 
 3025         if (comp_func)
 3026             return (*comp_func)(s1, s2);
 3027         return 0;
 3028     }
 3029 
 3030     if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
 3031         return a > b ? 1 : -1;
 3032     return a < b ? 1 : -1;
 3033 }
 3034 
 3035 
 3036 /*
 3037  * Compare the date of the last posted article of two threads.
 3038  * Used for sorting base[].
 3039  */
 3040 static int
 3041 last_date_comp_base_desc(
 3042     t_comptype p1,
 3043     t_comptype p2)
 3044 {
 3045     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3046     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3047 
 3048     if (s2_last < s1_last)
 3049         return -1;
 3050 
 3051     if (s2_last > s1_last)
 3052         return 1;
 3053 
 3054     return 0;
 3055 }
 3056 
 3057 
 3058 static int
 3059 last_date_comp_base_asc(
 3060     t_comptype p1,
 3061     t_comptype p2)
 3062 {
 3063     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3064     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3065 
 3066     if (s2_last > s1_last)
 3067         return -1;
 3068 
 3069     if (s2_last < s1_last)
 3070         return 1;
 3071 
 3072     return 0;
 3073 }
 3074 
 3075 
 3076 static time_t
 3077 get_last_posting_date(
 3078     long n)
 3079 {
 3080     long i;
 3081     time_t last = (time_t) 0;
 3082 
 3083     for (i = n; i >= 0; i = arts[i].thread) {
 3084         if (arts[i].date > last)
 3085             last = arts[i].date;
 3086     }
 3087 
 3088     return last;
 3089 }
 3090 
 3091 
 3092 void
 3093 set_article(
 3094     struct t_article *art)
 3095 {
 3096     art->subject = NULL;
 3097     art->from = NULL;
 3098     art->name = NULL;
 3099     art->date = (time_t) 0;
 3100     art->xref = NULL;
 3101     art->msgid = NULL;
 3102     art->refs = NULL;
 3103     art->refptr = NULL;
 3104     art->line_count = -1;
 3105     art->archive = NULL;
 3106     art->tagged = 0;
 3107     art->thread = ART_EXPIRED;
 3108     art->prev = ART_NORMAL;
 3109     art->score = 0;
 3110     art->status = ART_UNREAD;
 3111     art->killed = ART_NOTKILLED;
 3112     art->zombie = FALSE;
 3113     art->delete_it = FALSE;
 3114     art->selected = FALSE;
 3115     art->inrange = FALSE;
 3116     art->matched = FALSE;
 3117     art->keep_in_base = FALSE;
 3118 }
 3119 
 3120 
 3121 /*
 3122  * Do a binary chop to see if 'art' (an article number) exists in arts[]
 3123  * Naturally arts[] must be sorted on artnum
 3124  * Return index into arts[] or -1
 3125  */
 3126 static int
 3127 valid_artnum(
 3128     t_artnum art)
 3129 {
 3130     int prev, range;
 3131     int dctop = top_art;
 3132     int cur = 1;
 3133 
 3134     while ((dctop >>= 1))
 3135         cur <<= 1;
 3136 
 3137     range = cur >> 1;
 3138     cur--;
 3139 
 3140     forever {
 3141         if (arts[cur].artnum == art)
 3142             return cur;
 3143 
 3144         prev = cur;
 3145         cur += ((arts[cur].artnum < art) ? range : -range);
 3146         if (prev == cur)
 3147             break;
 3148 
 3149         if (cur >= top_art)
 3150             cur = top_art - 1;
 3151 
 3152         range >>= 1;
 3153     }
 3154     return -1;
 3155 }
 3156 
 3157 
 3158 /*
 3159  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
 3160  * Needed if arts[] is not sorted on artnum
 3161  * Return index into arts[] or -1
 3162  */
 3163 int
 3164 find_artnum(
 3165     t_artnum art)
 3166 {
 3167     int i;
 3168 
 3169     for_each_art(i) {
 3170         if (arts[i].artnum == art)
 3171             return i;
 3172     }
 3173     return -1;
 3174 }
 3175 
 3176 
 3177 /*----------------------------- Overview handling -----------------------*/
 3178 /* TODO: use
 3179  *           setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
 3180  *           my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
 3181  *       instead?
 3182  */
 3183 static char *
 3184 print_date(
 3185     time_t secs)
 3186 {
 3187     static char date[25];
 3188     struct tm *tm;
 3189     static const char *const months_a[] = {
 3190         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 3191         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
 3192     };
 3193 
 3194     if ((tm = gmtime(&secs)) != NULL)
 3195         snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
 3196                 tm->tm_mday,
 3197                 months_a[tm->tm_mon],
 3198                 tm->tm_year + 1900,
 3199                 tm->tm_hour, tm->tm_min, tm->tm_sec);
 3200     else
 3201         snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
 3202 
 3203     return date;
 3204 }
 3205 
 3206 
 3207 static char *
 3208 print_from(
 3209     struct t_group *group,
 3210     struct t_article *article,
 3211     int charset)
 3212 {
 3213     char *p, *q;
 3214     static char from[PATH_LEN];
 3215 
 3216     *from = '\0';
 3217 
 3218     if (article->name != NULL) {
 3219         q = my_strdup(article->name);
 3220 #ifdef CHARSET_CONVERSION
 3221         if (charset != -1) {
 3222             buffer_to_network(q, charset);
 3223         }
 3224 #endif /* CHARSET_CONVERSION */
 3225         p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
 3226         unfold_header(p);
 3227         if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
 3228             snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3229         else
 3230             snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3231 
 3232         free(p);
 3233         free(q);
 3234     } else
 3235         STRCPY(from, article->from);
 3236 
 3237     return from;
 3238 }
 3239 
 3240 
 3241 /*
 3242  * Open a group news overview file
 3243  * Use NNTP XOVER where possible unless 'local' is set
 3244  */
 3245 static FILE *
 3246 open_xover_fp(
 3247     struct t_group *group,
 3248     const char *mode,
 3249     t_artnum min,
 3250     t_artnum max,
 3251     t_bool local)
 3252 {
 3253 #ifdef NNTP_ABLE
 3254     if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
 3255         char line[NNTP_STRLEN];
 3256 
 3257         if (!max)
 3258             return NULL;
 3259         if (min == max)
 3260             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
 3261         else
 3262             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
 3263         return (nntp_command(line, OK_XOVER, NULL, 0));
 3264     }
 3265 #endif /* NNTP_ABLE */
 3266     {
 3267         FILE *fp;
 3268         char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
 3269 
 3270         if (nov_file != NULL) {
 3271             if ((fp = fopen(nov_file, mode)) != NULL)
 3272                 return fp;
 3273 
 3274             if (*mode != 'r')
 3275                 error_message(2, _(txt_cannot_open), nov_file);
 3276         }
 3277     }
 3278     return NULL;
 3279 }
 3280 
 3281 
 3282 #ifdef USE_HEAPSORT
 3283 int
 3284 tin_sort(
 3285     void *sbase,
 3286     size_t nel,
 3287     size_t width,
 3288     t_compfunc compar)
 3289 {
 3290     int rc;
 3291 
 3292     switch (tinrc.sort_function) {
 3293         case 0:
 3294             qsort(sbase, nel, width, compar);
 3295             rc = 0;
 3296             break;
 3297 
 3298         case 1:
 3299             rc = heapsort(sbase, nel, width, compar);
 3300             break;
 3301 
 3302         default:
 3303             rc = -1;
 3304             break;
 3305     }
 3306     return rc;
 3307 }
 3308 #endif /* USE_HEAPSORT */