"Fossies" - the Fresh Open Source Software Archive

Member "tin-2.4.4/src/art.c" (20 Nov 2019, 89949 Bytes) of package /linux/misc/tin-2.4.4.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "art.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.4.3_vs_2.4.4.

    1 /*
    2  *  Project   : tin - a Usenet reader
    3  *  Module    : art.c
    4  *  Author    : I.Lea & R.Skrenta
    5  *  Created   : 1991-04-01
    6  *  Updated   : 2019-06-12
    7  *  Notes     :
    8  *
    9  * Copyright (c) 1991-2020 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
   10  * All rights reserved.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  *
   16  * 1. Redistributions of source code must retain the above copyright notice,
   17  *    this list of conditions and the following disclaimer.
   18  *
   19  * 2. Redistributions in binary form must reproduce the above copyright
   20  *    notice, this list of conditions and the following disclaimer in the
   21  *    documentation and/or other materials provided with the distribution.
   22  *
   23  * 3. Neither the name of the copyright holder nor the names of its
   24  *    contributors may be used to endorse or promote products derived from
   25  *    this software without specific prior written permission.
   26  *
   27  * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   28  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   31  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   32  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   33  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   34  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   35  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   36  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   37  * POSSIBILITY OF SUCH DAMAGE.
   38  */
   39 
   40 
   41 #ifndef TIN_H
   42 #   include "tin.h"
   43 #endif /* !TIN_H */
   44 #ifndef TCURSES_H
   45 #   include "tcurses.h"
   46 #endif /* !TCURSES_H */
   47 #ifndef NEWSRC_H
   48 #   include "newsrc.h"
   49 #endif /* !NEWSRC_H */
   50 
   51 #ifndef STPWATCH_H
   52 #   include "stpwatch.h"
   53 #endif /* !STPWATCH_H */
   54 
   55 /*
   56  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
   57  */
   58 #define SortBy(func)    tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
   59 
   60 int top_art = 0;                /* # of articles in arts[] */
   61 
   62 /*
   63  * Local prototypes
   64  */
   65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
   66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
   67 static char *find_nov_file(struct t_group *group, int mode);
   68 static char *print_date(time_t secs);
   69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
   70 static int artnum_comp(t_comptype p1, t_comptype p2);
   71 static int base_comp(t_comptype p1, t_comptype p2);
   72 static int date_comp_asc(t_comptype p1, t_comptype p2);
   73 static int date_comp_desc(t_comptype p1, t_comptype p2);
   74 static int from_comp_asc(t_comptype p1, t_comptype p2);
   75 static int from_comp_desc(t_comptype p1, t_comptype p2);
   76 static int global_get_multiparts(int aindex, MultiPartInfo **malloc_and_setme_info);
   77 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
   78 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
   79 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
   80 static int lines_comp_asc(t_comptype p1, t_comptype p2);
   81 static int lines_comp_desc(t_comptype p1, t_comptype p2);
   82 static int read_art_headers(struct t_group *group, int total, t_artnum top);
   83 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
   84 static int score_comp_asc(t_comptype p1, t_comptype p2);
   85 static int score_comp_desc(t_comptype p1, t_comptype p2);
   86 static int score_comp_base(t_comptype p1, t_comptype p2);
   87 static int subj_comp_asc(t_comptype p1, t_comptype p2);
   88 static int subj_comp_desc(t_comptype p1, t_comptype p2);
   89 static int valid_artnum(t_artnum art);
   90 static t_artnum find_first_unread(struct t_group *group);
   91 static t_artnum setup_hard_base(struct t_group *group);
   92 static t_bool parse_headers(FILE *fp, struct t_article *h);
   93 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
   94 static time_t get_last_posting_date(long n);
   95 static void sort_base(unsigned int sort_threads_type);
   96 static void thread_by_multipart(void);
   97 static void thread_by_percentage(struct t_group *group);
   98 static void thread_by_subject(void);
   99 static void write_overview(struct t_group *group);
  100 #ifdef NNTP_ABLE
  101     static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
  102     static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
  103 #endif /* NNTP_ABLE */
  104 
  105 
  106 /*
  107  * Display a suitable 'entering group' message if screen needs redrawing
  108  * Allow for the non-printing %s, and the %-age counter
  109  */
  110 void
  111 show_art_msg(
  112     const char *group)
  113 {
  114     wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
  115 }
  116 
  117 
  118 /*
  119  * Construct the pointers to the first (base) article in each thread.
  120  * If we are showing only unread, then point to the first unread. I have
  121  * no idea why this should be so, it causes problems elsewhere [which_response]
  122  */
  123 void
  124 find_base(
  125     struct t_group *group)
  126 {
  127     int i, j;
  128 
  129     grpmenu.max = 0;
  130 
  131 #ifdef DEBUG
  132     debug_print_arts();
  133 #endif /* DEBUG */
  134 
  135     for_each_art(i) {
  136         /*
  137          * .prev will be set on each article that is after the first article in
  138          * the thread. Invalid articles which have been expired will have
  139          * .thread set to ART_EXPIRED
  140          */
  141         if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
  142             continue;
  143 
  144         if (grpmenu.max >= max_base)
  145             expand_base();
  146 
  147         if (group->attribute->show_only_unread_arts) {
  148             if (arts[i].status != ART_READ || arts[i].keep_in_base)
  149                 base[grpmenu.max++] = i;
  150             else {
  151                 /* Find 1st unread art in thread */
  152                 for (j = i; j >= 0; j = arts[j].thread) {
  153                     if (arts[j].status != ART_READ || arts[j].keep_in_base) {
  154                         base[grpmenu.max++] = i;
  155                         break;
  156                     }
  157                 }
  158             }
  159         } else
  160             base[grpmenu.max++] = i;
  161     }
  162     /* sort base[] */
  163     if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
  164         sort_base(group->attribute->sort_threads_type);
  165 }
  166 
  167 
  168 /*
  169  * Longword comparison routine for the tin_sort()
  170  */
  171 static int
  172 base_comp(
  173     t_comptype p1,
  174     t_comptype p2)
  175 {
  176     const t_artnum *a = (const t_artnum *) p1;
  177     const t_artnum *b = (const t_artnum *) p2;
  178 
  179     if (*a < *b)
  180         return -1;
  181 
  182     if (*a > *b)
  183         return 1;
  184 
  185     return 0;
  186 }
  187 
  188 
  189 /*
  190  * via NNTP:
  191  *   Issue a LISTGROUP command
  192  *   Read the article numbers existing in the group into base[]
  193  *   If the LISTGROUP failed, issue a GROUP command. Use the results to
  194  *   create a less accurate version of base[]
  195  *   This data will already be sorted
  196  *
  197  * on local spool:
  198  *   Read the spool dir to populate base[] as above. Sort it.
  199  *
  200  * Grow the arts[] and bitmaps as needed.
  201  * NB: the output will be sorted on artnum
  202  *
  203  * grpmenu.max is one past top.
  204  * Returns total number of articles in group, or -1 on error
  205  */
  206 static t_artnum
  207 setup_hard_base(
  208     struct t_group *group)
  209 {
  210     t_artnum total = 0;
  211 
  212     grpmenu.max = 0;
  213 
  214     /*
  215      * If reading with NNTP, issue a LISTGROUP
  216      */
  217     if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
  218 #ifdef NNTP_ABLE
  219         char buf[NNTP_STRLEN];
  220         char line[NNTP_STRLEN];
  221         int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  222         FILE *fp;
  223         t_artnum last, start, count = 0, j = 0;
  224         static t_bool skip_listgroup = FALSE;
  225 
  226         /*
  227          * Some nntp servers are broken and need an extra GROUP command
  228          * (reported by reorx@irc.pl). This affects (old?) versions of
  229          * nntpcache, leafnode and SurgeNews. Usually this should not be
  230          * needed.
  231          *
  232          * For getart_limit recheck lowwatermark as at least giganews gives
  233          * very different results for LIST ACTIVE (3 year retention for all)
  234          * and GROUP (based on the clients contract).
  235          * Calculate range and prepare base[] not to lose unread arts.
  236          */
  237         if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
  238             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  239             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  240                 return -1;
  241 
  242             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
  243                 return -1;
  244 
  245             if (getart_limit > 0) {
  246                 j = group->xmax - getart_limit;
  247                 count = MAX(find_first_unread(group), start);
  248             }
  249             if (getart_limit < 0) {
  250                 j = getart_limit + find_first_unread(group);
  251                 count = group->xmin;
  252             }
  253             if (j < group->xmin)
  254                 j = group->xmin;
  255 
  256             for (; count < j; count++) {
  257                 if (grpmenu.max >= max_base)
  258                     expand_base();
  259                 base[grpmenu.max++] = count;
  260             }
  261         }
  262 
  263         /*
  264          * See if LISTGROUP works
  265          */
  266         if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
  267             if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
  268                 /* RFC 3977 allows ranges in LISTGROUP */
  269                 if (getart_limit > 0)
  270                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
  271                 else /* getart_limit < 0; fetch till newest art */
  272                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
  273 
  274             } else /* for RFC 977 just use GROUP */
  275                 skip_listgroup = TRUE;
  276 
  277         } else /* get all article numbers */
  278             snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
  279 
  280         if (!skip_listgroup) {
  281             if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
  282                 char *ptr;
  283 
  284 #   ifdef DEBUG
  285                 if ((debug & DEBUG_NNTP) && verbose > 1)
  286                     debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  287 #   endif /* DEBUG */
  288 
  289                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
  290                     if (grpmenu.max >= max_base)
  291                         expand_base();
  292                     base[grpmenu.max++] = atoartnum(ptr);
  293                     total++;
  294                 }
  295 
  296                 if (tin_errno)
  297                     return -1;
  298             } else
  299                 skip_listgroup = TRUE;
  300         }
  301 
  302         if (skip_listgroup) { /* LISTGROUP was skipped or failed */
  303             /*
  304              * Handle the obscure case that the user aborted before the LISTGROUP
  305              * had a chance to respond
  306              */
  307             if (tin_errno)
  308                 return -1;
  309 
  310             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  311             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  312                 return -1;
  313 
  314             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
  315                 return -1;
  316 
  317 #   ifdef DEBUG
  318             if ((debug & DEBUG_NNTP) && verbose > 1)
  319                 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  320 #   endif /* DEBUG */
  321             total = count;
  322             grpmenu.max = 0;
  323             if (getart_limit > 0) {
  324                 if ((j = find_first_unread(group)) > start) {
  325                     if (group->xmax > getart_limit) {
  326                         start = MIN(j, group->xmax - getart_limit);
  327                         total = getart_limit;
  328                     } else
  329                         start = j;
  330                 }
  331             }
  332             if (getart_limit < 0) {
  333                 if ((j = (getart_limit + find_first_unread(group))) > start)
  334                     start = j;
  335             }
  336             while (start <= last) {
  337                 if (grpmenu.max >= max_base)
  338                     expand_base();
  339                 base[grpmenu.max++] = start++;
  340             }
  341         }
  342 #endif /* NNTP_ABLE */
  343     /*
  344      * Reading off local spool, read the directory files
  345      */
  346     } else {
  347         DIR *d;
  348         DIR_BUF *e;
  349         char group_path[PATH_LEN];
  350         t_artnum art;
  351 
  352         make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
  353 
  354         if ((d = opendir(group_path)) != NULL) {
  355             while ((e = readdir(d)) != NULL) {
  356                 art = atoartnum(e->d_name);
  357                 if (art >= 1) {
  358                     total++;
  359                     if (grpmenu.max >= max_base)
  360                         expand_base();
  361                     base[grpmenu.max++] = art;
  362                 }
  363             }
  364             CLOSEDIR(d);
  365             tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
  366         } else {
  367             perror_message(_(txt_cannot_open), group_path);
  368 #if 0
  369             if (access(group_path, R_OK) != 0)
  370                 error_message(2, _(txt_not_exist));
  371 #endif /* 0 */
  372             return -1;
  373         }
  374     }
  375 
  376     if (grpmenu.max) {
  377         if (base[grpmenu.max - 1] > group->xmax)
  378             group->xmax = base[grpmenu.max - 1];
  379         expand_bitmap(group, base[0]);
  380     }
  381 
  382     return total;
  383 }
  384 
  385 
  386 /*
  387  * Main group indexing routine.
  388  *
  389  * Will read any existing index, create or incrementally update
  390  * the index by looking at the articles in the spool directory,
  391  * and attempt to write a new index if necessary.
  392  *
  393  * Returns FALSE if the user aborted the indexing, otherwise TRUE
  394  */
  395 t_bool
  396 index_group(
  397     struct t_group *group)
  398 {
  399     int i;
  400     int changed;                /* Count of articles whose overview has changed */
  401     int getart_limit;
  402     int respnum;
  403     int total;
  404     t_artnum last_read_article;
  405     t_artnum min, new_min, max;
  406     t_bool caching_xover;
  407     t_bool filtered;
  408     t_bool path_in_nov = FALSE;
  409     t_bool rebuild_cache = FALSE;
  410 
  411     if (group == NULL)
  412         return TRUE;
  413 
  414     if (!batch_mode)
  415         show_art_msg(group->name);
  416 
  417     signal_context = cArt;          /* Set this only once curr_group is valid */
  418 
  419     hash_reclaim();
  420     free_art_array();
  421     free_msgids();
  422 
  423     BegStopWatch("setup_hard_base()");
  424 
  425     /*
  426      * Get list of valid article numbers
  427      */
  428     if (setup_hard_base(group) < 0)
  429         return FALSE;
  430 
  431     EndStopWatch();
  432     PrintStopWatch();
  433 
  434 #ifdef DEBUG
  435     if (debug & DEBUG_NEWSRC) {
  436         debug_print_comment("Before read_overview");
  437         debug_print_bitmap(group, NULL);
  438     }
  439 #endif /* DEBUG */
  440 
  441     min = grpmenu.max ? base[0] : group->xmin;
  442     max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
  443 
  444     getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  445 
  446     if (getart_limit > 0) {
  447         if (grpmenu.max && (grpmenu.max > getart_limit))
  448             min = base[grpmenu.max - getart_limit];
  449         else
  450             getart_limit = 0;
  451     } else if (getart_limit < 0) {
  452         t_artnum first_unread = find_first_unread(group);
  453 
  454         if (min - first_unread < getart_limit)
  455             min = first_unread + getart_limit;
  456         else
  457             getart_limit = 0;
  458     }
  459 
  460     /*
  461      * Quit now if no articles
  462      */
  463     if (max < 0)
  464         return FALSE;
  465 
  466     top_art = 0;
  467     last_read_article = T_ARTNUM_CONST(0);
  468 
  469     /*
  470      * Read in the existing overview data for min..max
  471      * This read has local=TRUE set if locally caching XOVER records to ensure
  472      * we pull in any private overview caches in preference to using OVER
  473      *
  474      * When reading local spool, this will pull in the system wide overview
  475      * cache (if found) otherwise the private overview cache will be read
  476      */
  477     caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
  478     if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
  479         return FALSE;   /* user aborted indexing */
  480 
  481     /*
  482      * Fill in the range last_read_article...max using XOVER
  483      * Only do this if the previous read_overview() was against private cache
  484      */
  485     if ((last_read_article < max) && caching_xover) {
  486         new_min = (last_read_article >= min) ? last_read_article + 1 : min;
  487 
  488         if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
  489             return FALSE;   /* user aborted indexing */
  490         else
  491             changed += i;
  492     } else
  493         caching_xover = FALSE;
  494 
  495     /*
  496      * Mark as UNTHREADED all articles that have been verified as valid
  497      * Get num of new arts to index so the user will have an idea of index time
  498      */
  499     for (i = 0, total = 0; i < grpmenu.max; i++) {
  500         if ((respnum = valid_artnum(base[i])) >= 0) {
  501             arts[respnum].thread = ART_UNTHREADED;
  502             continue;
  503         }
  504         if (base[i] <= last_read_article)       /* It is vital this test be done last */
  505             continue;
  506         total++;
  507     }
  508 
  509     /*
  510      * Add any articles to arts[] that are new or were killed
  511      */
  512     if (total > 0) {
  513         new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
  514 
  515         if ((i = read_art_headers(group, total, new_min)) == -1)
  516             return FALSE;       /* user aborted indexing */
  517         else
  518             changed += i;
  519     }
  520 
  521 #ifdef DEBUG
  522     if (debug & DEBUG_NEWSRC) {
  523         debug_print_comment("Before parse_unread_arts()");
  524         debug_print_bitmap(group, NULL);
  525     }
  526 #endif /* DEBUG */
  527     /*
  528      * Do this before calling art_mark(,, ART_READ) if you want
  529      * the unread count to be correct.
  530      */
  531     min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
  532     parse_unread_arts(group, min);
  533 #ifdef DEBUG
  534     if (debug & DEBUG_NEWSRC) {
  535         debug_print_comment("After parse_unread_arts()");
  536         debug_print_bitmap(group, NULL);
  537     }
  538 #endif /* DEBUG */
  539 
  540     /*
  541      * Stat all articles to see if any have expired
  542      */
  543     for_each_art(i) {
  544         if (arts[i].thread == ART_EXPIRED) {
  545             changed++;
  546 #ifdef DEBUG
  547             if (debug & DEBUG_NEWSRC)
  548                 debug_print_comment("art.c: index_group() purging...");
  549 #endif /* DEBUG */
  550             art_mark(group, &arts[i], ART_READ);
  551             if (group->attribute->show_only_unread_arts)
  552                 arts[i].keep_in_base = FALSE;
  553         }
  554         if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
  555             path_in_nov = TRUE;
  556     }
  557 
  558     /*
  559      * Only rewrite the index if it has changed
  560      * TODO review the exact logic behind "|| caching_xover"
  561      */
  562     if (changed || caching_xover || rebuild_cache)
  563         write_overview(group);
  564 
  565     /*
  566      * Create the reference tree. The msgid and ref ptrs will
  567      * be free()d now that the NovFile has been written.
  568      */
  569     build_references(group);
  570 
  571     /*
  572      * Needs access to the reference tree
  573      */
  574     filtered = filter_articles(group);
  575 
  576     BegStopWatch("make_threads()");
  577 
  578     /*
  579      * Thread the group
  580      */
  581     make_threads(group, FALSE);
  582 
  583     EndStopWatch();
  584     PrintStopWatch();
  585 
  586     if ((changed > 0 || filtered) && !batch_mode)
  587         clear_message();
  588 
  589     return TRUE;
  590 }
  591 
  592 
  593 /*
  594  * Returns number of first unread article
  595  */
  596 static t_artnum
  597 find_first_unread(
  598     struct t_group *group)
  599 {
  600     unsigned char *p;
  601     unsigned char *end = group->newsrc.xbitmap;
  602     t_artnum first = group->newsrc.xmin; /* initial value */
  603 
  604     if ((p = group->newsrc.xbitmap)) {
  605         end += group->newsrc.xbitlen / NBITS;
  606         for (; *p == '\0' && p < end; p++, first += NBITS)
  607             ;
  608     }
  609     return first;
  610 }
  611 
  612 
  613 /*
  614  * Open an article for reading just the header
  615  * 'next' is used/updated with the next article number
  616  * to optimise the number of 'HEAD' commands issued on
  617  * groups with holes.
  618  */
  619 static FILE *
  620 open_art_header(
  621     char *groupname,
  622     t_artnum art,
  623     t_artnum *next)
  624 {
  625     char buf[NNTP_STRLEN];
  626 #ifdef NNTP_ABLE
  627     FILE *fp;
  628     int i;
  629 
  630     if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
  631         /*
  632          * Don't bother requesting if we have not got there yet.
  633          * This is a big win if the group has got holes in it (ie. if 000's
  634          * of articles have expired between active files min & max values).
  635          */
  636         if (art < *next)
  637             return NULL;
  638 
  639         snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  640         if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  641             return fp;
  642 
  643         /*
  644          * TODO:
  645          * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
  646          * "503 Access denied" instead of 480 but NEXT still works,
  647          * so tin loops over all articles without getting useful data
  648          */
  649 
  650         /*
  651          * HEAD failed, try to find NEXT
  652          * Should return "223 artno message-id more text...."
  653          */
  654         i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
  655         switch (i) {
  656             case OK_NOTEXT:
  657                 *next = atoartnum(buf);     /* Set next art number */
  658                 break;
  659 
  660 #   ifndef BROKEN_LISTGROUP
  661             /*
  662              * might happen if LISTGROUP doesn't select group, but
  663              * we are not -DBROKEN_LISTGROUP
  664              */
  665             case ERR_NCING:
  666                 nntp_caps.broken_listgroup = TRUE;
  667                 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
  668                 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
  669                     return NULL;
  670                 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  671                 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  672                     return fp;
  673                 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
  674                     *next = atoartnum(buf);
  675                 break;
  676 #   endif /* !BROKEN_LISTGROUP */
  677 
  678             default:
  679                 /*
  680                  * TODO: abort loop over all arts on ERR_NONEXT
  681                  */
  682 #   ifndef BROKEN_LISTGROUP
  683                 /*
  684                  * to avoid out of sync responses
  685                  * (listgroup seems to work, but didn't select new group,
  686                  *  so xover seems to work but returns old data)
  687                  * we set listgroup_broken = TRUE; once we saw a
  688                  * ERR_NOARTIG / ERR_NONEXT or the like - even if
  689                  * ERR_NOARTIG may occur on servers where listgroup
  690                  * isn't broken...
  691                  */
  692                 nntp_caps.broken_listgroup = TRUE;
  693 #   endif /* !BROKEN_LISTGROUP */
  694                 break;
  695         }
  696 
  697         return NULL;
  698     }
  699 #endif /* NNTP_ABLE */
  700 
  701     snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
  702     return (fopen(buf, "r"));
  703 }
  704 
  705 
  706 /*
  707  * Called after XOVER/local/private overview databases have been loaded
  708  * Read and parse in headers for any arts not already found (usually
  709  * new articles that have not been indexed yet)
  710  * Any new articles that are added have ->thread set to ART_UNTHREADED
  711  * 'top' is the current highest artnum read
  712  *
  713  * Return values are:
  714  *   >0   Number of additional articles read in
  715  *    0   No additional (new) articles were found
  716  *   -1   user aborted during read
  717  */
  718 static int
  719 read_art_headers(
  720     struct t_group *group,
  721     int total,
  722     t_artnum top)
  723 {
  724     FILE *fp;
  725     char dir[PATH_LEN];
  726     char group_msg[LEN];
  727     int i;
  728     int modified = 0;
  729     t_artnum art;
  730     t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
  731     t_bool res;
  732 
  733     /*
  734      * Change to groups spooldir to optimize fopen()'s on local articles
  735      * NB open_art_header() requires this
  736      */
  737     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
  738         char buf[PATH_LEN];
  739 
  740         get_cwd(dir);
  741         make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
  742         if (chdir(buf) != 0) {
  743 #ifdef DEBUG
  744             if (debug & DEBUG_MISC)
  745                 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
  746 #endif /* DEBUG */
  747             return -1;
  748         }
  749     }
  750 
  751     snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
  752 
  753     for (i = 0; i < grpmenu.max; i++) { /* for each article number */
  754         art = base[i];
  755 
  756         /*
  757          * Skip articles that are below the low water mark or are
  758          * already present
  759          */
  760         if (valid_artnum(art) >= 0)
  761             continue;
  762         if (art <= top)
  763             continue;
  764 
  765         /*
  766          * Try and open the article
  767          */
  768         if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
  769             continue;
  770 
  771         /*
  772          * Add article to arts[]
  773          */
  774         if (top_art >= max_art)
  775             expand_art();
  776 
  777         set_article(&arts[top_art]);
  778         arts[top_art].artnum = art;
  779         arts[top_art].thread = ART_UNTHREADED;
  780 
  781         res = parse_headers(fp, &arts[top_art]);
  782 
  783         TIN_FCLOSE(fp);
  784         if (tin_errno) {
  785             modified = -1;
  786             break;
  787         }
  788 
  789         if (!res) {
  790 #ifdef DEBUG
  791             if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
  792                 char buf[PATH_LEN];
  793 
  794                 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
  795                 debug_print_file("ARTS", "read_art_headers() %s", buf);
  796             }
  797 #endif /* DEBUG */
  798             arts[top_art].artnum = T_ARTNUM_CONST(0);
  799             arts[top_art].date = (time_t) 0;
  800             FreeAndNull(arts[top_art].xref);
  801             FreeAndNull(arts[top_art].path);
  802             FreeAndNull(arts[top_art].refs);
  803             FreeAndNull(arts[top_art].msgid);
  804             if (arts[top_art].archive) {
  805                 FreeAndNull(arts[top_art].archive->partnum);
  806                 FreeAndNull(arts[top_art].archive);
  807             }
  808             arts[top_art].tagged = 0;
  809             arts[top_art].thread = ART_EXPIRED;
  810             arts[top_art].prev = ART_NORMAL;
  811             arts[top_art].status = ART_UNREAD;
  812             arts[top_art].killed = ART_NOTKILLED;
  813             arts[top_art].selected = FALSE;
  814             continue;
  815         }
  816 
  817         top = arts[top_art].artnum; /* used if arts are killed */
  818         top_art++;
  819 
  820         if (++modified % MODULO_COUNT_NUM == 0)
  821             show_progress(group_msg, modified, total);
  822     }
  823 
  824     /*
  825      * Change back to previous dir before indexing started
  826      */
  827     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
  828         chdir(dir);
  829 
  830     return modified;
  831 }
  832 
  833 
  834 /*
  835  * The algorithm is elegant, using the fact that identical Subject lines
  836  * are hashed to the same node in table[] (see hashstr.c)
  837  *
  838  * Mark i as being in j's thread list if
  839  * . The article is _not_ being ignored
  840  * . The article is not already threaded
  841  * . One of the following is true:
  842  *    1) The subject lines are the same
  843  *    2) Both are part of the same archive (name's match and arch bit set)
  844  * IMHO the tests for archive name are redundant and have been for years
  845  */
  846 static void
  847 thread_by_subject(
  848     void)
  849 {
  850     int i, j;
  851     struct t_hashnode *h;
  852 
  853     for_each_art(i) {
  854         if (IGNORE_ART_THREAD(i))
  855             continue;
  856 
  857         /*
  858          * Get the contents of the magic marker in the hashnode
  859          */
  860         h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
  861 
  862         j = h->aptr;
  863 
  864         if (j != -1 && j < i) {
  865 #if 1
  866             if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
  867 #else
  868             /* see also refs.c:collate_subjects() */
  869             if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
  870 #endif /* 1 */
  871             {
  872                 arts[j].thread = i;
  873                 arts[i].prev = j;
  874             }
  875         }
  876 
  877         /*
  878          * Update the magic marker with the highest numbered mesg in
  879          * arts[] that has been used in this thread so far
  880          */
  881         h->aptr = i;
  882     }
  883 
  884 #if 0
  885     fprintf(stderr, "Subj dump\n");
  886     fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
  887     for_each_art(i) {
  888         fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
  889             (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
  890             (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
  891             (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
  892             arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
  893     }
  894 #endif /* 0 */
  895 }
  896 
  897 
  898 /*
  899  * This Threading algorithm threads articles into 'buckets' where each bucket
  900  * contains all the articles which match the root article's subject line to
  901  * the configured percentage. Eg, if the root article had the subject "asdf"
  902  * and the match percentage was configured to be 75% then any article would
  903  * match if its subject was no different in more than a single character.
  904  */
  905 static void
  906 thread_by_percentage(
  907     struct t_group *group)
  908 {
  909     int i, j, k;
  910     int root_num = 0; /* The index number of the root we are currently working on. */
  911     unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
  912     unsigned int percentage = 100 - group->attribute->thread_perc;
  913     size_t slen;
  914 
  915     /* First we need to sort art[] to simplify and speed up the matching. */
  916     SortBy(subj_comp_asc);
  917 
  918     /*
  919      * Now we put all the articles which match enough into the thread. If
  920      * an article doesn't match enough we create a new thread and then add
  921      * to that and so on.
  922      */
  923     base[0] = 0;
  924     arts[0].prev = ART_NORMAL;
  925     for_each_art(i) {
  926         if (i == 0)
  927             continue;
  928 
  929         /* Check each character to see if it matched enough */
  930         k = 0;
  931         unmatched = 0;
  932         for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
  933             if (arts[base[root_num]].subject[j] != arts[i].subject[k])
  934                 unmatched++;
  935         }
  936 
  937         /*
  938          * By getting here we have a number of unmatched characters
  939          * between the two strings. We also have the length of the
  940          * strings available to us easily.
  941          * All we need to do is see if the match is good enough, but
  942          * we count differences in the length of the strings against
  943          * them matching.
  944          */
  945         if (!(slen = strlen(arts[base[root_num]].subject)))
  946             slen++;
  947         unmatched += slen - strlen(arts[i].subject);
  948         if (unmatched * 100 / slen > percentage) {
  949             /*
  950              * If there is less greater than percentage% different start a
  951              * new thread.
  952              */
  953             base[++root_num] = i;
  954             arts[i].prev = ART_NORMAL;
  955             continue;
  956         } else {
  957             /*
  958              * The subject lines match enough to consider them part of a single
  959              * thread, so add the current article to the thread.
  960              */
  961             if (arts[base[root_num]].thread < 0)
  962                 arts[base[root_num]].thread = i;
  963             arts[i].prev = i - 1;
  964             arts[i - 1].thread = i;
  965             continue;
  966         }
  967     }
  968 }
  969 
  970 
  971 /*
  972  * This was brought over from tags.c, however this version doesn't
  973  * operate on base_index
  974  *
  975  * Parses a subject header of the type "multipart message subject (01/42)"
  976  * into a MultiPartInfo struct, or fails if the message subject isn't in the
  977  * right form.
  978  *
  979  * @return nonzero on success
  980  */
  981 int
  982 global_get_multipart_info(
  983     int aindex,
  984     MultiPartInfo *setme)
  985 {
  986     int i, j, offi, offj;
  987     MultiPartInfo setmei, setmej;
  988 
  989     i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
  990     j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
  991 
  992     /* Ok i hits first */
  993     if (offi > offj) {
  994         *setme = setmei;
  995         return i;
  996     }
  997 
  998     /* Its j or they are both the same (which must be zero!) so we don't care */
  999     *setme = setmej;
 1000     return j;
 1001 }
 1002 
 1003 
 1004 /*
 1005  * Again this was taken from tags.c, but works on global indices into arts
 1006  * rather than on base_index.
 1007  */
 1008 static int
 1009 global_look_for_multipart_info(
 1010     int aindex,
 1011     MultiPartInfo* setme,
 1012     char start,
 1013     char stop,
 1014     int *offset)
 1015 {
 1016     char *subj;
 1017     char *pch;
 1018     MultiPartInfo tmp;
 1019 
 1020     *offset = 0;
 1021 
 1022     /* entry assertions */
 1023     assert(0 <= aindex && aindex < top_art && "invalid index");
 1024     assert(setme != NULL && "setme must not be NULL");
 1025 
 1026     /* parse the message */
 1027     subj = arts[aindex].subject;
 1028     pch = strrchr(subj, start);
 1029     if (!pch || !isdigit((int) pch[1]))
 1030         return 0;
 1031 
 1032     tmp.base_index = aindex; /* This could be confusing because we are actually storing the index into arts, not the base_index. */
 1033     tmp.subject_compare_len = pch - subj;
 1034     tmp.part_number = (int) strtol(pch + 1, &pch, 10);
 1035     if (*pch != '/' && *pch != '|')
 1036         return 0;
 1037 
 1038     if (!isdigit((int) pch[1]))
 1039         return 0;
 1040 
 1041     tmp.total = (int) strtol(pch + 1, &pch, 10);
 1042     if (*pch != stop)
 1043         return 0;
 1044 
 1045     tmp.subject = subj;
 1046     *setme = tmp;
 1047     *offset = pch - subj;
 1048     return 1;
 1049 }
 1050 
 1051 
 1052 /*
 1053  * Taken from tags.c but changed to use indices into arts[] instead of
 1054  * base_index. Changed so that even when we don't have all the parts we
 1055  * return a valid array.
 1056  *
 1057  * Tries to find all the parts to the multipart message pointed to by
 1058  * base_index.
 1059  *
 1060  * @return on success, the number of parts found. On failure, zero if not a
 1061  * multipart or the negative value of the first missing part.
 1062  * @param base_index index pointing to one of the messages in a multipart
 1063  * message.
 1064  * @param malloc_and_setme_info on success, set to a malloced array the
 1065  * parts found.
 1066  */
 1067 static int
 1068 global_get_multiparts(
 1069     int aindex,
 1070     MultiPartInfo **malloc_and_setme_info)
 1071 {
 1072     int i;
 1073     int part_index;
 1074     MultiPartInfo tmp, tmp2;
 1075     MultiPartInfo *info = NULL;
 1076 
 1077     /* entry assertions */
 1078     assert(0 <= aindex && aindex < top_art && "Invalid index");
 1079     assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
 1080 
 1081     /* make sure this is a multipart message... */
 1082     if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
 1083         return 0;
 1084 
 1085     /* make a temporary buffer to hold the multipart info... */
 1086     info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
 1087 
 1088     /* zero out part-number for the repost check below */
 1089     for (i = 0; i < tmp.total; ++i) {
 1090         info[i].total = tmp.total; /* Added this for thread_by_multipart */
 1091         info[i].part_number = -1;
 1092     }
 1093 
 1094     /* try to find all the multiparts... */
 1095     for_each_art(i) {
 1096         if (strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
 1097             continue;
 1098 
 1099         if (!global_get_multipart_info(i, &tmp2))
 1100             continue;
 1101 
 1102         /* 'test (1/5)' is not the same as 'test (1/15)' */
 1103         if (tmp.total != tmp2.total)
 1104             continue;
 1105 
 1106         part_index = tmp2.part_number - 1;
 1107 
 1108         /*
 1109          * skip "blah (00/102)" or "blah (103/102)" subjects
 1110          */
 1111         if (part_index < 0 || part_index >= tmp.total)
 1112             continue;
 1113 
 1114         /* repost check: do we already have this part? */
 1115         if (info[part_index].part_number != -1) {
 1116             assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
 1117             continue;
 1118         }
 1119 
 1120         /* we have a match, hooray! */
 1121         info[part_index] = tmp2;
 1122     }
 1123 
 1124     /* see if we got them all. */
 1125     for (i = 0; i < tmp.total; ++i) {
 1126         if (info[i].part_number != i + 1) {
 1127             *malloc_and_setme_info = info;
 1128             return -(i + 1); /* missing part #(i+1) */
 1129         }
 1130     }
 1131 
 1132     /* looks like a success .. */
 1133     *malloc_and_setme_info = info;
 1134     return tmp.total;
 1135 }
 1136 
 1137 
 1138 /*
 1139  *  The algorithm uses the tag multipart searches to thread articles together.
 1140  */
 1141 static void
 1142 thread_by_multipart(
 1143     void)
 1144 {
 1145     int i, j, threadNum;
 1146     MultiPartInfo *minfo = NULL;
 1147 
 1148     for_each_art(i) {
 1149         if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo)) {
 1150             FreeAndNull(minfo);
 1151             continue;
 1152         }
 1153 
 1154         threadNum = -1;
 1155         for (j = minfo[0].total - 1; j >= 0; j--) {
 1156             if (minfo[j].part_number != -1) {
 1157                 if (threadNum != -1) {
 1158                     arts[minfo[j].base_index].thread = threadNum;
 1159                     arts[threadNum].prev = minfo[j].base_index;
 1160                 }
 1161                 threadNum = minfo[j].base_index;
 1162             }
 1163         }
 1164         FreeAndNull(minfo);
 1165     }
 1166 }
 1167 
 1168 
 1169 /*
 1170  * Go through the articles in arts[] and create threads. There are
 1171  * 5 strategies currently defined :
 1172  *
 1173  *  THREAD_NONE     No threading
 1174  *  THREAD_SUBJ     Threads are created using like Subject lines
 1175  *  THREAD_REFS     Threads are created using the References headers
 1176  *  THREAD_BOTH     Threads created using References and then Subject
 1177  *  THREAD_MULTI    Threads created using Subject to search for Multiparts
 1178  *  THREAD_PERC     Threads based upon a char for char match of greater than x%
 1179  *
 1180  * .thread and .prev are used to hold the threading information, see tin.h for
 1181  * more information
 1182  * Only process valid (unexpired) articles we haven't visited yet
 1183  * (ie arts[].thread == ART_UNTHREADED)
 1184  *
 1185  * The rethread parameter is used to force the deletion of existing threading
 1186  * information before threading which happens anyway expect when using
 1187  * THREAD_NONE (I don't immediately see how this is useful)
 1188  */
 1189 /* TODO: rewrite that user can easily combine different 'threading'
 1190  *       methods, i.e:
 1191  *       - thread_by_multipart() + collate_subjects()
 1192  */
 1193 void
 1194 make_threads(
 1195     struct t_group *group,
 1196     t_bool rethread)
 1197 {
 1198     if (!cmd_line && !batch_mode)
 1199         info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
 1200 
 1201 #ifdef DEBUG
 1202     if (debug & DEBUG_MISC)
 1203         error_message(2, "rethread=[%d]  thread_articles=[%d]  attr_thread_articles=[%d]",
 1204                 rethread, tinrc.thread_articles, group->attribute->thread_articles);
 1205 #endif /* DEBUG */
 1206 
 1207     /*
 1208      * Sort all the articles using the preferred method
 1209      * When find_base() is called, the bases are created ordered
 1210      * on arts[] and so the base messages under all threading systems
 1211      * will be sorted in this way.
 1212      */
 1213     sort_arts(group->attribute->sort_article_type);
 1214 
 1215     /*
 1216      * Reset all the ptrs to articles following the above sort
 1217      */
 1218     clear_art_ptrs();
 1219 
 1220     /*
 1221      * The threading pointers need to be reset if re-threading
 1222      * If using ref threading, revector the links back to the articles
 1223      */
 1224     if (rethread || group->attribute->thread_articles) {
 1225         int i;
 1226 
 1227         for_each_art(i) {
 1228             if (arts[i].thread >= 0)
 1229                 arts[i].thread = ART_UNTHREADED;
 1230 
 1231             arts[i].prev = ART_NORMAL;
 1232 
 1233             /* Should never happen if tree is built properly */
 1234             if (arts[i].refptr == NULL) {
 1235 #ifdef DEBUG
 1236                 if (debug & DEBUG_REFS) {
 1237                     my_fprintf(stderr, "\nError  : art->refptr is NULL\n");
 1238                     my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
 1239                     my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
 1240                     my_fprintf(stderr, "From   : %s\n", arts[i].from);
 1241                     assert(arts[i].refptr != NULL);
 1242                 } else
 1243 #endif /* DEBUG */
 1244                     continue;
 1245             }
 1246             arts[i].refptr->article = i;
 1247         }
 1248     }
 1249 
 1250     /*
 1251      * Do the right thing according to the threading strategy
 1252      */
 1253     switch (group->attribute->thread_articles) {
 1254         case THREAD_NONE:
 1255             break;
 1256 
 1257         case THREAD_SUBJ:
 1258             thread_by_subject();
 1259             break;
 1260 
 1261         case THREAD_REFS:
 1262             thread_by_reference();
 1263             break;
 1264 
 1265         case THREAD_BOTH:
 1266             thread_by_reference();
 1267             collate_subjects();
 1268             break;
 1269 
 1270         case THREAD_MULTI:
 1271             thread_by_multipart();
 1272             break;
 1273 
 1274         case THREAD_PERC:
 1275             thread_by_percentage(group);
 1276             break;
 1277 
 1278         default: /* not reached */
 1279             break;
 1280     }
 1281 
 1282     /*
 1283      * Rebuild base[]
 1284      */
 1285     find_base(group);
 1286 }
 1287 
 1288 
 1289 static t_compfunc
 1290 eval_sort_arts_func(
 1291     unsigned int sort_art_type)
 1292 {
 1293     switch (sort_art_type) {
 1294         case SORT_ARTICLES_BY_NOTHING:      /* don't sort at all */
 1295             return artnum_comp;
 1296 
 1297         case SORT_ARTICLES_BY_SUBJ_DESCEND:
 1298             return subj_comp_desc;
 1299 
 1300         case SORT_ARTICLES_BY_SUBJ_ASCEND:
 1301             return subj_comp_asc;
 1302 
 1303         case SORT_ARTICLES_BY_FROM_DESCEND:
 1304             return from_comp_desc;
 1305 
 1306         case SORT_ARTICLES_BY_FROM_ASCEND:
 1307             return from_comp_asc;
 1308 
 1309         case SORT_ARTICLES_BY_DATE_DESCEND:
 1310             return date_comp_desc;
 1311 
 1312         case SORT_ARTICLES_BY_DATE_ASCEND:
 1313             return date_comp_asc;
 1314 
 1315         case SORT_ARTICLES_BY_SCORE_DESCEND:
 1316             return score_comp_desc;
 1317 
 1318         case SORT_ARTICLES_BY_SCORE_ASCEND:
 1319             return score_comp_asc;
 1320 
 1321         case SORT_ARTICLES_BY_LINES_DESCEND:
 1322             return lines_comp_desc;
 1323 
 1324         case SORT_ARTICLES_BY_LINES_ASCEND:
 1325             return lines_comp_asc;
 1326 
 1327         default:
 1328             break;
 1329     }
 1330     return NULL;
 1331 }
 1332 
 1333 
 1334 void
 1335 sort_arts(
 1336     unsigned int sort_art_type)
 1337 {
 1338     t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
 1339 
 1340     if (comp_func)
 1341         SortBy(comp_func);
 1342 }
 1343 
 1344 
 1345 static void
 1346 sort_base(
 1347     unsigned int sort_threads_type)
 1348 {
 1349     switch (sort_threads_type) {
 1350         case SORT_THREADS_BY_SCORE_DESCEND:
 1351         case SORT_THREADS_BY_SCORE_ASCEND:
 1352             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
 1353             break;
 1354 
 1355         case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
 1356             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
 1357             break;
 1358 
 1359         case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
 1360             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
 1361             break;
 1362     }
 1363 }
 1364 
 1365 
 1366 /*
 1367  * This is called to get header info for articles not already found in the
 1368  * overview files.
 1369  * Code reads (max_lineno) lines of article to catch headers like Archive-name:
 1370  * which are not normally included in XOVER or even the normal block of headers.
 1371  * How this is supposed to be useful when 99% of the time we'll have overview
 1372  * data I don't know...
 1373  * TODO: move Archive-name: parsing to article body parsing, remove the
 1374  * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
 1375  * TODO: currgrp->auto_save is set, otherwise it is redundant info
 1376  */
 1377 static t_bool
 1378 parse_headers(
 1379     FILE *fp,
 1380     struct t_article *h)
 1381 {
 1382     char art_from_addr[HEADER_LEN];
 1383     char art_full_name[HEADER_LEN];
 1384     char *s, *hdr, *ptr;
 1385     unsigned int lineno = 0;
 1386     unsigned int max_lineno = 25;
 1387     t_bool got_from, got_lines, got_received;
 1388 
 1389     got_from = got_lines = got_received = FALSE;
 1390 
 1391     while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
 1392         /*
 1393          * Look for the end of information which tin wants to get.
 1394          * Applies when reading local spool and via NNTP.
 1395          */
 1396 
 1397         /*
 1398          * as Archive-name: is placed in the body it's safe to exit
 1399          * the loop if it was found. Don't mix up with Archive: from
 1400          * RFC 5536.
 1401          */
 1402         if (lineno++ > max_lineno || h->archive)
 1403             break;
 1404 
 1405         unfold_header(ptr);
 1406         switch (my_toupper((unsigned char) *ptr)) {
 1407             case 'A':   /* Archive-name:  optional */
 1408                 /*
 1409                  * Archive-name: {name}/{part|patch}{number}
 1410                  * eg, acorn/faq/part01
 1411                  */
 1412                 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
 1413                     if ((s = strrchr(hdr, '/')) != NULL) {
 1414                         struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
 1415 
 1416                         if (STRNCASECMPEQ(s + 1, "part", 4)) {
 1417                             archptr->partnum = my_strdup(s + 5);
 1418                             archptr->ispart = TRUE;
 1419                         } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
 1420                             archptr->partnum = my_strdup(s + 6);
 1421                             archptr->ispart = FALSE;
 1422                         } else {        /* part or patch must be present */
 1423                             free(archptr);
 1424                             continue;
 1425                         }
 1426                         strtok(archptr->partnum, "\n");
 1427                         *s = '\0';
 1428                         archptr->name = hash_str(hdr);
 1429                         h->archive = archptr;
 1430                     }
 1431                 }
 1432                 break;
 1433 
 1434             case 'D':   /* Date:  mandatory */
 1435                 if (!h->date) {
 1436                     if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
 1437                         h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
 1438                 }
 1439                 break;
 1440 
 1441             case 'F':   /* From:  mandatory */
 1442                 if (!got_from) {
 1443                     if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
 1444                         h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
 1445                         h->from = hash_str(buffer_to_ascii(art_from_addr));
 1446                         if (*art_full_name)
 1447                             h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 1448                         got_from = TRUE;
 1449                     }
 1450                 }
 1451                 break;
 1452 
 1453             case 'L':   /* Lines:  optional */
 1454                 if (!got_lines) {
 1455                     if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
 1456                         h->line_count = atoi(hdr);
 1457                         got_lines = TRUE;
 1458                     }
 1459                 }
 1460                 break;
 1461 
 1462             case 'M':   /* Message-ID:  mandatory */
 1463                 if (!h->msgid) {
 1464                     if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
 1465                         h->msgid = my_strdup(hdr);
 1466                 }
 1467                 break;
 1468 
 1469             /* for Path:-filter when reading from local spool */
 1470             case 'P':   /* Path: */
 1471                 if (!h->path) {
 1472                     if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
 1473                         h->path = my_strdup(hdr);
 1474                 }
 1475                 break;
 1476 
 1477             case 'R':   /* References:  optional */
 1478                 if (!h->refs) {
 1479                     if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
 1480                         h->refs = my_strdup(hdr);
 1481                 }
 1482 
 1483                 /* Received:  If found it's probably a mail article */
 1484                 if (!got_received) {
 1485                     if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
 1486                         max_lineno <<= 1;       /* double the max number of line to read for mails */
 1487                         got_received = TRUE;
 1488                     }
 1489                 }
 1490                 break;
 1491 
 1492             case 'S':   /* Subject:  mandatory */
 1493                 if (!h->subject) {
 1494                     if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
 1495 #ifdef HAVE_UNICODE_NORMALIZATION
 1496                         if (IS_LOCAL_CHARSET("UTF-8"))
 1497                             s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1498                         else
 1499 #endif /* HAVE_UNICODE_NORMALIZATION */
 1500                             s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1501 
 1502                         h->subject = hash_str(s);
 1503                         free(s);
 1504                     }
 1505                 }
 1506                 break;
 1507 
 1508             case 'X':   /* Xref:  optional */
 1509                 if (!h->xref) {
 1510                     if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
 1511                         h->xref = my_strdup(hdr);
 1512                 }
 1513                 break;
 1514 
 1515             default:
 1516                 break;
 1517         } /* switch */
 1518 
 1519     } /* while */
 1520 
 1521 #ifdef NNTP_ABLE
 1522     if (ptr)
 1523         drain_buffer(fp);
 1524 #endif /* NNTP_ABLE */
 1525 
 1526     if (tin_errno)
 1527         return FALSE;
 1528 
 1529     /*
 1530      * The son of RFC 1036 states that the following hdrs are mandatory. It
 1531      * also states that Subject, Newsgroups and Path are too. Ho hum.
 1532      *
 1533      * What about reading mail from local spool via ~/.tin/active.mail,
 1534      * they might not have a Message-ID but got_received is very likely to
 1535      * be true.
 1536      */
 1537     if (got_from && h->date && h->msgid) {
 1538         if (!h->subject)
 1539             h->subject = hash_str("<No subject>");
 1540 
 1541 #ifdef DEBUG
 1542         debug_print_header(h);
 1543 #endif /* DEBUG */
 1544         return TRUE;
 1545     }
 1546 
 1547     return FALSE;
 1548 }
 1549 
 1550 #ifdef NNTP_ABLE
 1551 /*
 1552  * Loop over arts[] and find ranges without Path: header
 1553  * If there are any try to optimize the ranges regarding traffic consumption
 1554  * Start optimization if at least MIN_CNT ranges exist
 1555  * If there are more than MAX_CNT ranges after optimization, fetch all in one
 1556  * big range
 1557  */
 1558 #define MIN_CNT 10
 1559 #define MAX_CNT 50
 1560 static struct t_article_range *
 1561 build_range_list(
 1562     t_artnum min,
 1563     t_artnum max,
 1564     int *range_cnt)
 1565 {
 1566     int i, gap_cnt = 0;
 1567     struct t_article_range *res = NULL, *gap_list, *curr, *from;
 1568     t_artnum new_end;
 1569 
 1570     new_end = T_ARTNUM_CONST(0);
 1571     gap_list = my_malloc(sizeof(struct t_article_range));
 1572     curr = gap_list;
 1573     curr->start = min;
 1574     curr->end = max;
 1575     curr->cnt = T_ARTNUM_CONST(0);
 1576     curr->next = NULL;
 1577 
 1578     for_each_art(i) {
 1579         if (arts[i].artnum < min)
 1580             continue;
 1581         if (arts[i].artnum > max)
 1582             break;
 1583         if (arts[i].path) {
 1584             for (; i < top_art && arts[i].path; i++)
 1585                 ;
 1586             /*
 1587              * the current art has no path -> we use this one
 1588              * if we reached top_art all arts have path
 1589              * so we use max
 1590              */
 1591             curr->start = i == top_art ? max : arts[i--].artnum;
 1592         } else {
 1593             for (; i < top_art && !arts[i].path; i++)
 1594                 ;
 1595             /* the current art has path -> we use the last one */
 1596             new_end = curr->end = arts[--i].artnum;
 1597         }
 1598         if (new_end) {
 1599             curr->cnt = curr->end - curr->start + 1;
 1600             curr->next = my_malloc(sizeof(struct t_article_range));
 1601             curr = curr->next;
 1602             curr->start = new_end;
 1603             curr->end = max;
 1604             curr->cnt = T_ARTNUM_CONST(0);
 1605             curr->next = NULL;
 1606             new_end = T_ARTNUM_CONST(0);
 1607         }
 1608     }
 1609 
 1610     curr = gap_list;
 1611     while (curr && curr->cnt) {
 1612         ++gap_cnt;
 1613 #   ifdef DEBUG
 1614         if ((debug & DEBUG_NNTP) && verbose > 1)
 1615             debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1616 #   endif /* DEBUG */
 1617         curr = curr->next;
 1618     }
 1619 
 1620     /*
 1621      * Optimize only if there are at least MIN_CNT ranges
 1622      */
 1623     if (gap_cnt >= MIN_CNT) {
 1624         res = my_malloc(sizeof(struct t_article_range));
 1625         res->start = T_ARTNUM_CONST(0);
 1626         res->end = T_ARTNUM_CONST(0);
 1627         res->cnt = T_ARTNUM_CONST(0);
 1628         res->next = NULL;
 1629 
 1630         from = gap_list;
 1631         curr = res;
 1632         while (from) {
 1633             curr->start = from->start;
 1634             curr->end = from->end;
 1635             curr->cnt = from->cnt;
 1636             if ((from = from->next)) {
 1637                 /*
 1638                  * If the next range is grater then the gap between the current
 1639                  * one and the next one we build a new range including the
 1640                  * current one, the next one and the gap between
 1641                  */
 1642                 while (from && from->cnt >= from->start - curr->end - 1) {
 1643                     curr->end = from->end;
 1644                     from = from->next;
 1645                 }
 1646                 curr->cnt = curr->end - curr->start + 1;
 1647                 curr->next = my_malloc(sizeof(struct t_article_range));
 1648                 curr = curr->next;
 1649                 curr->start = T_ARTNUM_CONST(0);
 1650                 curr->end = T_ARTNUM_CONST(0);
 1651                 curr->cnt = T_ARTNUM_CONST(0);
 1652                 curr->next = NULL;
 1653             }
 1654         }
 1655     }
 1656 
 1657     /*
 1658      * If there are less then MIN_CNT ranges
 1659      * no res is build -> return the original list
 1660      */
 1661     if (res) {
 1662         while (gap_list) {
 1663             curr = gap_list;
 1664             gap_list = curr->next;
 1665             free(curr);
 1666         }
 1667     } else
 1668         res = gap_list;
 1669 
 1670     curr = res;
 1671     gap_cnt = 0;
 1672     while (curr && curr->cnt) {
 1673         ++gap_cnt;
 1674 #   ifdef DEBUG
 1675         if ((debug & DEBUG_NNTP) && verbose > 1)
 1676             debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1677 #   endif /* DEBUG */
 1678         curr = curr->next;
 1679     }
 1680 
 1681     if (gap_cnt >= MAX_CNT) {
 1682         curr = res;
 1683         while (curr->next && curr->next->cnt) {
 1684             res->end = curr->next->end;
 1685             curr->next->cnt = 0;
 1686             curr = curr->next;
 1687         }
 1688         res->cnt = res->end - res->start + 1;
 1689         gap_cnt = 1;
 1690 #   ifdef DEBUG
 1691         if ((debug & DEBUG_NNTP) && verbose > 1) {
 1692             debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
 1693         }
 1694 #   endif /* DEBUG */
 1695     }
 1696     *range_cnt = gap_cnt;
 1697 
 1698     return res;
 1699 }
 1700 
 1701 
 1702 /*
 1703  * Fetch the Path header in case we want to filter on that in the given group
 1704  *
 1705  * Try [X]HDR first, then XPAT
 1706  */
 1707 static t_bool
 1708 get_path_header(
 1709     int cur,
 1710     int cnt,
 1711     struct t_group *group,
 1712     t_artnum min,
 1713     t_artnum max)
 1714 {
 1715     FILE *fp = NULL;
 1716     char *prep_msg;
 1717     char *buf, *ptr;
 1718     char cmd[NNTP_STRLEN];
 1719     t_artnum artnum, i;
 1720     t_bool found = FALSE;
 1721     static t_bool supported = TRUE; /* assume HDR || XPAT works */
 1722 
 1723     if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
 1724         return FALSE;
 1725 
 1726 #   ifdef DEBUG
 1727     if ((debug & DEBUG_NNTP) && verbose > 1)
 1728         debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
 1729 #   endif /* DEBUG */
 1730 
 1731     if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
 1732         int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
 1733         switch (j) {
 1734             case 215:
 1735                 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 1736 #   ifdef DEBUG
 1737                     if (debug & DEBUG_NNTP)
 1738                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 1739 #   endif /* DEBUG */
 1740                     nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 1741                     strcat(nntp_caps.headers_range, ptr);
 1742                     strcat(nntp_caps.headers_range, "\n");
 1743                 }
 1744                 break;
 1745 
 1746             default:
 1747                 break;
 1748         }
 1749     }
 1750 
 1751     /* does HDR return Path? */
 1752     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
 1753         do {
 1754             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
 1755                 found = TRUE;
 1756         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 1757     }
 1758 
 1759     if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
 1760         if (min == max)
 1761             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
 1762         else
 1763             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
 1764         fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
 1765         if (!nntp_caps.hdr && fp)
 1766             nntp_caps.hdr = TRUE;
 1767     } else if (nntp_caps.xpat) {
 1768         if (min == max)
 1769             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
 1770         else
 1771             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
 1772         fp = nntp_command(cmd, OK_XPAT, NULL, 0);
 1773     }
 1774 
 1775     if (fp) {
 1776         prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
 1777         while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
 1778 #   ifdef DEBUG
 1779             if ((debug & DEBUG_NNTP) && verbose)
 1780                 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1781 #   endif /* DEBUG */
 1782             if ((ptr = tin_strtok(buf, " ")) == NULL)
 1783                 continue;
 1784             artnum = atoartnum(ptr);
 1785             if ((ptr = tin_strtok(NULL, " ")) == NULL)
 1786                 continue;
 1787             for_each_art(i) {
 1788                 if (arts[i].artnum == artnum) {
 1789                     FreeIfNeeded(arts[i].path);
 1790                     arts[i].path = my_strdup(ptr);
 1791                 }
 1792             }
 1793             if (++artnum % MODULO_COUNT_NUM == 0)
 1794                 show_progress(prep_msg, artnum - min, max - min);
 1795         }
 1796         free(prep_msg);
 1797         return supported;
 1798     }
 1799 
 1800     /* !fp */
 1801     supported = FALSE;
 1802     if (nntp_caps.xpat)
 1803         nntp_caps.xpat = FALSE;
 1804     /* as nntp_caps.hdr may work with other headers we don't disable it*/
 1805 
 1806 #   ifdef DEBUG
 1807     if ((debug & DEBUG_NNTP) && verbose > 1)
 1808         debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
 1809 #   endif /* DEBUG */
 1810     return supported;
 1811 }
 1812 #endif /* NNTP_ABLE */
 1813 
 1814 
 1815 /*
 1816  * Read in an overview index file. Fields are separated by TAB.
 1817  * return the number of expired articles encountered or -1 if the user aborted
 1818  * the read
 1819  * 'top' is set to the highest artnum read
 1820  * If 'local' is set then always open local overview cache in preference to
 1821  * using NNTP XOVER
 1822  *
 1823  * Format (mandatory as far as line count [RFC2980]):
 1824  *  1. article number (ie. 183)                [mandatory]
 1825  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 1826  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 1827  *  4. Date: line     (rfc822 format)          [mandatory]
 1828  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 1829  *  6. References:    (ie. <message-id> ....)  [optional]
 1830  *  7. Byte count     (Skipped - not used)     [mandatory]
 1831  *  8. Line count     (ie. 23)                 [mandatory]
 1832  *  9. Xref: line     (ie. alt.test:389)       [optional]
 1833  */
 1834 static int
 1835 read_overview(
 1836     struct t_group *group,
 1837     t_artnum min,
 1838     t_artnum max,
 1839     t_artnum *top,
 1840     t_bool local,
 1841     t_bool *rebuild_cache)
 1842 {
 1843     FILE *fp;
 1844     char *ptr;
 1845     char *q;
 1846     char *buf;
 1847     char *group_msg;
 1848     char art_full_name[HEADER_LEN];
 1849     char art_from_addr[HEADER_LEN];
 1850     unsigned int count;
 1851     int expired = 0;
 1852     t_artnum artnum;
 1853     t_bool path_found = FALSE, path_in_ofmt = FALSE;
 1854     struct t_article *art;
 1855     size_t over_fields = 1;
 1856 
 1857     /*
 1858      * open the overview file (whether it be local or via nntp)
 1859      */
 1860     if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
 1861         return expired;
 1862 
 1863     if (group->xmax > max)
 1864         group->xmax = max;
 1865 
 1866     group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
 1867 
 1868     /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
 1869     if (ofmt) {
 1870         for (; ofmt[over_fields].name; over_fields++) {
 1871             if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
 1872                 path_in_ofmt = TRUE;
 1873         }
 1874     }
 1875 
 1876     if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
 1877         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
 1878         ofmt[0].type = OVER_T_INT;
 1879         ofmt[0].name = my_strdup("Artnum:");
 1880         ofmt[1].type = OVER_T_STRING;
 1881         ofmt[1].name = my_strdup("Subject:");
 1882         ofmt[2].type = OVER_T_STRING;
 1883         ofmt[2].name = my_strdup("From:");
 1884         ofmt[3].type = OVER_T_STRING;
 1885         ofmt[3].name = my_strdup("Date:");
 1886         ofmt[4].type = OVER_T_STRING;
 1887         ofmt[4].name = my_strdup("Message-ID:");
 1888         ofmt[5].type = OVER_T_STRING;
 1889         ofmt[5].name = my_strdup("References:");
 1890         ofmt[6].type = OVER_T_INT;
 1891         ofmt[6].name = my_strdup("Bytes:");
 1892         ofmt[7].type = OVER_T_INT;
 1893         ofmt[7].name = my_strdup("Lines:");
 1894         ofmt[8].type = OVER_T_ERROR;
 1895         ofmt[8].name = NULL;
 1896         over_fields = 7;
 1897     }
 1898 
 1899     while ((buf = tin_fgets(fp, FALSE)) != NULL) {
 1900 #ifdef DEBUG
 1901         if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
 1902             debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1903 #endif /* DEBUG */
 1904 
 1905         if (need_resize) {
 1906             handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
 1907             need_resize = cNo;
 1908         }
 1909 
 1910         /*
 1911          * Read artnum
 1912          */
 1913         if ((ptr = tin_strtok(buf, "\t")) == NULL)
 1914             continue;
 1915 
 1916         /*
 1917          * read the article number, guaranteed to be the first field
 1918          */
 1919         artnum = atoartnum(ptr);
 1920 
 1921         /*
 1922          * artnum field invalid/corrupt or is 1st line of local cached overview
 1923          * (group name)
 1924          */
 1925         if (artnum <= 0)
 1926             continue;
 1927 
 1928         /*
 1929          * skip artnums below the given minimum (getart_limit)
 1930          */
 1931         if (artnum < min)
 1932             continue;
 1933 
 1934         /*
 1935          * Check to make sure article in nov file has not expired in group
 1936          */
 1937         if (artnum < group->xmin) {
 1938             expired++;
 1939             continue;
 1940         }
 1941 
 1942         /*
 1943          * artnum in overview data higher than groups high mark
 1944          *
 1945          * TODO: - warn user about broken overviews?
 1946          *       - try to parse the Xref:-line to get the correct artnum
 1947          *       - see also parse_unread_arts()
 1948          */
 1949         if (artnum > group->xmax)
 1950             continue;
 1951 
 1952         if (top_art >= max_art)
 1953             expand_art();
 1954 
 1955         art = &arts[top_art];
 1956         set_article(art);
 1957         art->artnum = *top = artnum;
 1958 
 1959         /*
 1960          * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
 1961          *       to check for additions like we do with xref_supported
 1962          */
 1963         for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
 1964             /* skip unexpected tailing fields */
 1965             if (count > over_fields) {
 1966 #ifdef DEBUG
 1967                 if ((debug & DEBUG_NNTP) && verbose > 1)
 1968                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
 1969 #endif /* DEBUG */
 1970 
 1971                 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
 1972                 if (count == over_fields + 1) {
 1973                     if (!strncasecmp(ptr, "Xref: ", 6)) {
 1974 #ifdef DEBUG
 1975                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1976                             debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
 1977 #endif /* DEBUG */
 1978                         over_fields++;
 1979                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 1980                         ofmt[over_fields].type = OVER_T_FSTRING;
 1981                         ofmt[over_fields].name = my_strdup("Xref:");
 1982                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 1983                         ofmt[over_fields + 1].name = NULL;
 1984                         xref_supported = TRUE;
 1985                     } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
 1986 #ifdef DEBUG
 1987                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1988                             debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
 1989 #endif /* DEBUG */
 1990                         over_fields++;
 1991                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 1992                         ofmt[over_fields].type = OVER_T_FSTRING;
 1993                         ofmt[over_fields].name = my_strdup("Path:");
 1994                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 1995                         ofmt[over_fields + 1].name = NULL;
 1996                         xref_supported = TRUE;
 1997                     } else
 1998                         continue;
 1999                 } else
 2000                     continue;
 2001             }
 2002 
 2003             /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
 2004             if (expensive_over_parse) { /* strange order */
 2005                 /* mandatory fields */
 2006                 if (ofmt[count].type == OVER_T_STRING) {
 2007                     if (!strcasecmp(ofmt[count].name, "Subject:")) {
 2008                         if (*ptr) {
 2009 #ifdef HAVE_UNICODE_NORMALIZATION
 2010                             if (IS_LOCAL_CHARSET("UTF-8"))
 2011                                 q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2012                             else
 2013 #endif /* HAVE_UNICODE_NORMALIZATION */
 2014                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2015 
 2016                             art->subject = hash_str(q);
 2017                             free(q);
 2018                         } else {
 2019                             art->subject = hash_str("");
 2020 #ifdef DEBUG
 2021                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2022                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2023 #endif /* DEBUG */
 2024                         }
 2025                         continue;
 2026                     }
 2027 
 2028                     if (!strcasecmp(ofmt[count].name, "From:")) {
 2029                         if (*ptr) {
 2030                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2031                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2032                             if (*art_full_name)
 2033                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2034                         } else {
 2035                             art->from = hash_str("");
 2036 #ifdef DEBUG
 2037                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2038                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2039 #endif /* DEBUG */
 2040                         }
 2041                         continue;
 2042                     }
 2043 
 2044                     if (!strcasecmp(ofmt[count].name, "Date:")) {
 2045                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2046 #ifdef DEBUG
 2047                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2048                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2049 #endif /* DEBUG */
 2050                         continue;
 2051                     }
 2052 
 2053                     if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
 2054                         if (*ptr) {
 2055                             FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
 2056                             art->msgid = my_strdup(ptr);
 2057                         } else {
 2058                             art->msgid = NULL;
 2059 #ifdef DEBUG
 2060                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2061                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2062 #endif /* DEBUG */
 2063                         }
 2064                         continue;
 2065                     }
 2066 
 2067                     if (!strcasecmp(ofmt[count].name, "References:")) {
 2068                         if (*ptr) {
 2069                             FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
 2070                             art->refs = my_strdup(ptr);
 2071                         } else
 2072                             art->refs = NULL;
 2073                         continue;
 2074                     }
 2075 
 2076                     /*
 2077                      * non std. fields when doing
 2078                      * expensive overview parsing (very
 2079                      * rare, just happens if RFC 3977
 2080                      * 8.4.2 is violated) go here
 2081                      */
 2082                     /* for Path:-filter */
 2083                     if (!strcasecmp(ofmt[count].name, "Path:")) {
 2084                         if (!path_found)
 2085                             path_found = TRUE;
 2086                         if (*ptr) {
 2087                             FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
 2088                             art->path = my_strdup(ptr);
 2089                         } else
 2090                             art->path = NULL;
 2091                         continue;
 2092                     }
 2093                 }
 2094                 /* metadata fields */
 2095                 if (ofmt[count].type == OVER_T_INT) {
 2096                     if (!strcasecmp(ofmt[count].name, "Bytes:")) {
 2097                         if (*ptr) {
 2098 #ifdef DEBUG
 2099                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2100                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2101 #endif /* DEBUG */
 2102                         }
 2103                         continue;
 2104                     }
 2105 
 2106                     if (!strcasecmp(ofmt[count].name, "Lines:")) {
 2107                         if (*ptr) {
 2108                             if (isdigit((unsigned char) *ptr))
 2109                                 art->line_count = atoi(ptr);
 2110                             else {
 2111                                 art->line_count = 0;
 2112 #ifdef DEBUG
 2113                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2114                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2115 #endif /* DEBUG */
 2116                             }
 2117                         } else
 2118                             art->line_count = 0;
 2119                         continue;
 2120                     }
 2121                 }
 2122             } else { /* first 7 fields are in RFC 3977 order */
 2123                 switch (count) {
 2124                     case 1: /* Subject: */
 2125                         if (*ptr) {
 2126 #ifdef HAVE_UNICODE_NORMALIZATION
 2127                             if (IS_LOCAL_CHARSET("UTF-8"))
 2128                                 q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2129                             else
 2130 #endif /* HAVE_UNICODE_NORMALIZATION */
 2131                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2132 
 2133                             art->subject = hash_str(q);
 2134                             free(q);
 2135                         }
 2136                         else {
 2137                             art->subject = hash_str("");
 2138 #ifdef DEBUG
 2139                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2140                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2141 #endif /* DEBUG */
 2142                         }
 2143                         break;
 2144 
 2145                     case 2: /* From: */
 2146                         if (*ptr) {
 2147                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2148                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2149                             if (*art_full_name)
 2150                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2151                         } else {
 2152                             art->from = hash_str("");
 2153 #ifdef DEBUG
 2154                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2155                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2156 #endif /* DEBUG */
 2157                         }
 2158                         break;
 2159 
 2160                     case 3: /* Date: */
 2161                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2162 #ifdef DEBUG
 2163                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2164                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2165 #endif /* DEBUG */
 2166                         break;
 2167 
 2168                     case 4: /* Message-ID: */
 2169                         if (*ptr)
 2170                             art->msgid = my_strdup(ptr);
 2171                         else {
 2172                             art->msgid = NULL;
 2173 #ifdef DEBUG
 2174                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2175                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2176 #endif /* DEBUG */
 2177                         }
 2178                         break;
 2179 
 2180                     case 5: /* References: */
 2181                         if (*ptr)
 2182                             art->refs = my_strdup(ptr);
 2183                         else
 2184                             art->refs = NULL;
 2185                         break;
 2186 
 2187                     case 6: /* :bytes || Bytes: */
 2188                         if (*ptr) {
 2189 #ifdef DEBUG
 2190                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2191                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2192 #endif /* DEBUG */
 2193                         }
 2194                         break;
 2195 
 2196                     case 7: /* :lines || Lines: */
 2197                         if (*ptr) {
 2198                             if (isdigit((unsigned char) *ptr))
 2199                                 art->line_count = atoi(ptr);
 2200                             else {
 2201                                 art->line_count = 0;
 2202 #ifdef DEBUG
 2203                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2204                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2205 #endif /* DEBUG */
 2206                             }
 2207                         } else
 2208                             art->line_count = 0;
 2209                         break;
 2210 
 2211                     default:
 2212                         break;
 2213                 }
 2214             }
 2215 
 2216             /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
 2217             if (ofmt[count].type == OVER_T_FSTRING) {
 2218                 if (*ptr) {
 2219                     if (!strcasecmp(ofmt[count].name, "Xref:")) {
 2220                         if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
 2221                             FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
 2222                             art->xref = my_strdup(q);
 2223                         }
 2224 #ifdef DEBUG
 2225                         else {
 2226                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2227                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2228                         }
 2229 #endif /* DEBUG */
 2230                         continue;
 2231                     }
 2232                     /*
 2233                      * handling of addition overview fields
 2234                      * goes here
 2235                      */
 2236 #ifdef DEBUG
 2237                     if ((debug & DEBUG_NNTP) && verbose > 1)
 2238                         debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
 2239 #endif /* DEBUG */
 2240                     /* if we're lucky we've Path in NOV */
 2241                     /*
 2242                      * if reading locally cached overview data try
 2243                      * path regardless of the server OVERVIEW.FMT
 2244                      */
 2245                     if (local || !strcasecmp(ofmt[count].name, "Path:")) {
 2246                         if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
 2247                             if (!path_found)
 2248                                 path_found = TRUE;
 2249                             FreeIfNeeded(art->path);
 2250                             art->path = my_strdup(q);
 2251 #ifdef DEBUG
 2252                             if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
 2253                                 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
 2254 #endif /* DEBUG */
 2255 
 2256                         }
 2257                         continue;
 2258                     }
 2259                 }
 2260                 continue;
 2261             }
 2262         }
 2263 
 2264         /*
 2265          * RFC says Message-ID is mandatory in newsgroups (but not in
 2266          * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
 2267          * threading in mailgroups
 2268          */
 2269         if (!art->msgid && group->type == GROUP_TYPE_NEWS)
 2270             continue;
 2271 
 2272         /* we might lose accuracy here, but that shouldn't hurt */
 2273         if (artnum % MODULO_COUNT_NUM == 0)
 2274             show_progress(group_msg, artnum - min, max - min);
 2275 
 2276         top_art++;              /* Basically this statement commits the article */
 2277     }
 2278 
 2279     free(group_msg);
 2280     TIN_FCLOSE(fp);
 2281 
 2282     if (tin_errno)
 2283         return -1;
 2284 
 2285 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
 2286     if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
 2287         char cbuf[HEADER_LEN];
 2288         int i;
 2289         static t_bool found;
 2290         static t_bool first = TRUE;
 2291 
 2292         if (first) {
 2293             found = TRUE;
 2294             /*
 2295              * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
 2296              */
 2297             if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
 2298                 if (!*nntp_caps.headers_range) {
 2299                     i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
 2300 
 2301                     found = FALSE;
 2302                     switch (i) {
 2303                         case 215:
 2304                             while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 2305 #   ifdef DEBUG
 2306                                 if (debug & DEBUG_NNTP)
 2307                                     debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2308 #   endif /* DEBUG */
 2309                                 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
 2310                                     found = TRUE;
 2311                                 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 2312                                 strcat(nntp_caps.headers_range, ptr);
 2313                                 strcat(nntp_caps.headers_range, "\n");
 2314                             }
 2315                             break;
 2316 
 2317                         default:
 2318                             break;
 2319                     }
 2320                     first = FALSE;
 2321                 } else {
 2322                     found = FALSE;
 2323                     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
 2324                         do {
 2325                             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
 2326                                 found = TRUE;
 2327                         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 2328                     }
 2329                 }
 2330             }
 2331         }
 2332 
 2333         if (found) {
 2334             snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
 2335             group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
 2336             if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
 2337                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
 2338 #   ifdef DEBUG
 2339                     if ((debug & DEBUG_NNTP) && verbose)
 2340                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2341 #   endif /* DEBUG */
 2342 
 2343                     artnum = atoartnum(ptr);
 2344                     if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
 2345                         continue;
 2346                     art = &arts[top_art];
 2347                     set_article(art);
 2348                     if (!art->xref && !strstr(ptr, "(none)")) {
 2349                         if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
 2350                             continue;
 2351                         ptr = q;
 2352                         while (*ptr && isspace((int) *ptr))
 2353                             ptr++;
 2354                         q = strchr(ptr, '\n');
 2355                         if (q)
 2356                             *q = '\0';
 2357                         art->xref = my_strdup(ptr);
 2358                     }
 2359                     /* we might lose accuracy here, but that shouldn't hurt */
 2360                     if (artnum % MODULO_COUNT_NUM == 0)
 2361                         show_progress(group_msg, artnum - min, max - min);
 2362                 }
 2363             }
 2364             free(group_msg);
 2365         }
 2366     }
 2367 #endif /* NNTP_ABLE && XHDR_XREF */
 2368 
 2369     if (local) {
 2370 #ifdef NNTP_ABLE
 2371         if (filter_on_path(group)) {
 2372             struct t_article_range *ranges, *curr;
 2373             t_bool supported = TRUE;
 2374             int curr_range, range_cnt;
 2375 
 2376             /*
 2377              * Get the ranges without Path: header and try to fetch the
 2378              * headers
 2379              */
 2380             if ((ranges = build_range_list(min, *top, &range_cnt))) {
 2381                 curr = ranges;
 2382                 curr_range = 1;
 2383                 while (curr && supported) {
 2384                     if (curr->cnt)
 2385                         supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
 2386                     curr = curr->next;
 2387                 }
 2388                 if (!supported && path_in_ofmt) {
 2389                     /*
 2390                      * fetching Path: headers via [X]HDR or XPAT has failed
 2391                      * Path: is in the servers overview so let the next
 2392                      * read_overview() fetch them
 2393                      */
 2394                     free_art_array();
 2395                     free_msgids();
 2396                     top_art = 0;
 2397                     *top = T_ARTNUM_CONST(0);
 2398                     expired = 0;
 2399                 }
 2400                 *rebuild_cache = TRUE;
 2401                 while (ranges) {
 2402                     curr = ranges;
 2403                     ranges = curr->next;
 2404                     free(curr);
 2405                 }
 2406             }
 2407         }
 2408 #endif /* NNTP_ABLE */
 2409     } else
 2410         if (!path_found && filter_on_path(group)) {
 2411 #ifdef NNTP_ABLE
 2412             if (!get_path_header(1, 1, group, min, *top))
 2413 #endif /* NNTP_ABLE */
 2414                 wait_message(2, _(txt_cannot_filter_on_path));
 2415         }
 2416     return expired;
 2417 }
 2418 
 2419 
 2420 /*
 2421  * Write an Nov/Xover index file. Fields are separated by '\t'.
 2422  *
 2423  * Format:
 2424  *  1. article number (ie. 183)                [mandatory]
 2425  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 2426  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 2427  *  4. Date: line     (rfc822 format)          [mandatory]
 2428  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 2429  *  6. References:    (ie. <message-id> ....)  [optional]
 2430  *  7. Byte count     (Skipped - not used)     [mandatory]
 2431  *  8. Line count     (ie. 23)                 [mandatory]
 2432  *  9. Xref: line     (ie. alt.test:389)       [optional]
 2433  *
 2434  * TODO: as we don't use the original data, we currently can't store
 2435  *       the data (from/subject) in the original charset (we don't store
 2436  *       that info). this has the advantage that we can avoid raw 8bit data
 2437  *       in our overviews, but the disadvantage that we might store the data
 2438  *       with a wrong charset and thus lose information. a similar problem
 2439  *       exists with the data for the from:-line, we don't store it in the
 2440  *       original format, whenever our from-parser (partially) fails we'll
 2441  *       lose information in our overviews (but those couldn't be handled
 2442  *       by tin anyway, so this is not a real problem).
 2443  *       long-term solution: store the original data in the overview
 2444  *       (tin has to handle raw 8bit data and other ugly stuff in the
 2445  *       overviews anyway and thus we preserver as much info as possible)
 2446  *       this would require some changes in read_overview() and
 2447  *       parse_headers(): don't do the decoding/unfolding there, but in a
 2448  *       second pass right after write_overview(), or two additional fields
 2449  *       which hold the raw data for from/subject. the latter has the
 2450  *       disadvantage that it costs (much) more memory.
 2451  */
 2452 static void
 2453 write_overview(
 2454     struct t_group *group)
 2455 {
 2456     FILE *fp;
 2457     int i;
 2458     struct t_article *article;
 2459 #ifdef CHARSET_CONVERSION
 2460     int c = -1;
 2461 #endif /* CHARSET_CONVERSION */
 2462 
 2463     /*
 2464      * Can't write or caching is off or getart_limit is set
 2465      */
 2466     if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
 2467         return;
 2468 
 2469     if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
 2470         return;
 2471 
 2472     if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
 2473         SortBy(artnum_comp);
 2474 
 2475     /*
 2476      * Needed to preserve uniqueness in hashed private overview files
 2477      */
 2478     fprintf(fp, "%s\n", group->name);
 2479 
 2480 #ifdef CHARSET_CONVERSION
 2481     /* get undeclared_charset number if required */
 2482     if (group->attribute->undeclared_charset) {
 2483         for (i = 0; txt_mime_charsets[i] != NULL; i++) {
 2484             if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
 2485                 c = i;
 2486                 break;
 2487             }
 2488         }
 2489     }
 2490 #endif /* CHARSET_CONVERSION */
 2491 
 2492     for_each_art(i) {
 2493         char *p;
 2494         char *q, *ref;
 2495 
 2496         article = &arts[i];
 2497 
 2498         if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
 2499             ref = NULL;
 2500 
 2501             if (!group->attribute->post_8bit_header) { /* write encoded data */
 2502                 /*
 2503                  * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
 2504                  *       here and in print_from() in the CHARSET_CONVERSION case.
 2505                  *       note that this requires something like
 2506                  *          buffer_to_network(article->subject, "UTF-8");
 2507                  *       right before the rfc1522_encode() call.
 2508                  *
 2509                  *       if we would cache the original undecoded data, we could
 2510                  *       ignore stuff like this.
 2511                  */
 2512                 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
 2513                 /* as the subject might now be folded we have to unfold it */
 2514                 unfold_header(p);
 2515             } else { /* raw data */
 2516                 p = my_strdup(article->subject);
 2517 #ifdef CHARSET_CONVERSION
 2518                 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
 2519                     buffer_to_network(p, c);
 2520 #endif /* CHARSET_CONVERSION */
 2521             }
 2522 
 2523             /*
 2524              * replace any '\t's with ' ' in the references-data
 2525              *
 2526              * TODO: nntpext-draft might come up with a new scheme:
 2527              *       For all fields, the value is processed by first
 2528              *       removing all US-ASCII CRLF pairs and then replacing
 2529              *       each remaining US-ASCII NUL, TAB, CR, or LF character
 2530              *       with a single US-ASCII space (for example, CR LF LF TAB
 2531              *       will become two spaces).
 2532              */
 2533             if (article->refs) {
 2534                 ref = q = my_strdup(article->refs);
 2535                 while (*q) {
 2536                     if (*q == '\t')
 2537                         *q = ' ';
 2538                     q++;
 2539                 }
 2540             }
 2541 
 2542             fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
 2543                 article->artnum,
 2544                 p,
 2545 #ifdef CHARSET_CONVERSION
 2546                 print_from(group, article, c),
 2547 #else
 2548                 print_from(group, article, -1),
 2549 #endif /* CHARSET_CONVERSION */
 2550                 print_date(article->date),
 2551                 BlankIfNull(article->msgid),
 2552                 BlankIfNull(ref),
 2553                 0,  /* bytes */
 2554                 article->line_count);
 2555 
 2556             if (article->xref)
 2557                 fprintf(fp, "\tXref: %s", article->xref);
 2558 
 2559             if (article->path)
 2560                 fprintf(fp, "\tPath: %s", article->path);
 2561 
 2562             fprintf(fp, "\n");
 2563             free(p);
 2564             if (article->refs) {
 2565                 FreeIfNeeded(ref);
 2566                 q = NULL;
 2567             }
 2568         }
 2569     }
 2570     fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
 2571     fclose(fp);
 2572 }
 2573 
 2574 
 2575 /*
 2576  * A complex little function to determine the correct overview index file
 2577  * according to 'mode' (read or write)
 2578  * NULL is returned if the current setup dictates otherwise
 2579  *
 2580  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
 2581  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
 2582  *
 2583  * Both of these are hashed
 2584  *
 2585  * GROUP_TYPE_NEWS index files are a little bit more complex
 2586  *
 2587  * When hashing the index filename will be in format number.number.
 2588  * Hashing the groupname gets a number. See if that #.1 file exists;
 2589  * if so, read first line. Is this the group we want? If no, try #.2.
 2590  * Repeat until no such file or we find an existing file that matches
 2591  * our group. Return pointer to path or NULL if not found.
 2592  */
 2593 static char *
 2594 find_nov_file(
 2595     struct t_group *group,
 2596     int mode)
 2597 {
 2598     FILE *fp;
 2599     const char *dir;
 2600     char buf[PATH_LEN];
 2601     int i;
 2602     struct stat sb;
 2603     unsigned long hash;
 2604     static char nov_file[PATH_LEN];
 2605     static t_bool once_only = FALSE;    /* Trap things that are done only 1 time */
 2606 
 2607     if (group == NULL || (mode != R_OK && mode != W_OK))
 2608         return NULL;
 2609 
 2610     switch (group->type) {
 2611         case GROUP_TYPE_MAIL:
 2612             dir = index_maildir;
 2613             break;
 2614 
 2615         case GROUP_TYPE_SAVE:
 2616             dir = index_savedir;
 2617             break;
 2618 
 2619         case GROUP_TYPE_NEWS:
 2620             /*
 2621              * nntp.caps.over_cmd is not an issue here, any gripes and warnings
 2622              * about [X]OVER are handled in nntp_open()
 2623              */
 2624 
 2625             /*
 2626              * When reading via NNTP, system wide overviews are irrelevant, of
 2627              * course, and the private overview filename will be the same for
 2628              * both reading and writing.
 2629              *
 2630              * When working locally, we only use a private cache for reading
 2631              * if requested and when system wide overviews don't already exist.
 2632              * When writing then only private overviews can be used since
 2633              * updating system wide overviews is not safe wrt locking etc.
 2634              *
 2635              * See if local overview file $SPOOLDIR/<groupname>/.overview exists
 2636              */
 2637 #ifndef NNTP_ONLY
 2638             if (!read_news_via_nntp) {
 2639                 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
 2640                 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
 2641                 if (access(nov_file, R_OK) == 0) {
 2642                     if (mode == R_OK)
 2643                         return nov_file;        /* Use system wide overviews */
 2644                     else
 2645                         return NULL;            /* Don't write cache in this case */
 2646                 }
 2647             }
 2648 #endif /* !NNTP_ONLY */
 2649 
 2650             /*
 2651              * We only get here when private overviews are going to be used
 2652              * Go no further if they are explicitly turned off
 2653              */
 2654             if (!tinrc.cache_overview_files)
 2655                 return NULL;
 2656 
 2657             /*
 2658              * Append -<nntpserver> to private cache dir
 2659              */
 2660             if (!once_only && nntp_server) {
 2661                 size_t sp, ln = strlen(index_newsdir);
 2662 
 2663                 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
 2664                     char *srv = my_strdup(nntp_server);
 2665 
 2666                     strcat(index_newsdir, "-");
 2667                     sp--;
 2668                     ln++;
 2669                     str_lwr(srv);
 2670                     my_strncpy(index_newsdir + ln, srv, sp);
 2671                     free(srv);
 2672                 }
 2673                 once_only = TRUE;
 2674             }
 2675 
 2676             /*
 2677              * Only try to set up the private cache when writing. If it
 2678              * doesn't exist yet, then ergo we can't read from it.
 2679              * The cache will be checked/created on every write; a previous
 2680              * bug report complained that this was not the case
 2681              */
 2682             if (stat(index_newsdir, &sb) == -1) {           /* Private cache doesn't exist */
 2683                 if (mode == R_OK)
 2684                     return NULL;
 2685                 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
 2686                     return NULL;
 2687             } else {
 2688                 if (!S_ISDIR(sb.st_mode))
 2689                     return NULL;
 2690             }
 2691 
 2692             /*
 2693              * Update the newsgroups cache to point to the new location
 2694              * now that we know it is valid
 2695              */
 2696             if (!once_only)
 2697                 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
 2698 
 2699             dir = index_newsdir;
 2700             break;
 2701 
 2702         default: /* not reached */
 2703             return NULL;
 2704     }
 2705 
 2706     /*
 2707      * We only get here if writing to a private overview.
 2708      * These always have hashed filenames.
 2709      * Try <hash>.<seqno> and check the group name tagline until
 2710      * matching index file is found. If not found return next unused
 2711      * filename
 2712      */
 2713     hash = hash_groupname(group->name);
 2714 
 2715     for (i = 1; ; i++) {
 2716         char *ptr;
 2717 
 2718         snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
 2719         joinpath(nov_file, sizeof(nov_file), dir, buf);
 2720 
 2721         if ((fp = fopen(nov_file, "r")) == NULL)
 2722             break;
 2723 
 2724         /*
 2725          * No group name header, so not a valid index file => overwrite it
 2726          */
 2727         if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
 2728             fclose(fp);
 2729             break;
 2730         }
 2731         fclose(fp);
 2732 
 2733         if ((ptr = strrchr(buf, '\n')) != NULL)
 2734             *ptr = '\0';
 2735 
 2736         if (strcmp(buf, group->name) == 0)
 2737             break;
 2738     }
 2739 
 2740     return nov_file;
 2741 }
 2742 
 2743 
 2744 /*
 2745  * Run the index file updater only for the groups we've loaded.
 2746  */
 2747 void
 2748 do_update(
 2749     t_bool catchup)
 2750 {
 2751     int i, j, k = 0;
 2752     time_t beg_epoch = 0;
 2753     struct t_article *art;
 2754     struct t_group *group;
 2755 
 2756     if (verbose)
 2757         (void) time(&beg_epoch);
 2758 
 2759     /*
 2760      * loop through groups and update any required index files
 2761      */
 2762     for (i = 0; i < selmenu.max; i++) {
 2763         group = &active[my_group[i]];
 2764         /*
 2765          * FIXME: workaround to get a valid CURR_GROUP
 2766          * it also points to the currently processed group so that
 2767          * the correct attributes are used
 2768          * The correct fix is to get rid of CURR_GROUP
 2769          */
 2770         selmenu.curr = i;
 2771 
 2772         if (group->bogus || !group->subscribed)
 2773             continue;
 2774 
 2775         if (!index_group(group)) {
 2776             for_each_art(j) {
 2777                 art = &arts[j];
 2778                 FreeAndNull(art->refs);
 2779                 FreeAndNull(art->msgid);
 2780             }
 2781             continue;
 2782         }
 2783 
 2784         k++;
 2785 
 2786         if (verbose) {
 2787             my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
 2788             my_flush();
 2789         }
 2790 
 2791         if (catchup) {
 2792             for_each_art(j)
 2793                 art_mark(group, &arts[j], ART_READ);
 2794         }
 2795     }
 2796 
 2797     if (verbose) {
 2798         wait_message(0, _(txt_catchup_update_info),
 2799             (catchup ? _(txt_caughtup) : _(txt_updated)), k,
 2800             PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
 2801     }
 2802 }
 2803 
 2804 
 2805 static int
 2806 artnum_comp(
 2807     t_comptype p1,
 2808     t_comptype p2)
 2809 {
 2810     const struct t_article *s1 = (const struct t_article *) p1;
 2811     const struct t_article *s2 = (const struct t_article *) p2;
 2812 
 2813     /*
 2814      * s1->artnum less than s2->artnum
 2815      */
 2816     if (s1->artnum < s2->artnum)
 2817         return -1;
 2818 
 2819     /*
 2820      * s1->artnum greater than s2->artnum
 2821      */
 2822     if (s1->artnum > s2->artnum)
 2823         return 1;
 2824 
 2825     return 0;
 2826 }
 2827 
 2828 
 2829 /*
 2830  * return result of strcmp (reversed for descending)
 2831  */
 2832 static int
 2833 subj_comp_asc(
 2834     t_comptype p1,
 2835     t_comptype p2)
 2836 {
 2837     int retval;
 2838     const struct t_article *s1 = (const struct t_article *) p1;
 2839     const struct t_article *s2 = (const struct t_article *) p2;
 2840 
 2841     if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
 2842         return retval;
 2843 
 2844     return s1->date - s2->date > 0 ? 1 : -1;
 2845 }
 2846 
 2847 
 2848 static int
 2849 subj_comp_desc(
 2850     t_comptype p1,
 2851     t_comptype p2)
 2852 {
 2853     int retval;
 2854     const struct t_article *s1 = (const struct t_article *) p1;
 2855     const struct t_article *s2 = (const struct t_article *) p2;
 2856 
 2857     if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
 2858         return retval;
 2859 
 2860     return s1->date - s2->date > 0 ? 1 : -1;
 2861 }
 2862 
 2863 
 2864 /*
 2865  * return result of strcmp (reversed for descending)
 2866  */
 2867 static int
 2868 from_comp_asc(
 2869     t_comptype p1,
 2870     t_comptype p2)
 2871 {
 2872     int retval;
 2873     const struct t_article *s1 = (const struct t_article *) p1;
 2874     const struct t_article *s2 = (const struct t_article *) p2;
 2875 
 2876     if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
 2877         return retval;
 2878 
 2879     return s1->date - s2->date > 0 ? 1 : -1;
 2880 }
 2881 
 2882 
 2883 static int
 2884 from_comp_desc(
 2885     t_comptype p1,
 2886     t_comptype p2)
 2887 {
 2888     int retval;
 2889     const struct t_article *s1 = (const struct t_article *) p1;
 2890     const struct t_article *s2 = (const struct t_article *) p2;
 2891 
 2892     if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
 2893         return retval;
 2894 
 2895     return s1->date - s2->date > 0 ? 1 : -1;
 2896 }
 2897 
 2898 
 2899 /*
 2900  * Works like strcmp() for comparing time_t type values
 2901  * Return codes:
 2902  *  -1:     If p1 is before p2
 2903  *   0:     If they are the same time
 2904  *   1:     If p1 is after p2
 2905  * If the sort order is _not_ DATE_ASCEND then the sense of the above
 2906  * is reversed.
 2907  */
 2908 static int
 2909 date_comp_asc(
 2910     t_comptype p1,
 2911     t_comptype p2)
 2912 {
 2913     const struct t_article *s1 = (const struct t_article *) p1;
 2914     const struct t_article *s2 = (const struct t_article *) p2;
 2915 
 2916     /*
 2917      * s1->date less than s2->date
 2918      */
 2919     if (s1->date < s2->date)
 2920         return -1;
 2921 
 2922     /*
 2923      * s1->date greater than s2->date
 2924      */
 2925     if (s1->date > s2->date)
 2926         return 1;
 2927 
 2928     return 0;
 2929 }
 2930 
 2931 
 2932 static int
 2933 date_comp_desc(
 2934     t_comptype p1,
 2935     t_comptype p2)
 2936 {
 2937     const struct t_article *s1 = (const struct t_article *) p1;
 2938     const struct t_article *s2 = (const struct t_article *) p2;
 2939 
 2940     /*
 2941      * s2->date less than s1->date
 2942      */
 2943     if (s2->date < s1->date)
 2944         return -1;
 2945 
 2946     /*
 2947      * s2->date greater than s1->date
 2948      */
 2949     if (s2->date > s1->date)
 2950         return 1;
 2951 
 2952     return 0;
 2953 }
 2954 
 2955 
 2956 /*
 2957  * Same again, but for art[].score
 2958  */
 2959 static int
 2960 score_comp_asc(
 2961     t_comptype p1,
 2962     t_comptype p2)
 2963 {
 2964     const struct t_article *s1 = (const struct t_article *) p1;
 2965     const struct t_article *s2 = (const struct t_article *) p2;
 2966 
 2967     if (s1->score < s2->score)
 2968         return -1;
 2969 
 2970     if (s1->score > s2->score)
 2971         return 1;
 2972 
 2973     return s1->date - s2->date > 0 ? 1 : -1;
 2974 }
 2975 
 2976 
 2977 static int
 2978 score_comp_desc(
 2979     t_comptype p1,
 2980     t_comptype p2)
 2981 {
 2982     const struct t_article *s1 = (const struct t_article *) p1;
 2983     const struct t_article *s2 = (const struct t_article *) p2;
 2984 
 2985     if (s2->score < s1->score)
 2986         return -1;
 2987 
 2988     if (s2->score > s1->score)
 2989         return 1;
 2990 
 2991     return s1->date - s2->date > 0 ? 1 : -1;
 2992 }
 2993 
 2994 
 2995 /*
 2996  * Same again, but for art[].line_count
 2997  */
 2998 static int
 2999 lines_comp_asc(
 3000     t_comptype p1,
 3001     t_comptype p2)
 3002 {
 3003     const struct t_article *s1 = (const struct t_article *) p1;
 3004     const struct t_article *s2 = (const struct t_article *) p2;
 3005 
 3006     if (s1->line_count < s2->line_count)
 3007         return -1;
 3008 
 3009     if (s1->line_count > s2->line_count)
 3010         return 1;
 3011 
 3012     return s1->date - s2->date > 0 ? 1 : -1;
 3013 }
 3014 
 3015 
 3016 static int
 3017 lines_comp_desc(
 3018     t_comptype p1,
 3019     t_comptype p2)
 3020 {
 3021     const struct t_article *s1 = (const struct t_article *) p1;
 3022     const struct t_article *s2 = (const struct t_article *) p2;
 3023 
 3024     if (s2->line_count < s1->line_count)
 3025         return -1;
 3026 
 3027     if (s2->line_count > s1->line_count)
 3028         return 1;
 3029 
 3030     return s1->date - s2->date > 0 ? 1 : -1;
 3031 }
 3032 
 3033 
 3034 /*
 3035  * Compares the total score of two threads. Used for sorting base[].
 3036  */
 3037 static int
 3038 score_comp_base(
 3039     t_comptype p1,
 3040     t_comptype p2)
 3041 {
 3042     int a = get_score_of_thread(*(const long *) p1);
 3043     int b = get_score_of_thread(*(const long *) p2);
 3044 
 3045     /* If scores are equal, compare using the article sort order.
 3046      * This determines the order in a group of equally scored threads.
 3047      */
 3048     if (a == b) {
 3049         const struct t_article *s1 = &arts[*(const long *) p1];
 3050         const struct t_article *s2 = &arts[*(const long *) p2];
 3051         t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
 3052 
 3053         if (comp_func)
 3054             return (*comp_func)(s1, s2);
 3055         return 0;
 3056     }
 3057 
 3058     if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
 3059         return a > b ? 1 : -1;
 3060     return a < b ? 1 : -1;
 3061 }
 3062 
 3063 
 3064 /*
 3065  * Compare the date of the last posted article of two threads.
 3066  * Used for sorting base[].
 3067  */
 3068 static int
 3069 last_date_comp_base_desc(
 3070     t_comptype p1,
 3071     t_comptype p2)
 3072 {
 3073     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3074     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3075 
 3076     if (s2_last < s1_last)
 3077         return -1;
 3078 
 3079     if (s2_last > s1_last)
 3080         return 1;
 3081 
 3082     return 0;
 3083 }
 3084 
 3085 
 3086 static int
 3087 last_date_comp_base_asc(
 3088     t_comptype p1,
 3089     t_comptype p2)
 3090 {
 3091     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3092     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3093 
 3094     if (s2_last > s1_last)
 3095         return -1;
 3096 
 3097     if (s2_last < s1_last)
 3098         return 1;
 3099 
 3100     return 0;
 3101 }
 3102 
 3103 
 3104 static time_t
 3105 get_last_posting_date(
 3106     long n)
 3107 {
 3108     long i;
 3109     time_t last = (time_t) 0;
 3110 
 3111     for (i = n; i >= 0; i = arts[i].thread) {
 3112         if (arts[i].date > last)
 3113             last = arts[i].date;
 3114     }
 3115 
 3116     return last;
 3117 }
 3118 
 3119 
 3120 void
 3121 set_article(
 3122     struct t_article *art)
 3123 {
 3124     art->subject = NULL;
 3125     art->from = NULL;
 3126     art->name = NULL;
 3127     art->date = (time_t) 0;
 3128     art->xref = NULL;
 3129     art->msgid = NULL;
 3130     art->refs = NULL;
 3131     art->refptr = NULL;
 3132     art->line_count = -1;
 3133     art->archive = NULL;
 3134     art->tagged = 0;
 3135     art->thread = ART_EXPIRED;
 3136     art->prev = ART_NORMAL;
 3137     art->score = 0;
 3138     art->status = ART_UNREAD;
 3139     art->killed = ART_NOTKILLED;
 3140     art->zombie = FALSE;
 3141     art->delete_it = FALSE;
 3142     art->selected = FALSE;
 3143     art->inrange = FALSE;
 3144     art->matched = FALSE;
 3145     art->keep_in_base = FALSE;
 3146 }
 3147 
 3148 
 3149 /*
 3150  * Do a binary chop to see if 'art' (an article number) exists in arts[]
 3151  * Naturally arts[] must be sorted on artnum
 3152  * Return index into arts[] or -1
 3153  */
 3154 static int
 3155 valid_artnum(
 3156     t_artnum art)
 3157 {
 3158     int prev, range;
 3159     int dctop = top_art;
 3160     int cur = 1;
 3161 
 3162     while ((dctop >>= 1))
 3163         cur <<= 1;
 3164 
 3165     range = cur >> 1;
 3166     cur--;
 3167 
 3168     forever {
 3169         if (arts[cur].artnum == art)
 3170             return cur;
 3171 
 3172         prev = cur;
 3173         cur += ((arts[cur].artnum < art) ? range : -range);
 3174         if (prev == cur)
 3175             break;
 3176 
 3177         if (cur >= top_art)
 3178             cur = top_art - 1;
 3179 
 3180         range >>= 1;
 3181     }
 3182     return -1;
 3183 }
 3184 
 3185 
 3186 /*
 3187  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
 3188  * Needed if arts[] is not sorted on artnum
 3189  * Return index into arts[] or -1
 3190  */
 3191 int
 3192 find_artnum(
 3193     t_artnum art)
 3194 {
 3195     int i;
 3196 
 3197     for_each_art(i) {
 3198         if (arts[i].artnum == art)
 3199             return i;
 3200     }
 3201     return -1;
 3202 }
 3203 
 3204 
 3205 /*----------------------------- Overview handling -----------------------*/
 3206 /* TODO: use
 3207  *           setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
 3208  *           my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
 3209  *       instead?
 3210  */
 3211 static char *
 3212 print_date(
 3213     time_t secs)
 3214 {
 3215     static char date[25];
 3216     struct tm *tm;
 3217     static const char *const months_a[] = {
 3218         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 3219         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
 3220     };
 3221 
 3222     if ((tm = gmtime(&secs)) != NULL)
 3223         snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
 3224                 tm->tm_mday,
 3225                 months_a[tm->tm_mon],
 3226                 tm->tm_year + 1900,
 3227                 tm->tm_hour, tm->tm_min, tm->tm_sec);
 3228     else
 3229         snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
 3230 
 3231     return date;
 3232 }
 3233 
 3234 
 3235 static char *
 3236 print_from(
 3237     struct t_group *group,
 3238     struct t_article *article,
 3239     int charset)
 3240 {
 3241     char *p, *q;
 3242     static char from[PATH_LEN];
 3243 
 3244     *from = '\0';
 3245 
 3246     if (article->name != NULL) {
 3247         q = my_strdup(article->name);
 3248 #ifdef CHARSET_CONVERSION
 3249         if (charset != -1) {
 3250             buffer_to_network(q, charset);
 3251         }
 3252 #endif /* CHARSET_CONVERSION */
 3253         p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
 3254         unfold_header(p);
 3255         if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
 3256             snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3257         else
 3258             snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3259 
 3260         free(p);
 3261         free(q);
 3262     } else
 3263         STRCPY(from, article->from);
 3264 
 3265     return from;
 3266 }
 3267 
 3268 
 3269 /*
 3270  * Open a group news overview file
 3271  * Use NNTP XOVER where possible unless 'local' is set
 3272  */
 3273 static FILE *
 3274 open_xover_fp(
 3275     struct t_group *group,
 3276     const char *mode,
 3277     t_artnum min,
 3278     t_artnum max,
 3279     t_bool local)
 3280 {
 3281 #ifdef NNTP_ABLE
 3282     if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
 3283         char line[NNTP_STRLEN];
 3284 
 3285         if (!max)
 3286             return NULL;
 3287         if (min == max)
 3288             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
 3289         else
 3290             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
 3291         return (nntp_command(line, OK_XOVER, NULL, 0));
 3292     }
 3293 #endif /* NNTP_ABLE */
 3294     {
 3295         FILE *fp;
 3296         char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
 3297 
 3298         if (nov_file != NULL) {
 3299             if ((fp = fopen(nov_file, mode)) != NULL)
 3300                 return fp;
 3301 
 3302             if (*mode != 'r')
 3303                 error_message(2, _(txt_cannot_open), nov_file);
 3304         }
 3305     }
 3306     return NULL;
 3307 }
 3308 
 3309 
 3310 #ifdef USE_HEAPSORT
 3311 int
 3312 tin_sort(
 3313     void *sbase,
 3314     size_t nel,
 3315     size_t width,
 3316     t_compfunc compar)
 3317 {
 3318     int rc;
 3319 
 3320     switch (tinrc.sort_function) {
 3321         case 0:
 3322             qsort(sbase, nel, width, compar);
 3323             rc = 0;
 3324             break;
 3325 
 3326         case 1:
 3327             rc = heapsort(sbase, nel, width, compar);
 3328             break;
 3329 
 3330         default:
 3331             rc = -1;
 3332             break;
 3333     }
 3334     return rc;
 3335 }
 3336 #endif /* USE_HEAPSORT */