"Fossies" - the Fresh Open Source Software Archive

Member "tin-2.6.2/src/art.c" (9 Dec 2022, 90179 Bytes) of package /linux/misc/tin-2.6.2.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "art.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.6.1_vs_2.6.2.

    1 /*
    2  *  Project   : tin - a Usenet reader
    3  *  Module    : art.c
    4  *  Author    : I.Lea & R.Skrenta
    5  *  Created   : 1991-04-01
    6  *  Updated   : 2022-09-22
    7  *  Notes     :
    8  *
    9  * Copyright (c) 1991-2023 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
   10  * All rights reserved.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  *
   16  * 1. Redistributions of source code must retain the above copyright notice,
   17  *    this list of conditions and the following disclaimer.
   18  *
   19  * 2. Redistributions in binary form must reproduce the above copyright
   20  *    notice, this list of conditions and the following disclaimer in the
   21  *    documentation and/or other materials provided with the distribution.
   22  *
   23  * 3. Neither the name of the copyright holder nor the names of its
   24  *    contributors may be used to endorse or promote products derived from
   25  *    this software without specific prior written permission.
   26  *
   27  * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   28  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   31  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   32  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   33  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   34  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   35  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   36  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   37  * POSSIBILITY OF SUCH DAMAGE.
   38  */
   39 
   40 
   41 #ifndef TIN_H
   42 #   include "tin.h"
   43 #endif /* !TIN_H */
   44 #ifndef TCURSES_H
   45 #   include "tcurses.h"
   46 #endif /* !TCURSES_H */
   47 #ifndef NEWSRC_H
   48 #   include "newsrc.h"
   49 #endif /* !NEWSRC_H */
   50 
   51 #ifndef STPWATCH_H
   52 #   include "stpwatch.h"
   53 #endif /* !STPWATCH_H */
   54 
   55 /*
   56  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
   57  */
   58 #define SortBy(func)    tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
   59 
   60 int top_art = 0;                /* # of articles in arts[] */
   61 
   62 /*
   63  * Local prototypes
   64  */
   65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
   66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
   67 static char *find_nov_file(struct t_group *group, int mode);
   68 static char *print_date(time_t secs);
   69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
   70 static int artnum_comp(t_comptype p1, t_comptype p2);
   71 static int base_comp(t_comptype p1, t_comptype p2);
   72 static int date_comp_asc(t_comptype p1, t_comptype p2);
   73 static int date_comp_desc(t_comptype p1, t_comptype p2);
   74 static int from_comp_asc(t_comptype p1, t_comptype p2);
   75 static int from_comp_desc(t_comptype p1, t_comptype p2);
   76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
   77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
   78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
   79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
   80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
   81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
   82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
   83 static int score_comp_asc(t_comptype p1, t_comptype p2);
   84 static int score_comp_desc(t_comptype p1, t_comptype p2);
   85 static int score_comp_base(t_comptype p1, t_comptype p2);
   86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
   87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
   88 static int valid_artnum(t_artnum art);
   89 static t_artnum find_first_unread(struct t_group *group);
   90 static t_artnum setup_hard_base(struct t_group *group);
   91 static t_bool parse_headers(FILE *fp, struct t_article *h);
   92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
   93 static time_t get_last_posting_date(long n);
   94 static void sort_base(unsigned int sort_threads_type);
   95 static void thread_by_multipart(void);
   96 static void thread_by_percentage(unsigned int percentage);
   97 static void thread_by_subject(void);
   98 static void write_overview(struct t_group *group);
   99 #ifdef NNTP_ABLE
  100     static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
  101     static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
  102 #endif /* NNTP_ABLE */
  103 
  104 
  105 /*
  106  * Display a suitable 'entering group' message if screen needs redrawing
  107  * Allow for the non-printing %s, and the %-age counter
  108  */
  109 void
  110 show_art_msg(
  111     const char *group)
  112 {
  113     wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
  114 }
  115 
  116 
  117 /*
  118  * Construct the pointers to the first (base) article in each thread.
  119  * If we are showing only unread, then point to the first unread. I have
  120  * no idea why this should be so, it causes problems elsewhere [which_response]
  121  */
  122 void
  123 find_base(
  124     struct t_group *group)
  125 {
  126     int i, j;
  127 
  128     grpmenu.max = 0;
  129 
  130 #ifdef DEBUG
  131     if (debug & DEBUG_FILTER)
  132         debug_print_arts();
  133 #endif /* DEBUG */
  134 
  135     for_each_art(i) {
  136         /*
  137          * .prev will be set on each article that is after the first article in
  138          * the thread. Invalid articles which have been expired will have
  139          * .thread set to ART_EXPIRED
  140          */
  141         if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
  142             continue;
  143 
  144         if (grpmenu.max >= max_base)
  145             expand_base();
  146 
  147         if (group->attribute->show_only_unread_arts) {
  148             if (arts[i].status != ART_READ || arts[i].keep_in_base)
  149                 base[grpmenu.max++] = i;
  150             else {
  151                 /* Find 1st unread art in thread */
  152                 for (j = i; j >= 0; j = arts[j].thread) {
  153                     if (arts[j].status != ART_READ || arts[j].keep_in_base) {
  154                         base[grpmenu.max++] = i;
  155                         break;
  156                     }
  157                 }
  158             }
  159         } else
  160             base[grpmenu.max++] = i;
  161     }
  162     /* sort base[] */
  163     if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
  164         sort_base(group->attribute->sort_threads_type);
  165 }
  166 
  167 
  168 /*
  169  * Longword comparison routine for the tin_sort()
  170  */
  171 static int
  172 base_comp(
  173     t_comptype p1,
  174     t_comptype p2)
  175 {
  176     const t_artnum *a = (const t_artnum *) p1;
  177     const t_artnum *b = (const t_artnum *) p2;
  178 
  179     if (*a < *b)
  180         return -1;
  181 
  182     if (*a > *b)
  183         return 1;
  184 
  185     return 0;
  186 }
  187 
  188 
  189 /*
  190  * via NNTP:
  191  *   Issue a LISTGROUP command
  192  *   Read the article numbers existing in the group into base[]
  193  *   If the LISTGROUP failed, issue a GROUP command. Use the results to
  194  *   create a less accurate version of base[]
  195  *   This data will already be sorted
  196  *
  197  * on local spool:
  198  *   Read the spool dir to populate base[] as above. Sort it.
  199  *
  200  * Grow the arts[] and bitmaps as needed.
  201  * NB: the output will be sorted on artnum
  202  *
  203  * grpmenu.max is one past top.
  204  * Returns total number of articles in group, or -1 on error
  205  */
  206 static t_artnum
  207 setup_hard_base(
  208     struct t_group *group)
  209 {
  210     t_artnum total = 0;
  211 
  212     grpmenu.max = 0;
  213 
  214     /*
  215      * If reading with NNTP, issue a LISTGROUP
  216      */
  217     if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
  218 #ifdef NNTP_ABLE
  219         char buf[NNTP_STRLEN];
  220         char line[NNTP_STRLEN];
  221         int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  222         FILE *fp;
  223         t_artnum last, start, count = 0, j = 0;
  224         static t_bool skip_listgroup = FALSE;
  225 
  226         /*
  227          * Some nntp servers are broken and need an extra GROUP command
  228          * (reported by reorx@irc.pl). This affects (old?) versions of
  229          * nntpcache, leafnode and SurgeNews. Usually this should not be
  230          * needed.
  231          *
  232          * For getart_limit recheck lowwatermark as at least giganews gives
  233          * very different results for LIST ACTIVE (3 year retention for all)
  234          * and GROUP (based on the clients contract).
  235          * Calculate range and prepare base[] not to lose unread arts.
  236          */
  237         if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
  238             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  239             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  240                 return -1;
  241 
  242             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
  243                 return -1;
  244 
  245             if (getart_limit > 0) {
  246                 j = group->xmax - getart_limit;
  247                 count = MAX(find_first_unread(group), start);
  248             }
  249             if (getart_limit < 0) {
  250                 j = getart_limit + find_first_unread(group);
  251                 count = group->xmin;
  252             }
  253             if (j < group->xmin)
  254                 j = group->xmin;
  255 
  256             for (; count < j; count++) {
  257                 if (grpmenu.max >= max_base)
  258                     expand_base();
  259                 base[grpmenu.max++] = count;
  260             }
  261         }
  262 
  263         /*
  264          * See if LISTGROUP works
  265          */
  266         if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
  267             if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
  268                 /* RFC 3977 allows ranges in LISTGROUP */
  269                 if (getart_limit > 0)
  270                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
  271                 else /* getart_limit < 0; fetch till newest art */
  272                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
  273 
  274             } else /* for RFC 977 just use GROUP */
  275                 skip_listgroup = TRUE;
  276 
  277         } else /* get all article numbers */
  278             snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
  279 
  280         if (!skip_listgroup) {
  281             if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
  282                 char *ptr;
  283 
  284 #   ifdef DEBUG
  285                 if ((debug & DEBUG_NNTP) && verbose > 1)
  286                     debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  287 #   endif /* DEBUG */
  288 
  289                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
  290                     if (grpmenu.max >= max_base)
  291                         expand_base();
  292                     base[grpmenu.max++] = atoartnum(ptr);
  293                     total++;
  294                 }
  295 
  296                 if (tin_errno)
  297                     return -1;
  298             } else
  299                 skip_listgroup = TRUE;
  300         }
  301 
  302         if (skip_listgroup) { /* LISTGROUP was skipped or failed */
  303             /*
  304              * Handle the obscure case that the user aborted before the LISTGROUP
  305              * had a chance to respond
  306              */
  307             if (tin_errno)
  308                 return -1;
  309 
  310             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  311             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  312                 return -1;
  313 
  314             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
  315                 return -1;
  316 
  317 #   ifdef DEBUG
  318             if ((debug & DEBUG_NNTP) && verbose > 1)
  319                 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  320 #   endif /* DEBUG */
  321             total = count;
  322             grpmenu.max = 0;
  323             if (getart_limit > 0) {
  324                 if ((j = find_first_unread(group)) > start) {
  325                     if (group->xmax > getart_limit) {
  326                         start = MIN(j, group->xmax - getart_limit);
  327                         total = getart_limit;
  328                     } else
  329                         start = j;
  330                 }
  331             }
  332             if (getart_limit < 0) {
  333                 if ((j = (getart_limit + find_first_unread(group))) > start)
  334                     start = j;
  335             }
  336             while (start <= last) {
  337                 if (grpmenu.max >= max_base)
  338                     expand_base();
  339                 base[grpmenu.max++] = start++;
  340             }
  341         }
  342 #endif /* NNTP_ABLE */
  343     /*
  344      * Reading off local spool, read the directory files
  345      */
  346     } else {
  347         DIR *d;
  348         DIR_BUF *e;
  349         char group_path[PATH_LEN];
  350         t_artnum art;
  351 
  352         make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
  353 
  354         if ((d = opendir(group_path)) != NULL) {
  355             while ((e = readdir(d)) != NULL) {
  356                 art = atoartnum(e->d_name);
  357                 if (art >= 1) {
  358                     total++;
  359                     if (grpmenu.max >= max_base)
  360                         expand_base();
  361                     base[grpmenu.max++] = art;
  362                 }
  363             }
  364             CLOSEDIR(d);
  365             tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
  366         } else {
  367             perror_message(_(txt_cannot_open), group_path);
  368 #if 0
  369             if (access(group_path, R_OK) != 0)
  370                 error_message(2, _(txt_not_exist));
  371 #endif /* 0 */
  372             return -1;
  373         }
  374     }
  375 
  376     if (grpmenu.max) {
  377         if (base[grpmenu.max - 1] > group->xmax)
  378             group->xmax = base[grpmenu.max - 1];
  379         expand_bitmap(group, base[0]);
  380     }
  381 
  382     return total;
  383 }
  384 
  385 
  386 /*
  387  * Main group indexing routine.
  388  *
  389  * Will read any existing index, create or incrementally update
  390  * the index by looking at the articles in the spool directory,
  391  * and attempt to write a new index if necessary.
  392  *
  393  * Returns FALSE if the user aborted the indexing, otherwise TRUE
  394  */
  395 t_bool
  396 index_group(
  397     struct t_group *group)
  398 {
  399     int i;
  400     int changed;                /* Count of articles whose overview has changed */
  401     int getart_limit;
  402     int respnum;
  403     int total;
  404     t_artnum last_read_article;
  405     t_artnum min, new_min, max;
  406     t_bool caching_xover;
  407     t_bool filtered;
  408     t_bool path_in_nov = FALSE;
  409     t_bool rebuild_cache = FALSE;
  410 
  411     if (group == NULL)
  412         return TRUE;
  413 
  414     if (!batch_mode)
  415         show_art_msg(group->name);
  416     else {
  417         if (verbose) /* -> lang.c */
  418             wait_message(0, _("Reading %s\n"), group->name);
  419     }
  420 
  421     signal_context = cArt;          /* Set this only once curr_group is valid */
  422 
  423     hash_reclaim();
  424     free_art_array();
  425     free_msgids();
  426 
  427     BegStopWatch("setup_hard_base()");
  428 
  429     /*
  430      * Get list of valid article numbers
  431      */
  432     if (setup_hard_base(group) < 0)
  433         return FALSE;
  434 
  435     EndStopWatch();
  436     PrintStopWatch();
  437 
  438 #ifdef DEBUG
  439     if (debug & DEBUG_NEWSRC) {
  440         debug_print_comment("Before read_overview");
  441         debug_print_bitmap(group, NULL);
  442     }
  443 #endif /* DEBUG */
  444 
  445     min = grpmenu.max ? base[0] : group->xmin;
  446     max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
  447 
  448     getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  449 
  450     if (getart_limit > 0) {
  451         if (grpmenu.max && (grpmenu.max > getart_limit))
  452             min = base[grpmenu.max - getart_limit];
  453         else
  454             getart_limit = 0;
  455     } else if (getart_limit < 0) {
  456         t_artnum first_unread = find_first_unread(group);
  457 
  458         if (min - first_unread < getart_limit)
  459             min = first_unread + getart_limit;
  460         else
  461             getart_limit = 0;
  462     }
  463 
  464     /*
  465      * Quit now if no articles
  466      */
  467     if (max < 0)
  468         return FALSE;
  469 
  470     top_art = 0;
  471     last_read_article = T_ARTNUM_CONST(0);
  472 
  473     /*
  474      * Read in the existing overview data for min..max
  475      * This read has local=TRUE set if locally caching XOVER records to ensure
  476      * we pull in any private overview caches in preference to using OVER
  477      *
  478      * When reading local spool, this will pull in the system wide overview
  479      * cache (if found) otherwise the private overview cache will be read
  480      */
  481     caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
  482     if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
  483         return FALSE;   /* user aborted indexing */
  484 
  485     /*
  486      * Fill in the range last_read_article...max using XOVER
  487      * Only do this if the previous read_overview() was against private cache
  488      */
  489     if ((last_read_article < max) && caching_xover) {
  490         new_min = (last_read_article >= min) ? last_read_article + 1 : min;
  491 
  492         if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
  493             return FALSE;   /* user aborted indexing */
  494         else
  495             changed += i;
  496     } else
  497         caching_xover = FALSE;
  498 
  499     /*
  500      * Mark as UNTHREADED all articles that have been verified as valid
  501      * Get num of new arts to index so the user will have an idea of index time
  502      */
  503     for (i = 0, total = 0; i < grpmenu.max; i++) {
  504         if ((respnum = valid_artnum(base[i])) >= 0) {
  505             arts[respnum].thread = ART_UNTHREADED;
  506             continue;
  507         }
  508         if (base[i] <= last_read_article)       /* It is vital this test be done last */
  509             continue;
  510         total++;
  511     }
  512 
  513     /*
  514      * Add any articles to arts[] that are new or were killed
  515      */
  516     if (total > 0) {
  517         new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
  518 
  519         if ((i = read_art_headers(group, total, new_min)) == -1)
  520             return FALSE;       /* user aborted indexing */
  521         else
  522             changed += i;
  523     }
  524 
  525 #ifdef DEBUG
  526     if (debug & DEBUG_NEWSRC) {
  527         debug_print_comment("Before parse_unread_arts()");
  528         debug_print_bitmap(group, NULL);
  529     }
  530 #endif /* DEBUG */
  531     /*
  532      * Do this before calling art_mark(,, ART_READ) if you want
  533      * the unread count to be correct.
  534      */
  535     min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
  536     parse_unread_arts(group, min);
  537 #ifdef DEBUG
  538     if (debug & DEBUG_NEWSRC) {
  539         debug_print_comment("After parse_unread_arts()");
  540         debug_print_bitmap(group, NULL);
  541     }
  542 #endif /* DEBUG */
  543 
  544     /*
  545      * Stat all articles to see if any have expired
  546      */
  547     for_each_art(i) {
  548         if (arts[i].thread == ART_EXPIRED) {
  549             changed++;
  550 #ifdef DEBUG
  551             if (debug & DEBUG_NEWSRC)
  552                 debug_print_comment("art.c: index_group() purging...");
  553 #endif /* DEBUG */
  554             art_mark(group, &arts[i], ART_READ);
  555             if (group->attribute->show_only_unread_arts)
  556                 arts[i].keep_in_base = FALSE;
  557         }
  558         if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
  559             path_in_nov = TRUE;
  560     }
  561 
  562     /*
  563      * Only rewrite the index if it has changed
  564      * TODO review the exact logic behind "|| caching_xover"
  565      */
  566     if (changed || caching_xover || rebuild_cache)
  567         write_overview(group);
  568 
  569     /*
  570      * Create the reference tree. The msgid and ref ptrs will
  571      * be free()d now that the NovFile has been written.
  572      */
  573     build_references(group);
  574 
  575     /*
  576      * Needs access to the reference tree
  577      */
  578     filtered = filter_articles(group);
  579 
  580     BegStopWatch("make_threads()");
  581 
  582     /*
  583      * Thread the group
  584      */
  585     make_threads(group, FALSE);
  586 
  587     EndStopWatch();
  588     PrintStopWatch();
  589 
  590     if ((changed > 0 || filtered) && !batch_mode)
  591         clear_message();
  592 
  593     return TRUE;
  594 }
  595 
  596 
  597 /*
  598  * Returns number of first unread article
  599  */
  600 static t_artnum
  601 find_first_unread(
  602     struct t_group *group)
  603 {
  604     unsigned char *p;
  605     unsigned char *end = group->newsrc.xbitmap;
  606     t_artnum first = group->newsrc.xmin; /* initial value */
  607 
  608     if ((p = group->newsrc.xbitmap)) {
  609         end += group->newsrc.xbitlen / NBITS;
  610         for (; p < end && *p == '\0'; p++, first += NBITS)
  611             ;
  612     }
  613     return first;
  614 }
  615 
  616 
  617 /*
  618  * Open an article for reading just the header
  619  * 'NEXT' is used/updated with the next article number
  620  * to optimise the number of 'HEAD' commands issued on
  621  * groups with holes.
  622  */
  623 static FILE *
  624 open_art_header(
  625     char *groupname,
  626     t_artnum art,
  627     t_artnum *next)
  628 {
  629     char buf[NNTP_STRLEN];
  630 #ifdef NNTP_ABLE
  631     FILE *fp;
  632     int i;
  633 
  634     if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
  635         static t_bool no_next = FALSE; /* TODO: move to t_capabilities ? */
  636         /*
  637          * Don't bother requesting if we have not got there yet.
  638          * This is a big win if the group has got holes in it (ie. if 000's
  639          * of articles have expired between active files min & max values).
  640          */
  641         if (art < *next)
  642             return NULL;
  643 
  644         snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  645         if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  646             return fp;
  647 
  648         /*
  649          * TODO:
  650          * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
  651          * "503 Access denied" instead of 480 but NEXT still works,
  652          * so tin loops over all articles without getting useful data
  653          *
  654          * usenet.farm may return ERR_GOODBYE and NEXT then also fails
  655          * we could flag that like no_next ...
  656          */
  657 
  658         if (!no_next) { /* usenet.farm doesn't do NEXT */
  659             /*
  660              * HEAD failed, try to find NEXT
  661              * Should return "223 artno message-id more text...."
  662              */
  663             i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
  664             switch (i) {
  665                 case OK_NOTEXT:
  666                     *next = atoartnum(buf);     /* Set next art number */
  667                     break;
  668 
  669 #   ifndef BROKEN_LISTGROUP
  670                 /*
  671                  * might happen if LISTGROUP doesn't select group, but
  672                  * we are not -DBROKEN_LISTGROUP
  673                  */
  674                 case ERR_NCING:
  675                     nntp_caps.broken_listgroup = TRUE;
  676                     snprintf(buf, sizeof(buf), "GROUP %s", groupname);
  677                     if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
  678                         return NULL;
  679                     snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  680                     if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  681                         return fp;
  682                     if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
  683                         *next = atoartnum(buf);
  684                     break;
  685 #   endif /* !BROKEN_LISTGROUP */
  686 
  687                 case ERR_COMMAND:   /* TODO: abort loop over all arts */
  688                     no_next = TRUE;
  689 #   ifdef DEBUG
  690                     if ((debug & DEBUG_NNTP) && verbose > 1)
  691                         debug_print_file("NNTP", "!!! NEXT disabled after %d response", ERR_COMMAND);
  692 #   endif /* DEBUG */
  693                     break;
  694 
  695                 default:
  696                     /*
  697                      * TODO: abort loop over all arts on ERR_NONEXT
  698                      */
  699 #   ifndef BROKEN_LISTGROUP
  700                     /*
  701                      * to avoid out of sync responses
  702                      * (listgroup seems to work, but didn't select new group,
  703                      *  so xover seems to work but returns old data)
  704                      * we set listgroup_broken = TRUE; once we saw a
  705                      * ERR_NOARTIG / ERR_NONEXT or the like - even if
  706                      * ERR_NOARTIG may occur on servers where listgroup
  707                      * isn't broken...
  708                      */
  709                     nntp_caps.broken_listgroup = TRUE;
  710 #   endif /* !BROKEN_LISTGROUP */
  711                     break;
  712             }
  713         }
  714         return NULL;
  715     }
  716 #endif /* NNTP_ABLE */
  717 
  718     snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
  719     return (fopen(buf, "r"));
  720 }
  721 
  722 
  723 /*
  724  * Called after XOVER/local/private overview databases have been loaded
  725  * Read and parse in headers for any arts not already found (usually
  726  * new articles that have not been indexed yet)
  727  * Any new articles that are added have ->thread set to ART_UNTHREADED
  728  * 'top' is the current highest artnum read
  729  *
  730  * Return values are:
  731  *   >0   Number of additional articles read in
  732  *    0   No additional (new) articles were found
  733  *   -1   user aborted during read
  734  */
  735 static int
  736 read_art_headers(
  737     struct t_group *group,
  738     int total,
  739     t_artnum top)
  740 {
  741     FILE *fp;
  742     char dir[PATH_LEN];
  743     char group_msg[LEN];
  744     int i;
  745     int modified = 0;
  746     t_artnum art;
  747     t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
  748     t_bool res;
  749 
  750     /*
  751      * Change to groups spooldir to optimize fopen()'s on local articles
  752      * NB open_art_header() requires this
  753      */
  754     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
  755         char buf[PATH_LEN];
  756 
  757         get_cwd(dir);
  758         make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
  759         if (chdir(buf) != 0) {
  760 #ifdef DEBUG
  761             if (debug & DEBUG_MISC)
  762                 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
  763 #endif /* DEBUG */
  764             return -1;
  765         }
  766     }
  767 
  768     snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
  769 
  770     /* TODO: add progress meter? HEAD/NEXT is slow */
  771     for (i = 0; i < grpmenu.max; i++) { /* for each article number */
  772         art = base[i];
  773 
  774         /*
  775          * Skip articles that are below the low water mark or are
  776          * already present
  777          */
  778         if (valid_artnum(art) >= 0)
  779             continue;
  780         if (art <= top)
  781             continue;
  782 
  783         /*
  784          * Try and open the article
  785          */
  786         if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
  787             continue;
  788 
  789         /*
  790          * Add article to arts[]
  791          */
  792         if (top_art >= max_art)
  793             expand_art();
  794 
  795         set_article(&arts[top_art]);
  796         arts[top_art].artnum = art;
  797         arts[top_art].thread = ART_UNTHREADED;
  798 
  799         res = parse_headers(fp, &arts[top_art]);
  800 
  801         TIN_FCLOSE(fp);
  802         if (tin_errno) {
  803             modified = -1;
  804             break;
  805         }
  806 
  807         if (!res) {
  808 #ifdef DEBUG
  809             if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
  810                 char buf[PATH_LEN];
  811 
  812                 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
  813                 debug_print_file("ARTS", "read_art_headers() %s", buf);
  814             }
  815 #endif /* DEBUG */
  816             arts[top_art].artnum = T_ARTNUM_CONST(0);
  817             arts[top_art].date = (time_t) 0;
  818             FreeAndNull(arts[top_art].xref);
  819             FreeAndNull(arts[top_art].path);
  820             FreeAndNull(arts[top_art].refs);
  821             FreeAndNull(arts[top_art].msgid);
  822             arts[top_art].tagged = 0;
  823             arts[top_art].thread = ART_EXPIRED;
  824             arts[top_art].prev = ART_NORMAL;
  825             arts[top_art].status = ART_UNREAD;
  826             arts[top_art].killed = ART_NOTKILLED;
  827             arts[top_art].selected = FALSE;
  828             continue;
  829         }
  830 
  831         top = arts[top_art].artnum; /* used if arts are killed */
  832         top_art++;
  833 
  834         if (++modified % (MODULO_COUNT_NUM * 20) == 0)
  835             show_progress(group_msg, modified, total);
  836     }
  837 
  838     /*
  839      * Change back to previous dir before indexing started
  840      */
  841     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
  842         if (chdir(dir) == -1) {
  843 #ifdef DEBUG
  844             int e = errno;
  845             if (debug & DEBUG_MISC)
  846                 error_message(2, "chdir(%s): Error: %s", dir, strerror(e));
  847 #endif /* DEBUG */
  848         }
  849     }
  850 
  851     return modified;
  852 }
  853 
  854 
  855 /*
  856  * The algorithm is elegant, using the fact that identical Subject lines
  857  * are hashed to the same node in table[] (see hashstr.c)
  858  *
  859  * Mark i as being in j's thread list if
  860  * . The article is _not_ being ignored
  861  * . The article is not already threaded
  862  * . The subject lines are the same
  863  */
  864 static void
  865 thread_by_subject(
  866     void)
  867 {
  868     int i, j;
  869     struct t_hashnode *h;
  870 
  871     for_each_art(i) {
  872         if (IGNORE_ART_THREAD(i))
  873             continue;
  874 
  875         /*
  876          * Get the contents of the magic marker in the hashnode
  877          */
  878         h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
  879 
  880         j = h->aptr;
  881 
  882         if (j != -1 && j < i) {
  883             if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject)) {
  884                 arts[j].thread = i;
  885                 arts[i].prev = j;
  886             }
  887         }
  888 
  889         /*
  890          * Update the magic marker with the highest numbered mesg in
  891          * arts[] that has been used in this thread so far
  892          */
  893         h->aptr = i;
  894     }
  895 
  896 #if 0
  897     fprintf(stderr, "Subj dump\n");
  898     fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
  899     for_each_art(i) {
  900         fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
  901             (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
  902             (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
  903             (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
  904             arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
  905     }
  906 #endif /* 0 */
  907 }
  908 
  909 
  910 /*
  911  * This Threading algorithm threads articles into 'buckets' where each bucket
  912  * contains all the articles which match the root article's subject line to
  913  * the configured percentage. Eg, if the root article had the subject "asdf"
  914  * and the match percentage was configured to be 75% then any article would
  915  * match if its subject was no different in more than a single character.
  916  */
  917 static void
  918 thread_by_percentage(
  919     unsigned int percentage)
  920 {
  921     int i, j, k;
  922     int root_num = 0; /* The index number of the root we are currently working on. */
  923     unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
  924     size_t slen;
  925 
  926     /* First we need to sort art[] to simplify and speed up the matching. */
  927     SortBy(subj_comp_asc);
  928 
  929     /*
  930      * Now we put all the articles which match enough into the thread. If
  931      * an article doesn't match enough we create a new thread and then add
  932      * to that and so on.
  933      */
  934     base[0] = 0;
  935     arts[0].prev = ART_NORMAL;
  936     for_each_art(i) {
  937         if (i == 0)
  938             continue;
  939 
  940         /* Check each character to see if it matched enough */
  941         k = 0;
  942         unmatched = 0;
  943         for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
  944             if (arts[base[root_num]].subject[j] != arts[i].subject[k])
  945                 unmatched++;
  946         }
  947 
  948         /*
  949          * By getting here we have a number of unmatched characters
  950          * between the two strings. We also have the length of the
  951          * strings available to us easily.
  952          * All we need to do is see if the match is good enough, but
  953          * we count differences in the length of the strings against
  954          * them matching.
  955          */
  956         if (!(slen = strlen(arts[base[root_num]].subject)))
  957             slen++;
  958         unmatched += (unsigned) (slen - strlen(arts[i].subject));
  959         if (unmatched * 100 / slen > percentage) {
  960             /*
  961              * If there is less greater than percentage% different start a
  962              * new thread.
  963              */
  964             base[++root_num] = i;
  965             arts[i].prev = ART_NORMAL;
  966             continue;
  967         } else {
  968             /*
  969              * The subject lines match enough to consider them part of a single
  970              * thread, so add the current article to the thread.
  971              */
  972             if (arts[base[root_num]].thread < 0)
  973                 arts[base[root_num]].thread = i;
  974             arts[i].prev = i - 1;
  975             arts[i - 1].thread = i;
  976             continue;
  977         }
  978     }
  979 }
  980 
  981 
  982 /*
  983  * Parses a subject header of the type "multipart message subject (01/42)"
  984  * into a MultiPartInfo struct, or fails if the message subject isn't in the
  985  * right form.
  986  *
  987  * @return nonzero on success
  988  */
  989 int
  990 global_get_multipart_info(
  991     int aindex,
  992     MultiPartInfo *setme)
  993 {
  994     int i, j, offi, offj;
  995     MultiPartInfo setmei, setmej;
  996 
  997     i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
  998     j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
  999 
 1000     /* Ok i hits first */
 1001     if (offi > offj) {
 1002         *setme = setmei;
 1003         return i;
 1004     }
 1005 
 1006     /* Its j or they are both the same (which must be zero!) so we don't care */
 1007     *setme = setmej;
 1008     return j;
 1009 }
 1010 
 1011 
 1012 static int
 1013 global_look_for_multipart_info(
 1014     int aindex,
 1015     MultiPartInfo* setme,
 1016     char start,
 1017     char stop,
 1018     int *offset)
 1019 {
 1020     char *subj;
 1021     char *pch;
 1022     MultiPartInfo tmp;
 1023 
 1024     *offset = 0;
 1025 
 1026     /* entry assertions */
 1027     assert(((void) "invalid index", 0 <= aindex && aindex < top_art));
 1028     assert(((void) "setme must not be NULL", setme != NULL));
 1029 
 1030     /* parse the message */
 1031     subj = arts[aindex].subject;
 1032     pch = strrchr(subj, start);
 1033     if (!pch || !isdigit((int) pch[1]))
 1034         return 0;
 1035 
 1036     tmp.arts_index = aindex;
 1037     tmp.subject_compare_len = (int) (pch - subj);
 1038     tmp.part_number = (int) strtol(pch + 1, &pch, 10);
 1039     if (*pch != '/' && *pch != '|')
 1040         return 0;
 1041 
 1042     if (!isdigit((int) pch[1]))
 1043         return 0;
 1044 
 1045     tmp.total = (int) strtol(pch + 1, &pch, 10);
 1046     if (*pch != stop)
 1047         return 0;
 1048 
 1049     /*
 1050      * skip "blah (00/102)" or "blah (103/102)" subjects
 1051      */
 1052     if (tmp.part_number < 1 || tmp.part_number > tmp.total)
 1053         return 0;
 1054 
 1055     tmp.subject = subj;
 1056     *setme = tmp;
 1057     *offset = (int) (pch - subj);
 1058     return 1;
 1059 }
 1060 
 1061 
 1062 t_bool
 1063 global_look_for_multipart(
 1064     int aindex,
 1065     char start,
 1066     char stop)
 1067 {
 1068     char *pch;
 1069 
 1070     pch = strrchr(arts[aindex].subject, start);
 1071     if (!pch || !isdigit((int) pch[1]))
 1072         return FALSE;
 1073 
 1074     strtol(pch + 1, &pch, 10);
 1075     if (*pch != '/' && *pch != '|')
 1076         return FALSE;
 1077 
 1078     if (!isdigit((int) pch[1]))
 1079         return FALSE;
 1080 
 1081     strtol(pch + 1, &pch, 10);
 1082     if (*pch != stop)
 1083         return FALSE;
 1084 
 1085     arts[aindex].multipart_subj = TRUE;
 1086     return TRUE;
 1087 }
 1088 
 1089 
 1090 /*
 1091  * Tries to find all the parts to the multipart message pointed to by
 1092  * aindex.
 1093  *
 1094  * @return on success, the number of parts found. On failure, zero if not
 1095  * a multipart or the negative value of the first missing part in case of
 1096  * tagging.
 1097  * @param aindex index pointing to one of the messages in a multipart
 1098  * message.
 1099  * @param malloc_and_setme_info on success, set to a malloced array the
 1100  * parts found.
 1101  */
 1102 int
 1103 global_get_multiparts(
 1104     int aindex,
 1105     MultiPartInfo **malloc_and_setme_info,
 1106     t_bool tagging)
 1107 {
 1108     int i, part_index, part_cnt = 0;
 1109     MultiPartInfo tmp, tmp2;
 1110     MultiPartInfo *info;
 1111 
 1112     /* entry assertions */
 1113     assert(((void) "Invalid index", 0 <= aindex && aindex < top_art));
 1114     assert(((void) "malloc_and_setme_info must not be NULL", malloc_and_setme_info != NULL));
 1115 
 1116     /* make sure this is a multipart message... */
 1117     if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
 1118         return 0;
 1119 
 1120     /* make a temporary buffer to hold the multipart info... */
 1121     info = my_malloc(sizeof(MultiPartInfo) * (size_t) tmp.total);
 1122 
 1123     /* zero out part-number for the repost check below */
 1124     for (i = 0; i < tmp.total; ++i) {
 1125         info[i].total = tmp.total; /* Added this for thread_by_multipart */
 1126         info[i].part_number = -1;
 1127     }
 1128 
 1129     /* try to find all the multiparts... */
 1130     for (i = (tagging ? 0 : aindex); i < top_art; i++) {
 1131         if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, (size_t) tmp.subject_compare_len))
 1132             continue;
 1133 
 1134         if (!global_get_multipart_info(i, &tmp2))
 1135             continue;
 1136 
 1137         /* 'test (1/5)' is not the same as 'test (1/15)' */
 1138         if (tmp.total != tmp2.total)
 1139             continue;
 1140 
 1141         part_index = tmp2.part_number - 1;
 1142 
 1143         /* repost check: do we already have this part? */
 1144         if (info[part_index].part_number != -1) {
 1145             assert(((void) "bookkeeping error", info[part_index].part_number == tmp2.part_number));
 1146             continue;
 1147         }
 1148 
 1149         /* we have a match, hooray! */
 1150         info[part_index] = tmp2;
 1151 
 1152         arts[i].multipart_subj = FALSE;
 1153 
 1154         /* all parts found? */
 1155         if (++part_cnt == tmp.total)
 1156             break;
 1157     }
 1158 
 1159     /* see if we got them all. */
 1160     if (tagging) {
 1161         for (i = 0; i < tmp.total; ++i) {
 1162             if (info[i].part_number != i + 1) {
 1163                 free(info);
 1164                 return -(i + 1); /* missing part #(i+1) */
 1165             }
 1166         }
 1167     }
 1168 
 1169     /* looks like a success .. */
 1170     *malloc_and_setme_info = info;
 1171     return tmp.total;
 1172 }
 1173 
 1174 
 1175 /*
 1176  *  The algorithm uses the tag multipart searches to thread articles together.
 1177  */
 1178 static void
 1179 thread_by_multipart(
 1180     void)
 1181 {
 1182     int i, j, threadNum;
 1183     MultiPartInfo *minfo = NULL;
 1184 
 1185     for_each_art(i) {
 1186         if (!global_look_for_multipart(i, '[', ']'))
 1187             global_look_for_multipart(i, '(', ')');
 1188     }
 1189 
 1190     for_each_art(i) {
 1191         if (!arts[i].multipart_subj)
 1192             continue;
 1193 
 1194         if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
 1195             FreeAndNull(minfo);
 1196             arts[i].multipart_subj = FALSE;
 1197             continue;
 1198         }
 1199 
 1200         threadNum = -1;
 1201         for (j = minfo[0].total - 1; j >= 0; j--) {
 1202             if (minfo[j].part_number != -1) {
 1203                 if (threadNum != -1) {
 1204                     arts[minfo[j].arts_index].thread = threadNum;
 1205                     arts[threadNum].prev = minfo[j].arts_index;
 1206                 }
 1207                 threadNum = minfo[j].arts_index;
 1208             }
 1209         }
 1210         FreeAndNull(minfo);
 1211         arts[i].multipart_subj = FALSE;
 1212         if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
 1213             show_progress(_("Threading by multipart"), i, top_art);
 1214     }
 1215 }
 1216 
 1217 
 1218 /*
 1219  * Go through the articles in arts[] and create threads. There are
 1220  * 5 strategies currently defined :
 1221  *
 1222  *  THREAD_NONE     No threading
 1223  *  THREAD_SUBJ     Threads are created using like Subject lines
 1224  *  THREAD_REFS     Threads are created using the References headers
 1225  *  THREAD_BOTH     Threads created using References and then Subject
 1226  *  THREAD_MULTI    Threads created using Subject to search for Multiparts
 1227  *  THREAD_PERC     Threads based upon a char for char match of greater than x%
 1228  *
 1229  * .thread and .prev are used to hold the threading information, see tin.h for
 1230  * more information
 1231  * Only process valid (unexpired) articles we haven't visited yet
 1232  * (ie arts[].thread == ART_UNTHREADED)
 1233  *
 1234  * The rethread parameter is used to force the deletion of existing threading
 1235  * information before threading which happens anyway expect when using
 1236  * THREAD_NONE (I don't immediately see how this is useful)
 1237  */
 1238 /* TODO: rewrite that user can easily combine different 'threading'
 1239  *       methods, i.e:
 1240  *       - thread_by_multipart() + collate_subjects()
 1241  */
 1242 void
 1243 make_threads(
 1244     struct t_group *group,
 1245     t_bool rethread)
 1246 {
 1247     if (!cmd_line && !batch_mode) {
 1248         info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
 1249         my_flush();
 1250     }
 1251 
 1252 #ifdef DEBUG
 1253     if (debug & DEBUG_MISC)
 1254         error_message(2, "rethread=[%d]  thread_articles=[%d]  attr_thread_articles=[%d]",
 1255             rethread, tinrc.thread_articles, group->attribute->thread_articles);
 1256 #endif /* DEBUG */
 1257 
 1258     /*
 1259      * Sort all the articles using the preferred method
 1260      * When find_base() is called, the bases are created ordered
 1261      * on arts[] and so the base messages under all threading systems
 1262      * will be sorted in this way.
 1263      */
 1264     sort_arts(group->attribute->sort_article_type);
 1265 
 1266     /*
 1267      * Reset all the ptrs to articles following the above sort
 1268      */
 1269     clear_art_ptrs();
 1270 
 1271     /*
 1272      * The threading pointers need to be reset if re-threading
 1273      * If using ref threading, revector the links back to the articles
 1274      */
 1275     if (rethread || group->attribute->thread_articles) {
 1276         int i;
 1277 
 1278         for_each_art(i) {
 1279             if (arts[i].thread >= 0)
 1280                 arts[i].thread = ART_UNTHREADED;
 1281 
 1282             arts[i].prev = ART_NORMAL;
 1283 
 1284             /* Should never happen if tree is built properly */
 1285             if (arts[i].refptr == NULL) {
 1286 #ifdef DEBUG
 1287                 if (debug & DEBUG_REFS) {
 1288                     my_fprintf(stderr, "\nError  : art->refptr is NULL\n");
 1289                     my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
 1290                     my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
 1291                     my_fprintf(stderr, "From   : %s\n", arts[i].from);
 1292                     assert(arts[i].refptr != NULL);
 1293                 } else
 1294 #endif /* DEBUG */
 1295                     continue;
 1296             }
 1297             arts[i].refptr->article = i;
 1298         }
 1299     }
 1300 
 1301     /*
 1302      * Do the right thing according to the threading strategy
 1303      */
 1304     switch (group->attribute->thread_articles) {
 1305         case THREAD_NONE:
 1306             break;
 1307 
 1308         case THREAD_SUBJ:
 1309             thread_by_subject();
 1310             break;
 1311 
 1312         case THREAD_REFS:
 1313             thread_by_reference();
 1314             break;
 1315 
 1316         case THREAD_BOTH:
 1317             thread_by_reference();
 1318             collate_subjects();
 1319             break;
 1320 
 1321         case THREAD_MULTI:
 1322             thread_by_multipart();
 1323             break;
 1324 
 1325         case THREAD_PERC:
 1326             thread_by_percentage((unsigned) (100 - group->attribute->thread_perc));
 1327             break;
 1328 
 1329         default: /* not reached */
 1330             break;
 1331     }
 1332 
 1333     /*
 1334      * Rebuild base[]
 1335      */
 1336     find_base(group);
 1337 }
 1338 
 1339 
 1340 static t_compfunc
 1341 eval_sort_arts_func(
 1342     unsigned int sort_art_type)
 1343 {
 1344     switch (sort_art_type) {
 1345         case SORT_ARTICLES_BY_NOTHING:      /* don't sort at all */
 1346             return artnum_comp;
 1347 
 1348         case SORT_ARTICLES_BY_SUBJ_DESCEND:
 1349             return subj_comp_desc;
 1350 
 1351         case SORT_ARTICLES_BY_SUBJ_ASCEND:
 1352             return subj_comp_asc;
 1353 
 1354         case SORT_ARTICLES_BY_FROM_DESCEND:
 1355             return from_comp_desc;
 1356 
 1357         case SORT_ARTICLES_BY_FROM_ASCEND:
 1358             return from_comp_asc;
 1359 
 1360         case SORT_ARTICLES_BY_DATE_DESCEND:
 1361             return date_comp_desc;
 1362 
 1363         case SORT_ARTICLES_BY_DATE_ASCEND:
 1364             return date_comp_asc;
 1365 
 1366         case SORT_ARTICLES_BY_SCORE_DESCEND:
 1367             return score_comp_desc;
 1368 
 1369         case SORT_ARTICLES_BY_SCORE_ASCEND:
 1370             return score_comp_asc;
 1371 
 1372         case SORT_ARTICLES_BY_LINES_DESCEND:
 1373             return lines_comp_desc;
 1374 
 1375         case SORT_ARTICLES_BY_LINES_ASCEND:
 1376             return lines_comp_asc;
 1377 
 1378         default:
 1379             break;
 1380     }
 1381     return NULL;
 1382 }
 1383 
 1384 
 1385 void
 1386 sort_arts(
 1387     unsigned int sort_art_type)
 1388 {
 1389     t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
 1390 
 1391     if (comp_func)
 1392         SortBy(comp_func);
 1393 }
 1394 
 1395 
 1396 static void
 1397 sort_base(
 1398     unsigned int sort_threads_type)
 1399 {
 1400     switch (sort_threads_type) {
 1401         case SORT_THREADS_BY_SCORE_DESCEND:
 1402         case SORT_THREADS_BY_SCORE_ASCEND:
 1403             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
 1404             break;
 1405 
 1406         case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
 1407             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
 1408             break;
 1409 
 1410         case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
 1411             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
 1412             break;
 1413     }
 1414 }
 1415 
 1416 
 1417 /*
 1418  * This is called to get header info for articles not already found in the
 1419  * overview files.
 1420  */
 1421 static t_bool
 1422 parse_headers(
 1423     FILE *fp,
 1424     struct t_article *h)
 1425 {
 1426     char art_from_addr[HEADER_LEN];
 1427     char art_full_name[HEADER_LEN];
 1428     char *s, *hdr, *ptr;
 1429     t_bool got_from, got_lines;
 1430 
 1431     got_from = got_lines = FALSE;
 1432 
 1433     while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
 1434         /*
 1435          * Look for the end of information which tin wants to get.
 1436          * Applies when reading local spool and via NNTP.
 1437          */
 1438 
 1439         /*
 1440          * End of headers ?
 1441          */
 1442         if (ptr[0] == '\0')
 1443             break;
 1444 
 1445         unfold_header(ptr);
 1446         switch (my_toupper((unsigned char) *ptr)) {
 1447             case 'D':   /* Date:  mandatory */
 1448                 if (!h->date) {
 1449                     if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
 1450                         h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
 1451                 }
 1452                 break;
 1453 
 1454             case 'F':   /* From:  mandatory */
 1455                 if (!got_from) {
 1456                     if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
 1457                         h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
 1458                         h->from = hash_str(buffer_to_ascii(art_from_addr));
 1459                         if (*art_full_name)
 1460                             h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 1461                         got_from = TRUE;
 1462                     }
 1463                 }
 1464                 break;
 1465 
 1466             case 'L':   /* Lines:  optional */
 1467                 if (!got_lines) {
 1468                     if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
 1469                         h->line_count = atoi(hdr);
 1470                         got_lines = TRUE;
 1471                     }
 1472                 }
 1473                 break;
 1474 
 1475             case 'M':   /* Message-ID:  mandatory */
 1476                 if (!h->msgid) {
 1477                     if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
 1478                         h->msgid = my_strdup(hdr);
 1479                 }
 1480                 break;
 1481 
 1482             /* for Path:-filter when reading from local spool */
 1483             case 'P':   /* Path: */
 1484                 if (!h->path) {
 1485                     if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
 1486                         h->path = my_strdup(hdr);
 1487                 }
 1488                 break;
 1489 
 1490             case 'R':   /* References:  optional */
 1491                 if (!h->refs) {
 1492                     if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
 1493                         h->refs = my_strdup(hdr);
 1494                 }
 1495                 break;
 1496 
 1497             case 'S':   /* Subject:  mandatory */
 1498                 if (!h->subject) {
 1499                     if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
 1500 #ifdef HAVE_UNICODE_NORMALIZATION
 1501                         if (IS_LOCAL_CHARSET("UTF-8"))
 1502                             s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1503                         else
 1504 #endif /* HAVE_UNICODE_NORMALIZATION */
 1505                             s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1506 
 1507                         h->subject = hash_str(s);
 1508                         free(s);
 1509                     }
 1510                 }
 1511                 break;
 1512 
 1513             case 'X':   /* Xref:  optional */
 1514                 if (!h->xref) {
 1515                     if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
 1516                         h->xref = my_strdup(hdr);
 1517                 }
 1518                 break;
 1519 
 1520             default:
 1521                 break;
 1522         } /* switch */
 1523 
 1524     } /* while */
 1525 
 1526 #ifdef NNTP_ABLE
 1527     if (ptr)
 1528         drain_buffer(fp);
 1529 #endif /* NNTP_ABLE */
 1530 
 1531     if (tin_errno)
 1532         return FALSE;
 1533 
 1534     /*
 1535      * The son of RFC 1036 states that the following hdrs are mandatory. It
 1536      * also states that Subject, Newsgroups and Path are too. Ho hum.
 1537      *
 1538      * What about reading mail from local spool via ~/.tin/active.mail,
 1539      * they might not have a Message-ID.
 1540      */
 1541     if (got_from && h->date && h->msgid) {
 1542         if (!h->subject)
 1543             h->subject = hash_str("");
 1544 
 1545 #ifdef DEBUG
 1546         if (debug & DEBUG_FILTER)
 1547             debug_print_header(h);
 1548 #endif /* DEBUG */
 1549         return TRUE;
 1550     }
 1551 
 1552     return FALSE;
 1553 }
 1554 
 1555 #ifdef NNTP_ABLE
 1556 /*
 1557  * Loop over arts[] and find ranges without Path: header
 1558  * If there are any try to optimize the ranges regarding traffic consumption
 1559  * Start optimization if at least MIN_CNT ranges exist
 1560  * If there are more than MAX_CNT ranges after optimization, fetch all in one
 1561  * big range
 1562  */
 1563 #define MIN_CNT 10
 1564 #define MAX_CNT 50
 1565 static struct t_article_range *
 1566 build_range_list(
 1567     t_artnum min,
 1568     t_artnum max,
 1569     int *range_cnt)
 1570 {
 1571     int i, gap_cnt = 0;
 1572     struct t_article_range *res = NULL, *gap_list, *curr, *from;
 1573     t_artnum new_end;
 1574 
 1575     new_end = T_ARTNUM_CONST(0);
 1576     gap_list = my_malloc(sizeof(struct t_article_range));
 1577     curr = gap_list;
 1578     curr->start = min;
 1579     curr->end = max;
 1580     curr->cnt = T_ARTNUM_CONST(0);
 1581     curr->next = NULL;
 1582 
 1583     for_each_art(i) {
 1584         if (arts[i].artnum < min)
 1585             continue;
 1586         if (arts[i].artnum > max)
 1587             break;
 1588         if (arts[i].path) {
 1589             for (; i < top_art && arts[i].path; i++)
 1590                 ;
 1591             /*
 1592              * the current art has no path -> we use this one
 1593              * if we reached top_art all arts have path
 1594              * so we use max
 1595              */
 1596             curr->start = i == top_art ? max : arts[i--].artnum;
 1597         } else {
 1598             for (; i < top_art && !arts[i].path; i++)
 1599                 ;
 1600             /* the current art has path -> we use the last one */
 1601             new_end = curr->end = arts[--i].artnum;
 1602         }
 1603         if (new_end) {
 1604             curr->cnt = curr->end - curr->start + 1;
 1605             curr->next = my_malloc(sizeof(struct t_article_range));
 1606             curr = curr->next;
 1607             curr->start = new_end;
 1608             curr->end = max;
 1609             curr->cnt = T_ARTNUM_CONST(0);
 1610             curr->next = NULL;
 1611             new_end = T_ARTNUM_CONST(0);
 1612         }
 1613     }
 1614 
 1615     curr = gap_list;
 1616     while (curr && curr->cnt) {
 1617         ++gap_cnt;
 1618 #   ifdef DEBUG
 1619         if ((debug & DEBUG_NNTP) && verbose > 1)
 1620             debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1621 #   endif /* DEBUG */
 1622         curr = curr->next;
 1623     }
 1624 
 1625     /*
 1626      * Optimize only if there are at least MIN_CNT ranges
 1627      */
 1628     if (gap_cnt >= MIN_CNT) {
 1629         res = my_malloc(sizeof(struct t_article_range));
 1630         res->start = T_ARTNUM_CONST(0);
 1631         res->end = T_ARTNUM_CONST(0);
 1632         res->cnt = T_ARTNUM_CONST(0);
 1633         res->next = NULL;
 1634 
 1635         from = gap_list;
 1636         curr = res;
 1637         while (from) {
 1638             curr->start = from->start;
 1639             curr->end = from->end;
 1640             curr->cnt = from->cnt;
 1641             if ((from = from->next)) {
 1642                 /*
 1643                  * If the next range is grater then the gap between the current
 1644                  * one and the next one we build a new range including the
 1645                  * current one, the next one and the gap between
 1646                  */
 1647                 while (from && from->cnt >= from->start - curr->end - 1) {
 1648                     curr->end = from->end;
 1649                     from = from->next;
 1650                 }
 1651                 curr->cnt = curr->end - curr->start + 1;
 1652                 curr->next = my_malloc(sizeof(struct t_article_range));
 1653                 curr = curr->next;
 1654                 curr->start = T_ARTNUM_CONST(0);
 1655                 curr->end = T_ARTNUM_CONST(0);
 1656                 curr->cnt = T_ARTNUM_CONST(0);
 1657                 curr->next = NULL;
 1658             }
 1659         }
 1660     }
 1661 
 1662     /*
 1663      * If there are less then MIN_CNT ranges
 1664      * no res is build -> return the original list
 1665      */
 1666     if (res) {
 1667         while (gap_list) {
 1668             curr = gap_list;
 1669             gap_list = curr->next;
 1670             free(curr);
 1671         }
 1672     } else
 1673         res = gap_list;
 1674 
 1675     curr = res;
 1676     gap_cnt = 0;
 1677     while (curr && curr->cnt) {
 1678         ++gap_cnt;
 1679 #   ifdef DEBUG
 1680         if ((debug & DEBUG_NNTP) && verbose > 1)
 1681             debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1682 #   endif /* DEBUG */
 1683         curr = curr->next;
 1684     }
 1685 
 1686     if (gap_cnt >= MAX_CNT) {
 1687         curr = res;
 1688         while (curr->next && curr->next->cnt) {
 1689             res->end = curr->next->end;
 1690             curr->next->cnt = 0;
 1691             curr = curr->next;
 1692         }
 1693         res->cnt = res->end - res->start + 1;
 1694         gap_cnt = 1;
 1695 #   ifdef DEBUG
 1696         if ((debug & DEBUG_NNTP) && verbose > 1)
 1697             debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
 1698 #   endif /* DEBUG */
 1699     }
 1700     *range_cnt = gap_cnt;
 1701 
 1702     return res;
 1703 }
 1704 
 1705 
 1706 /*
 1707  * Fetch the Path header in case we want to filter on that in the given group
 1708  *
 1709  * Try [X]HDR first, then XPAT
 1710  */
 1711 static t_bool
 1712 get_path_header(
 1713     int cur,
 1714     int cnt,
 1715     struct t_group *group,
 1716     t_artnum min,
 1717     t_artnum max)
 1718 {
 1719     FILE *fp = NULL;
 1720     char *prep_msg;
 1721     char *buf, *ptr;
 1722     char cmd[NNTP_STRLEN];
 1723     t_artnum artnum, i;
 1724     t_bool found = FALSE;
 1725     static t_bool supported = TRUE; /* assume HDR || XPAT works */
 1726 
 1727     if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
 1728         return FALSE;
 1729 
 1730 #   ifdef DEBUG
 1731     if ((debug & DEBUG_NNTP) && verbose > 1)
 1732         debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
 1733 #   endif /* DEBUG */
 1734 
 1735     if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
 1736         int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
 1737         switch (j) {
 1738             case 215:
 1739                 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 1740 #   ifdef DEBUG
 1741                     if (debug & DEBUG_NNTP)
 1742                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 1743 #   endif /* DEBUG */
 1744                     nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 1745                     strcat(nntp_caps.headers_range, ptr);
 1746                     strcat(nntp_caps.headers_range, "\n");
 1747                 }
 1748                 break;
 1749 
 1750             default:
 1751                 break;
 1752         }
 1753     }
 1754 
 1755     /* does HDR return Path? */
 1756     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
 1757         do {
 1758             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
 1759                 found = TRUE;
 1760         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 1761     }
 1762 
 1763     if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
 1764         if (min == max)
 1765             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
 1766         else
 1767             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
 1768         fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
 1769         if (!nntp_caps.hdr && fp)
 1770             nntp_caps.hdr = TRUE;
 1771     } else if (nntp_caps.xpat) {
 1772         if (min == max)
 1773             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
 1774         else
 1775             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
 1776         fp = nntp_command(cmd, OK_XPAT, NULL, 0);
 1777     }
 1778 
 1779     if (fp) {
 1780         int j = 0;
 1781 
 1782         prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
 1783         while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
 1784 #   ifdef DEBUG
 1785             if ((debug & DEBUG_NNTP) && verbose)
 1786                 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1787 #   endif /* DEBUG */
 1788             if ((ptr = tin_strtok(buf, " ")) == NULL)
 1789                 continue;
 1790             artnum = atoartnum(ptr);
 1791             if ((ptr = tin_strtok(NULL, " ")) == NULL)
 1792                 continue;
 1793             for (i = j; i < top_art; i++) {
 1794                 if (arts[i].artnum == artnum) {
 1795                     FreeIfNeeded(arts[i].path);
 1796                     arts[i].path = my_strdup(ptr);
 1797                     j = (int) i;
 1798                     break;
 1799                 }
 1800             }
 1801             if (++artnum % MODULO_COUNT_NUM == 0)
 1802                 show_progress(prep_msg, artnum - min, max - min);
 1803         }
 1804         free(prep_msg);
 1805         return supported;
 1806     }
 1807 
 1808     /* !fp */
 1809     supported = FALSE;
 1810     if (nntp_caps.xpat)
 1811         nntp_caps.xpat = FALSE;
 1812     /* as nntp_caps.hdr may work with other headers we don't disable it */
 1813 
 1814 #   ifdef DEBUG
 1815     if ((debug & DEBUG_NNTP) && verbose > 1)
 1816         debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
 1817 #   endif /* DEBUG */
 1818     return supported;
 1819 }
 1820 #endif /* NNTP_ABLE */
 1821 
 1822 
 1823 /*
 1824  * Read in an overview index file. Fields are separated by TAB.
 1825  * return the number of expired articles encountered or -1 if the user aborted
 1826  * the read
 1827  * 'top' is set to the highest artnum read
 1828  * If 'local' is set then always open local overview cache in preference to
 1829  * using NNTP XOVER
 1830  *
 1831  * Format (mandatory as far as line count [RFC2980]):
 1832  *  1. article number (ie. 183)                [mandatory]
 1833  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 1834  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 1835  *  4. Date: line     (rfc822 format)          [mandatory]
 1836  *  5. MessageID:     (ie. <123@example.net>)  [mandatory]
 1837  *  6. References:    (ie. <message-id> ....)  [optional]
 1838  *  7. Byte count     (Skipped - not used)     [mandatory]
 1839  *  8. Line count     (ie. 23)                 [mandatory]
 1840  *  9. Xref: line     (ie. alt.test:389)       [optional]
 1841  */
 1842 static int
 1843 read_overview(
 1844     struct t_group *group,
 1845     t_artnum min,
 1846     t_artnum max,
 1847     t_artnum *top,
 1848     t_bool local,
 1849     t_bool *rebuild_cache)
 1850 {
 1851     FILE *fp;
 1852     char *ptr;
 1853     char *q;
 1854     char *buf;
 1855     char *group_msg;
 1856     char art_full_name[HEADER_LEN];
 1857     char art_from_addr[HEADER_LEN];
 1858     unsigned int count;
 1859     int expired = 0;
 1860     t_artnum artnum;
 1861     t_bool path_found = FALSE, path_in_ofmt = FALSE;
 1862     struct t_article *art;
 1863     size_t over_fields = 1;
 1864 
 1865     /*
 1866      * open the overview file (whether it be local or via nntp)
 1867      */
 1868     if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
 1869         return expired;
 1870 
 1871     if (group->xmax > max)
 1872         group->xmax = max;
 1873 
 1874     group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
 1875 
 1876     /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
 1877     if (ofmt) {
 1878         for (; ofmt[over_fields].name; over_fields++) {
 1879             if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
 1880                 path_in_ofmt = TRUE;
 1881         }
 1882     }
 1883 
 1884     if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
 1885         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
 1886         ofmt[0].type = OVER_T_INT;
 1887         ofmt[0].name = my_strdup("Artnum:");
 1888         ofmt[1].type = OVER_T_STRING;
 1889         ofmt[1].name = my_strdup("Subject:");
 1890         ofmt[2].type = OVER_T_STRING;
 1891         ofmt[2].name = my_strdup("From:");
 1892         ofmt[3].type = OVER_T_STRING;
 1893         ofmt[3].name = my_strdup("Date:");
 1894         ofmt[4].type = OVER_T_STRING;
 1895         ofmt[4].name = my_strdup("Message-ID:");
 1896         ofmt[5].type = OVER_T_STRING;
 1897         ofmt[5].name = my_strdup("References:");
 1898         ofmt[6].type = OVER_T_INT;
 1899         ofmt[6].name = my_strdup("Bytes:");
 1900         ofmt[7].type = OVER_T_INT;
 1901         ofmt[7].name = my_strdup("Lines:");
 1902         ofmt[8].type = OVER_T_ERROR;
 1903         ofmt[8].name = NULL;
 1904         over_fields = 7;
 1905     }
 1906 
 1907     while ((buf = tin_fgets(fp, FALSE)) != NULL) {
 1908 #ifdef DEBUG
 1909         if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
 1910             debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1911 #endif /* DEBUG */
 1912 
 1913         if (need_resize) {
 1914             handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
 1915             need_resize = cNo;
 1916         }
 1917 
 1918         /*
 1919          * Read artnum
 1920          */
 1921         if ((ptr = tin_strtok(buf, "\t")) == NULL)
 1922             continue;
 1923 
 1924         /*
 1925          * read the article number, guaranteed to be the first field
 1926          */
 1927         artnum = atoartnum(ptr);
 1928 
 1929         /*
 1930          * artnum field invalid/corrupt or is 1st line of local cached overview
 1931          * (group name)
 1932          */
 1933         if (artnum <= 0)
 1934             continue;
 1935 
 1936         /*
 1937          * skip artnums below the given minimum (getart_limit)
 1938          */
 1939         if (artnum < min)
 1940             continue;
 1941 
 1942         /*
 1943          * Check to make sure article in nov file has not expired in group
 1944          */
 1945         if (artnum < group->xmin) {
 1946             expired++;
 1947             continue;
 1948         }
 1949 
 1950         /*
 1951          * artnum in overview data higher than groups high mark
 1952          *
 1953          * TODO: - warn user about broken overviews?
 1954          *       - try to parse the Xref:-line to get the correct artnum
 1955          *       - see also parse_unread_arts()
 1956          */
 1957         if (artnum > group->xmax)
 1958             continue;
 1959 
 1960         if (top_art >= max_art)
 1961             expand_art();
 1962 
 1963         art = &arts[top_art];
 1964         set_article(art);
 1965         art->artnum = *top = artnum;
 1966 
 1967         /*
 1968          * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
 1969          *       to check for additions like we do with xref_supported
 1970          */
 1971         for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
 1972             /* skip unexpected tailing fields */
 1973             if (count > over_fields) {
 1974 #ifdef DEBUG
 1975                 if ((debug & DEBUG_NNTP) && verbose > 1)
 1976                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
 1977 #endif /* DEBUG */
 1978 
 1979                 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
 1980                 if (count == over_fields + 1) {
 1981                     if (!strncasecmp(ptr, "Xref: ", 6)) {
 1982 #ifdef DEBUG
 1983                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1984                             debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
 1985 #endif /* DEBUG */
 1986                         over_fields++;
 1987                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 1988                         ofmt[over_fields].type = OVER_T_FSTRING;
 1989                         ofmt[over_fields].name = my_strdup("Xref:");
 1990                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 1991                         ofmt[over_fields + 1].name = NULL;
 1992                         xref_supported = TRUE;
 1993                     } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
 1994 #ifdef DEBUG
 1995                         if ((debug & DEBUG_NNTP) && verbose > 1)
 1996                             debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
 1997 #endif /* DEBUG */
 1998                         over_fields++;
 1999                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 2000                         ofmt[over_fields].type = OVER_T_FSTRING;
 2001                         ofmt[over_fields].name = my_strdup("Path:");
 2002                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 2003                         ofmt[over_fields + 1].name = NULL;
 2004                         xref_supported = TRUE;
 2005                     } else
 2006                         continue;
 2007                 } else
 2008                     continue;
 2009             }
 2010 
 2011             /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
 2012             if (expensive_over_parse) { /* strange order */
 2013                 /* mandatory fields */
 2014                 if (ofmt[count].type == OVER_T_STRING) {
 2015                     if (!strcasecmp(ofmt[count].name, "Subject:")) {
 2016                         if (*ptr) {
 2017 #ifdef HAVE_UNICODE_NORMALIZATION
 2018                             if (IS_LOCAL_CHARSET("UTF-8"))
 2019                                 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2020                             else
 2021 #endif /* HAVE_UNICODE_NORMALIZATION */
 2022                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2023 
 2024                             art->subject = hash_str(q);
 2025                             free(q);
 2026                         } else {
 2027                             art->subject = hash_str("");
 2028 #ifdef DEBUG
 2029                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2030                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2031 #endif /* DEBUG */
 2032                         }
 2033                         continue;
 2034                     }
 2035 
 2036                     if (!strcasecmp(ofmt[count].name, "From:")) {
 2037                         if (*ptr) {
 2038                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2039                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2040                             if (*art_full_name)
 2041                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2042                         } else {
 2043                             art->from = hash_str("");
 2044 #ifdef DEBUG
 2045                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2046                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2047 #endif /* DEBUG */
 2048                         }
 2049                         continue;
 2050                     }
 2051 
 2052                     if (!strcasecmp(ofmt[count].name, "Date:")) {
 2053                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2054 #ifdef DEBUG
 2055                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2056                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2057 #endif /* DEBUG */
 2058                         continue;
 2059                     }
 2060 
 2061                     if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
 2062                         if (*ptr) {
 2063                             FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
 2064                             art->msgid = my_strdup(ptr);
 2065                         } else {
 2066                             art->msgid = NULL;
 2067 #ifdef DEBUG
 2068                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2069                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2070 #endif /* DEBUG */
 2071                         }
 2072                         continue;
 2073                     }
 2074 
 2075                     if (!strcasecmp(ofmt[count].name, "References:")) {
 2076                         if (*ptr) {
 2077                             FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
 2078                             art->refs = my_strdup(ptr);
 2079                         } else
 2080                             art->refs = NULL;
 2081                         continue;
 2082                     }
 2083 
 2084                     /*
 2085                      * non std. fields when doing
 2086                      * expensive overview parsing (very
 2087                      * rare, just happens if RFC 3977
 2088                      * 8.4.2 is violated) go here
 2089                      */
 2090                     /* for Path:-filter */
 2091                     if (!strcasecmp(ofmt[count].name, "Path:")) {
 2092                         if (!path_found)
 2093                             path_found = TRUE;
 2094                         if (*ptr) {
 2095                             FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
 2096                             art->path = my_strdup(ptr);
 2097                         } else
 2098                             art->path = NULL;
 2099                         continue;
 2100                     }
 2101                 }
 2102                 /* metadata fields */
 2103                 if (ofmt[count].type == OVER_T_INT) {
 2104                     if (!strcasecmp(ofmt[count].name, "Bytes:")) {
 2105                         if (*ptr) {
 2106 #ifdef DEBUG
 2107                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2108                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2109 #endif /* DEBUG */
 2110                         }
 2111                         continue;
 2112                     }
 2113 
 2114                     if (!strcasecmp(ofmt[count].name, "Lines:")) {
 2115                         if (*ptr) {
 2116                             if (isdigit((unsigned char) *ptr))
 2117                                 art->line_count = atoi(ptr);
 2118                             else {
 2119                                 art->line_count = 0;
 2120 #ifdef DEBUG
 2121                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2122                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2123 #endif /* DEBUG */
 2124                             }
 2125                         } else
 2126                             art->line_count = 0;
 2127                         continue;
 2128                     }
 2129                 }
 2130             } else { /* first 7 fields are in RFC 3977 order */
 2131                 switch (count) {
 2132                     case 1: /* Subject: */
 2133                         if (*ptr) {
 2134 #ifdef HAVE_UNICODE_NORMALIZATION
 2135                             if (IS_LOCAL_CHARSET("UTF-8"))
 2136                                 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2137                             else
 2138 #endif /* HAVE_UNICODE_NORMALIZATION */
 2139                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2140 
 2141                             art->subject = hash_str(q);
 2142                             free(q);
 2143                         } else {
 2144                             art->subject = hash_str("");
 2145 #ifdef DEBUG
 2146                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2147                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2148 #endif /* DEBUG */
 2149                         }
 2150                         break;
 2151 
 2152                     case 2: /* From: */
 2153                         if (*ptr) {
 2154                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2155                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2156                             if (*art_full_name)
 2157                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2158                         } else {
 2159                             art->from = hash_str("");
 2160 #ifdef DEBUG
 2161                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2162                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2163 #endif /* DEBUG */
 2164                         }
 2165                         break;
 2166 
 2167                     case 3: /* Date: */
 2168                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2169 #ifdef DEBUG
 2170                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2171                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2172 #endif /* DEBUG */
 2173                         break;
 2174 
 2175                     case 4: /* Message-ID: */
 2176                         if (*ptr)
 2177                             art->msgid = my_strdup(ptr);
 2178                         else {
 2179                             art->msgid = NULL;
 2180 #ifdef DEBUG
 2181                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2182                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2183 #endif /* DEBUG */
 2184                         }
 2185                         break;
 2186 
 2187                     case 5: /* References: */
 2188                         if (*ptr)
 2189                             art->refs = my_strdup(ptr);
 2190                         else
 2191                             art->refs = NULL;
 2192                         break;
 2193 
 2194                     case 6: /* :bytes || Bytes: */
 2195                         if (*ptr) {
 2196 #ifdef DEBUG
 2197                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2198                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2199 #endif /* DEBUG */
 2200                         }
 2201                         break;
 2202 
 2203                     case 7: /* :lines || Lines: */
 2204                         if (*ptr) {
 2205                             if (isdigit((unsigned char) *ptr))
 2206                                 art->line_count = atoi(ptr);
 2207                             else {
 2208                                 art->line_count = 0;
 2209 #ifdef DEBUG
 2210                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2211                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2212 #endif /* DEBUG */
 2213                             }
 2214                         } else
 2215                             art->line_count = 0;
 2216                         break;
 2217 
 2218                     default:
 2219                         break;
 2220                 }
 2221             }
 2222 
 2223             /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
 2224             if (ofmt[count].type == OVER_T_FSTRING) {
 2225                 if (*ptr) {
 2226                     if (!strcasecmp(ofmt[count].name, "Xref:")) {
 2227                         if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
 2228                             FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
 2229                             art->xref = my_strdup(q);
 2230                         }
 2231 #ifdef DEBUG
 2232                         else {
 2233                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2234                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2235                         }
 2236 #endif /* DEBUG */
 2237                         continue;
 2238                     }
 2239                     /*
 2240                      * handling of addition overview fields
 2241                      * goes here
 2242                      */
 2243 #ifdef DEBUG
 2244                     if ((debug & DEBUG_NNTP) && verbose > 1)
 2245                         debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
 2246 #endif /* DEBUG */
 2247                     /* if we're lucky we've Path in NOV */
 2248                     /*
 2249                      * if reading locally cached overview data try
 2250                      * path regardless of the server OVERVIEW.FMT
 2251                      */
 2252                     if (local || !strcasecmp(ofmt[count].name, "Path:")) {
 2253                         if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
 2254                             if (!path_found)
 2255                                 path_found = TRUE;
 2256                             FreeIfNeeded(art->path);
 2257                             art->path = my_strdup(q);
 2258 #ifdef DEBUG
 2259                             if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
 2260                                 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
 2261 #endif /* DEBUG */
 2262 
 2263                         }
 2264                         continue;
 2265                     }
 2266                 }
 2267                 continue;
 2268             }
 2269         }
 2270 
 2271         /*
 2272          * RFC says Message-ID is mandatory in newsgroups (but not in
 2273          * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
 2274          * threading in mailgroups
 2275          */
 2276         if (!art->msgid && group->type == GROUP_TYPE_NEWS)
 2277             continue;
 2278 
 2279         /* we might lose accuracy here, but that shouldn't hurt */
 2280         if (artnum % (MODULO_COUNT_NUM * 20) == 0)
 2281             show_progress(group_msg, artnum - min, max - min);
 2282 
 2283         top_art++;              /* Basically this statement commits the article */
 2284     }
 2285 
 2286     free(group_msg);
 2287     TIN_FCLOSE(fp);
 2288 
 2289     if (tin_errno)
 2290         return -1;
 2291 
 2292 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
 2293     if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
 2294         char cbuf[HEADER_LEN];
 2295         int i;
 2296         static t_bool found;
 2297         static t_bool first = TRUE;
 2298 
 2299         if (first) {
 2300             found = TRUE;
 2301             /*
 2302              * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
 2303              */
 2304             if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
 2305                 if (!*nntp_caps.headers_range) {
 2306                     i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
 2307 
 2308                     found = FALSE;
 2309                     switch (i) {
 2310                         case 215:
 2311                             while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 2312 #   ifdef DEBUG
 2313                                 if (debug & DEBUG_NNTP)
 2314                                     debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2315 #   endif /* DEBUG */
 2316                                 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
 2317                                     found = TRUE;
 2318                                 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 2319                                 strcat(nntp_caps.headers_range, ptr);
 2320                                 strcat(nntp_caps.headers_range, "\n");
 2321                             }
 2322                             break;
 2323 
 2324                         default:
 2325                             break;
 2326                     }
 2327                     first = FALSE;
 2328                 } else {
 2329                     found = FALSE;
 2330                     if ((ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
 2331                         do {
 2332                             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
 2333                                 found = TRUE;
 2334                         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 2335                     }
 2336                 }
 2337             }
 2338         }
 2339 
 2340         if (found) {
 2341             snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
 2342             group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
 2343             if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
 2344                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
 2345 #   ifdef DEBUG
 2346                     if ((debug & DEBUG_NNTP) && verbose)
 2347                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2348 #   endif /* DEBUG */
 2349 
 2350                     artnum = atoartnum(ptr);
 2351                     if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
 2352                         continue;
 2353                     art = &arts[top_art];
 2354                     set_article(art);
 2355                     if (!art->xref && !strstr(ptr, "(none)")) {
 2356                         if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
 2357                             continue;
 2358                         ptr = q;
 2359                         while (*ptr && isspace((int) *ptr))
 2360                             ptr++;
 2361                         q = strchr(ptr, '\n');
 2362                         if (q)
 2363                             *q = '\0';
 2364                         art->xref = my_strdup(ptr);
 2365                     }
 2366                     /* we might lose accuracy here, but that shouldn't hurt */
 2367                     if (artnum % (MODULO_COUNT_NUM * 20) == 0)
 2368                         show_progress(group_msg, artnum - min, max - min);
 2369                 }
 2370             }
 2371             free(group_msg);
 2372         }
 2373     }
 2374 #endif /* NNTP_ABLE && XHDR_XREF */
 2375 
 2376     if (local) {
 2377 #ifdef NNTP_ABLE
 2378         if (filter_on_path(group)) {
 2379             struct t_article_range *ranges, *curr;
 2380             t_bool supported = TRUE;
 2381             int curr_range, range_cnt;
 2382 
 2383             /*
 2384              * Get the ranges without Path: header and try to fetch the
 2385              * headers
 2386              */
 2387             if ((ranges = build_range_list(min, *top, &range_cnt))) {
 2388                 curr = ranges;
 2389                 curr_range = 1;
 2390                 while (curr && supported) {
 2391                     if (curr->cnt)
 2392                         supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
 2393                     curr = curr->next;
 2394                 }
 2395                 if (!supported && path_in_ofmt) {
 2396                     /*
 2397                      * fetching Path: headers via [X]HDR or XPAT has failed
 2398                      * Path: is in the servers overview so let the next
 2399                      * read_overview() fetch them
 2400                      */
 2401                     free_art_array();
 2402                     free_msgids();
 2403                     top_art = 0;
 2404                     *top = T_ARTNUM_CONST(0);
 2405                     expired = 0;
 2406                 }
 2407                 *rebuild_cache = TRUE;
 2408                 while (ranges) {
 2409                     curr = ranges;
 2410                     ranges = curr->next;
 2411                     free(curr);
 2412                 }
 2413             }
 2414         }
 2415 #endif /* NNTP_ABLE */
 2416     } else
 2417         if (!path_found && filter_on_path(group)) {
 2418 #ifdef NNTP_ABLE
 2419             if (!get_path_header(1, 1, group, min, *top))
 2420 #endif /* NNTP_ABLE */
 2421                 wait_message(2, _(txt_cannot_filter_on_path));
 2422         }
 2423     return expired;
 2424 }
 2425 
 2426 
 2427 /*
 2428  * Write an Nov/Xover index file. Fields are separated by '\t'.
 2429  *
 2430  * Format:
 2431  *  1. article number (ie. 183)                [mandatory]
 2432  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 2433  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 2434  *  4. Date: line     (rfc822 format)          [mandatory]
 2435  *  5. MessageID:     (ie. <123@example.net>)  [mandatory]
 2436  *  6. References:    (ie. <message-id> ....)  [optional]
 2437  *  7. Byte count     (Skipped - not used)     [mandatory]
 2438  *  8. Line count     (ie. 23)                 [mandatory]
 2439  *  9. Xref: line     (ie. alt.test:389)       [optional]
 2440  *
 2441  * TODO: as we don't use the original data, we currently can't store
 2442  *       the data (from/subject) in the original charset (we don't store
 2443  *       that info). this has the advantage that we can avoid raw 8bit data
 2444  *       in our overviews, but the disadvantage that we might store the data
 2445  *       with a wrong charset and thus lose information. a similar problem
 2446  *       exists with the data for the from:-line, we don't store it in the
 2447  *       original format, whenever our from-parser (partially) fails we'll
 2448  *       lose information in our overviews (but those couldn't be handled
 2449  *       by tin anyway, so this is not a real problem).
 2450  *       long-term solution: store the original data in the overview
 2451  *       (tin has to handle raw 8bit data and other ugly stuff in the
 2452  *       overviews anyway and thus we preserver as much info as possible)
 2453  *       this would require some changes in read_overview() and
 2454  *       parse_headers(): don't do the decoding/unfolding there, but in a
 2455  *       second pass right after write_overview(), or two additional fields
 2456  *       which hold the raw data for from/subject. the latter has the
 2457  *       disadvantage that it costs (much) more memory.
 2458  */
 2459 static void
 2460 write_overview(
 2461     struct t_group *group)
 2462 {
 2463     FILE *fp;
 2464     int i;
 2465     struct t_article *article;
 2466 #ifdef CHARSET_CONVERSION
 2467     int c = -1;
 2468 #endif /* CHARSET_CONVERSION */
 2469 
 2470     /*
 2471      * Can't write or caching is off or getart_limit is set
 2472      */
 2473     if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
 2474         return;
 2475 
 2476     if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
 2477         return;
 2478 
 2479     if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
 2480         SortBy(artnum_comp);
 2481 
 2482     /*
 2483      * Needed to preserve uniqueness in hashed private overview files
 2484      */
 2485     fprintf(fp, "%s\n", group->name);
 2486 
 2487 #ifdef CHARSET_CONVERSION
 2488     /* get undeclared_charset number if required */
 2489     if (group->attribute->undeclared_charset) {
 2490         for (i = 0; txt_mime_charsets[i] != NULL; i++) {
 2491             if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
 2492                 c = i;
 2493                 break;
 2494             }
 2495         }
 2496     }
 2497 #endif /* CHARSET_CONVERSION */
 2498 
 2499     if (verbose && batch_mode) /* -> lang.c */
 2500         wait_message(0, _("Writing %s\n"), group->name);
 2501 
 2502     for_each_art(i) {
 2503         char *p;
 2504         char *q, *ref;
 2505 
 2506         article = &arts[i];
 2507 
 2508         if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
 2509             ref = NULL;
 2510 
 2511             if (!group->attribute->post_8bit_header) { /* write encoded data */
 2512                 /*
 2513                  * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
 2514                  *       here and in print_from() in the CHARSET_CONVERSION case.
 2515                  *       note that this requires something like
 2516                  *          buffer_to_network(article->subject, "UTF-8");
 2517                  *       right before the rfc1522_encode() call.
 2518                  *
 2519                  *       if we would cache the original undecoded data, we could
 2520                  *       ignore stuff like this.
 2521                  */
 2522                 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
 2523                 /* as the subject might now be folded we have to unfold it */
 2524                 unfold_header(p);
 2525             } else { /* raw data */
 2526                 p = my_strdup(article->subject);
 2527 #ifdef CHARSET_CONVERSION
 2528                 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
 2529                     buffer_to_network(p, c);
 2530 #endif /* CHARSET_CONVERSION */
 2531             }
 2532 
 2533             /*
 2534              * replace any '\t's with ' ' in the references-data
 2535              *
 2536              * TODO: nntpext-draft might come up with a new scheme:
 2537              *       For all fields, the value is processed by first
 2538              *       removing all US-ASCII CRLF pairs and then replacing
 2539              *       each remaining US-ASCII NUL, TAB, CR, or LF character
 2540              *       with a single US-ASCII space (for example, CR LF LF TAB
 2541              *       will become two spaces).
 2542              */
 2543             if (article->refs) {
 2544                 ref = q = my_strdup(article->refs);
 2545                 while (*q) {
 2546                     if (*q == '\t')
 2547                         *q = ' ';
 2548                     q++;
 2549                 }
 2550             }
 2551 
 2552             fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
 2553                 article->artnum,
 2554                 p,
 2555 #ifdef CHARSET_CONVERSION
 2556                 print_from(group, article, c),
 2557 #else
 2558                 print_from(group, article, -1),
 2559 #endif /* CHARSET_CONVERSION */
 2560                 print_date(article->date),
 2561                 BlankIfNull(article->msgid),
 2562                 BlankIfNull(ref),
 2563                 0,  /* bytes */
 2564                 article->line_count);
 2565 
 2566             if (article->xref)
 2567                 fprintf(fp, "\tXref: %s", article->xref);
 2568 
 2569             if (article->path)
 2570                 fprintf(fp, "\tPath: %s", article->path);
 2571 
 2572             fprintf(fp, "\n");
 2573             free(p);
 2574             if (article->refs) {
 2575                 FreeIfNeeded(ref);
 2576             }
 2577         }
 2578         if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
 2579             show_progress(_("Writing overview cache..."), i, top_art);
 2580     }
 2581 #ifdef HAVE_FCHMOD
 2582     fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
 2583     /*
 2584      * TODO:
 2585      * add code for !HAVE_FCHMOD && HAVE_CHMOD
 2586      */
 2587 #endif /* HAVE_FCHMOD */
 2588     fclose(fp);
 2589 }
 2590 
 2591 
 2592 /*
 2593  * A complex little function to determine the correct overview index file
 2594  * according to 'mode' (read or write)
 2595  * NULL is returned if the current setup dictates otherwise
 2596  *
 2597  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
 2598  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
 2599  *
 2600  * Both of these are hashed
 2601  *
 2602  * GROUP_TYPE_NEWS index files are a little bit more complex
 2603  *
 2604  * When hashing the index filename will be in format number.number.
 2605  * Hashing the groupname gets a number. See if that #.1 file exists;
 2606  * if so, read first line. Is this the group we want? If no, try #.2.
 2607  * Repeat until no such file or we find an existing file that matches
 2608  * our group. Return pointer to path or NULL if not found.
 2609  */
 2610 static char *
 2611 find_nov_file(
 2612     struct t_group *group,
 2613     int mode)
 2614 {
 2615     FILE *fp;
 2616     const char *dir;
 2617     char buf[PATH_LEN];
 2618     int i;
 2619     struct stat sb;
 2620     unsigned long hash;
 2621     static char nov_file[PATH_LEN];
 2622     static t_bool once_only = FALSE;    /* Trap things that are done only 1 time */
 2623 
 2624     if (group == NULL || (mode != R_OK && mode != W_OK))
 2625         return NULL;
 2626 
 2627     switch (group->type) {
 2628         case GROUP_TYPE_MAIL:
 2629             dir = index_maildir;
 2630             break;
 2631 
 2632         case GROUP_TYPE_SAVE:
 2633             dir = index_savedir;
 2634             break;
 2635 
 2636         case GROUP_TYPE_NEWS:
 2637             /*
 2638              * nntp.caps.over_cmd is not an issue here, any gripes and warnings
 2639              * about [X]OVER are handled in nntp_open()
 2640              */
 2641 
 2642             /*
 2643              * When reading via NNTP, system wide overviews are irrelevant, of
 2644              * course, and the private overview filename will be the same for
 2645              * both reading and writing.
 2646              *
 2647              * When working locally, we only use a private cache for reading
 2648              * if requested and when system wide overviews don't already exist.
 2649              * When writing then only private overviews can be used since
 2650              * updating system wide overviews is not safe wrt locking etc.
 2651              *
 2652              * See if local overview file $SPOOLDIR/<groupname>/.overview exists
 2653              *
 2654              * INN >= 2.3.0 seems to use a new naming schemme with tradindexed
 2655              * see untested gross hack below; buffindexed and ovdb are not
 2656              * covered by the code at all.
 2657              */
 2658 #ifndef NNTP_ONLY
 2659             if (!read_news_via_nntp) {
 2660                 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
 2661                 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
 2662 #   if 0    /* TODO: FIXME - ugly hack for inn >= 2.3.0 with ovmethod tradindexed */
 2663                 {
 2664                     char *gn = my_strdup(group->name);
 2665                     size_t t, j;
 2666                     t_bool w = FALSE;
 2667 
 2668                     for (t = 1, j = 1; t < strlen(group->name); t++) {
 2669                         if (!w) {
 2670                             if (group->name[t] == '.') {
 2671                                 gn[j++] = '/';
 2672                                 w = TRUE;
 2673                             }
 2674                         } else {
 2675                             if (group->name[t] != '.') { /* illegal .. in name? */
 2676                                 gn[j++] = group->name[t];
 2677                                 w = FALSE;
 2678                             }
 2679                         }
 2680                     }
 2681                     gn[j] = '\0';
 2682 
 2683                     joinpath(nov_file, sizeof(nov_file), novrootdir, gn);
 2684                     free(gn);
 2685                     snprintf(nov_file + strlen(nov_file), sizeof(nov_file) - strlen(nov_file), "/%s.DAT", group->name);
 2686                 }
 2687 #   endif /* 0 */
 2688                 if (access(nov_file, R_OK) == 0) {
 2689                     if (mode == R_OK)
 2690                         return nov_file;        /* Use system wide overviews */
 2691                     else
 2692                         return NULL;            /* Don't write cache in this case */
 2693                 }
 2694             }
 2695 #endif /* !NNTP_ONLY */
 2696 
 2697             /*
 2698              * We only get here when private overviews are going to be used
 2699              * Go no further if they are explicitly turned off
 2700              */
 2701             if (!tinrc.cache_overview_files)
 2702                 return NULL;
 2703 
 2704             /*
 2705              * Append -<nntpserver> to private cache dir
 2706              */
 2707             if (!once_only && nntp_server) {
 2708                 size_t sp = sizeof(index_newsdir), ln = strlen(index_newsdir);
 2709 
 2710                 if (--sp - ln >= 2) {
 2711                     char *srv = my_strdup(nntp_server);
 2712 
 2713                     str_lwr(srv);
 2714                     strcat(index_newsdir, "-");
 2715                     my_strncpy(index_newsdir + ln + 1, srv, sp);
 2716                     free(srv);
 2717                 }
 2718                 once_only = TRUE;
 2719             }
 2720 
 2721             /*
 2722              * Only try to set up the private cache when writing. If it
 2723              * doesn't exist yet, then ergo we can't read from it.
 2724              * The cache will be checked/created on every write; a previous
 2725              * bug report complained that this was not the case
 2726              */
 2727             if (stat(index_newsdir, &sb) == -1) {           /* Private cache doesn't exist */
 2728                 if (mode == R_OK)
 2729                     return NULL;
 2730                 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
 2731                     return NULL;
 2732             } else {
 2733                 if (!S_ISDIR(sb.st_mode))
 2734                     return NULL;
 2735             }
 2736 
 2737             /*
 2738              * Update the newsgroups cache to point to the new location
 2739              * now that we know it is valid
 2740              */
 2741             if (!once_only)
 2742                 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
 2743 
 2744             dir = index_newsdir;
 2745             break;
 2746 
 2747         default: /* not reached */
 2748             return NULL;
 2749     }
 2750 
 2751     /*
 2752      * We only get here if writing to a private overview.
 2753      * These always have hashed filenames.
 2754      * Try <hash>.<seqno> and check the group name tagline until
 2755      * matching index file is found. If not found return next unused
 2756      * filename
 2757      */
 2758     hash = hash_groupname(group->name);
 2759 
 2760     for (i = 1; ; i++) {
 2761         char *ptr;
 2762 
 2763         snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
 2764         joinpath(nov_file, sizeof(nov_file), dir, buf);
 2765 
 2766         if ((fp = fopen(nov_file, "r")) == NULL)
 2767             break;
 2768 
 2769         /*
 2770          * No group name header, so not a valid index file => overwrite it
 2771          */
 2772         if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
 2773             fclose(fp);
 2774             break;
 2775         }
 2776         fclose(fp);
 2777 
 2778         if ((ptr = strrchr(buf, '\n')) != NULL)
 2779             *ptr = '\0';
 2780 
 2781         if (strcmp(buf, group->name) == 0)
 2782             break;
 2783     }
 2784 
 2785     return nov_file;
 2786 }
 2787 
 2788 
 2789 /*
 2790  * Run the index file updater only for the groups we've loaded.
 2791  */
 2792 void
 2793 do_update(
 2794     t_bool catchup)
 2795 {
 2796     int i, j, k = 0;
 2797     time_t beg_epoch = 0;
 2798     struct t_article *art;
 2799     struct t_group *group;
 2800 
 2801     if (verbose)
 2802         (void) time(&beg_epoch);
 2803 
 2804     /*
 2805      * loop through groups and update any required index files
 2806      */
 2807     for (i = 0; i < selmenu.max; i++) {
 2808         group = &active[my_group[i]];
 2809         /*
 2810          * FIXME: workaround to get a valid CURR_GROUP
 2811          * it also points to the currently processed group so that
 2812          * the correct attributes are used
 2813          * The correct fix is to get rid of CURR_GROUP
 2814          */
 2815         selmenu.curr = i;
 2816 
 2817         if (group->bogus || !group->subscribed)
 2818             continue;
 2819 
 2820         if (!index_group(group)) {
 2821             for_each_art(j) {
 2822                 art = &arts[j];
 2823                 FreeAndNull(art->refs);
 2824                 FreeAndNull(art->msgid);
 2825             }
 2826             continue;
 2827         }
 2828 
 2829         k++;
 2830 
 2831         if (verbose) {
 2832             my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
 2833             my_flush();
 2834         }
 2835 
 2836         if (catchup) {
 2837             for_each_art(j)
 2838                 art_mark(group, &arts[j], ART_READ);
 2839         }
 2840     }
 2841 
 2842     if (verbose) {
 2843         wait_message(0, _(txt_catchup_update_info),
 2844             (catchup ? _(txt_caughtup) : _(txt_updated)), k,
 2845             PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
 2846     }
 2847 }
 2848 
 2849 
 2850 static int
 2851 artnum_comp(
 2852     t_comptype p1,
 2853     t_comptype p2)
 2854 {
 2855     const struct t_article *s1 = (const struct t_article *) p1;
 2856     const struct t_article *s2 = (const struct t_article *) p2;
 2857 
 2858     /*
 2859      * s1->artnum less than s2->artnum
 2860      */
 2861     if (s1->artnum < s2->artnum)
 2862         return -1;
 2863 
 2864     /*
 2865      * s1->artnum greater than s2->artnum
 2866      */
 2867     if (s1->artnum > s2->artnum)
 2868         return 1;
 2869 
 2870     return 0;
 2871 }
 2872 
 2873 
 2874 /*
 2875  * return result of strcmp (reversed for descending)
 2876  */
 2877 static int
 2878 subj_comp_asc(
 2879     t_comptype p1,
 2880     t_comptype p2)
 2881 {
 2882     int retval;
 2883     const struct t_article *s1 = (const struct t_article *) p1;
 2884     const struct t_article *s2 = (const struct t_article *) p2;
 2885 
 2886     if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
 2887         return retval;
 2888 
 2889     return s1->date - s2->date > 0 ? 1 : -1;
 2890 }
 2891 
 2892 
 2893 static int
 2894 subj_comp_desc(
 2895     t_comptype p1,
 2896     t_comptype p2)
 2897 {
 2898     int retval;
 2899     const struct t_article *s1 = (const struct t_article *) p1;
 2900     const struct t_article *s2 = (const struct t_article *) p2;
 2901 
 2902     if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
 2903         return retval;
 2904 
 2905     return s1->date - s2->date > 0 ? 1 : -1;
 2906 }
 2907 
 2908 
 2909 /*
 2910  * return result of strcmp (reversed for descending)
 2911  */
 2912 static int
 2913 from_comp_asc(
 2914     t_comptype p1,
 2915     t_comptype p2)
 2916 {
 2917     int retval;
 2918     const struct t_article *s1 = (const struct t_article *) p1;
 2919     const struct t_article *s2 = (const struct t_article *) p2;
 2920 
 2921     if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
 2922         return retval;
 2923 
 2924     return s1->date - s2->date > 0 ? 1 : -1;
 2925 }
 2926 
 2927 
 2928 static int
 2929 from_comp_desc(
 2930     t_comptype p1,
 2931     t_comptype p2)
 2932 {
 2933     int retval;
 2934     const struct t_article *s1 = (const struct t_article *) p1;
 2935     const struct t_article *s2 = (const struct t_article *) p2;
 2936 
 2937     if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
 2938         return retval;
 2939 
 2940     return s1->date - s2->date > 0 ? 1 : -1;
 2941 }
 2942 
 2943 
 2944 /*
 2945  * Works like strcmp() for comparing time_t type values
 2946  * Return codes:
 2947  *  -1:     If p1 is before p2
 2948  *   0:     If they are the same time
 2949  *   1:     If p1 is after p2
 2950  * If the sort order is _not_ DATE_ASCEND then the sense of the above
 2951  * is reversed.
 2952  */
 2953 static int
 2954 date_comp_asc(
 2955     t_comptype p1,
 2956     t_comptype p2)
 2957 {
 2958     const struct t_article *s1 = (const struct t_article *) p1;
 2959     const struct t_article *s2 = (const struct t_article *) p2;
 2960 
 2961     /*
 2962      * s1->date less than s2->date
 2963      */
 2964     if (s1->date < s2->date)
 2965         return -1;
 2966 
 2967     /*
 2968      * s1->date greater than s2->date
 2969      */
 2970     if (s1->date > s2->date)
 2971         return 1;
 2972 
 2973     return 0;
 2974 }
 2975 
 2976 
 2977 static int
 2978 date_comp_desc(
 2979     t_comptype p1,
 2980     t_comptype p2)
 2981 {
 2982     const struct t_article *s1 = (const struct t_article *) p1;
 2983     const struct t_article *s2 = (const struct t_article *) p2;
 2984 
 2985     /*
 2986      * s2->date less than s1->date
 2987      */
 2988     if (s2->date < s1->date)
 2989         return -1;
 2990 
 2991     /*
 2992      * s2->date greater than s1->date
 2993      */
 2994     if (s2->date > s1->date)
 2995         return 1;
 2996 
 2997     return 0;
 2998 }
 2999 
 3000 
 3001 /*
 3002  * Same again, but for art[].score
 3003  */
 3004 static int
 3005 score_comp_asc(
 3006     t_comptype p1,
 3007     t_comptype p2)
 3008 {
 3009     const struct t_article *s1 = (const struct t_article *) p1;
 3010     const struct t_article *s2 = (const struct t_article *) p2;
 3011 
 3012     if (s1->score < s2->score)
 3013         return -1;
 3014 
 3015     if (s1->score > s2->score)
 3016         return 1;
 3017 
 3018     return s1->date - s2->date > 0 ? 1 : -1;
 3019 }
 3020 
 3021 
 3022 static int
 3023 score_comp_desc(
 3024     t_comptype p1,
 3025     t_comptype p2)
 3026 {
 3027     const struct t_article *s1 = (const struct t_article *) p1;
 3028     const struct t_article *s2 = (const struct t_article *) p2;
 3029 
 3030     if (s2->score < s1->score)
 3031         return -1;
 3032 
 3033     if (s2->score > s1->score)
 3034         return 1;
 3035 
 3036     return s1->date - s2->date > 0 ? 1 : -1;
 3037 }
 3038 
 3039 
 3040 /*
 3041  * Same again, but for art[].line_count
 3042  */
 3043 static int
 3044 lines_comp_asc(
 3045     t_comptype p1,
 3046     t_comptype p2)
 3047 {
 3048     const struct t_article *s1 = (const struct t_article *) p1;
 3049     const struct t_article *s2 = (const struct t_article *) p2;
 3050 
 3051     if (s1->line_count < s2->line_count)
 3052         return -1;
 3053 
 3054     if (s1->line_count > s2->line_count)
 3055         return 1;
 3056 
 3057     return s1->date - s2->date > 0 ? 1 : -1;
 3058 }
 3059 
 3060 
 3061 static int
 3062 lines_comp_desc(
 3063     t_comptype p1,
 3064     t_comptype p2)
 3065 {
 3066     const struct t_article *s1 = (const struct t_article *) p1;
 3067     const struct t_article *s2 = (const struct t_article *) p2;
 3068 
 3069     if (s2->line_count < s1->line_count)
 3070         return -1;
 3071 
 3072     if (s2->line_count > s1->line_count)
 3073         return 1;
 3074 
 3075     return s1->date - s2->date > 0 ? 1 : -1;
 3076 }
 3077 
 3078 
 3079 /*
 3080  * Compares the total score of two threads. Used for sorting base[].
 3081  */
 3082 static int
 3083 score_comp_base(
 3084     t_comptype p1,
 3085     t_comptype p2)
 3086 {
 3087     int a = get_score_of_thread((int) *(const long *) p1);
 3088     int b = get_score_of_thread((int) *(const long *) p2);
 3089 
 3090     /* If scores are equal, compare using the article sort order.
 3091      * This determines the order in a group of equally scored threads.
 3092      */
 3093     if (a == b) {
 3094         const struct t_article *s1 = &arts[*(const long *) p1];
 3095         const struct t_article *s2 = &arts[*(const long *) p2];
 3096         t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
 3097 
 3098         if (comp_func)
 3099             return (*comp_func)(s1, s2);
 3100         return 0;
 3101     }
 3102 
 3103     if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
 3104         return a > b ? 1 : -1;
 3105     return a < b ? 1 : -1;
 3106 }
 3107 
 3108 
 3109 /*
 3110  * Compare the date of the last posted article of two threads.
 3111  * Used for sorting base[].
 3112  */
 3113 static int
 3114 last_date_comp_base_desc(
 3115     t_comptype p1,
 3116     t_comptype p2)
 3117 {
 3118     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3119     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3120 
 3121     if (s2_last < s1_last)
 3122         return -1;
 3123 
 3124     if (s2_last > s1_last)
 3125         return 1;
 3126 
 3127     return 0;
 3128 }
 3129 
 3130 
 3131 static int
 3132 last_date_comp_base_asc(
 3133     t_comptype p1,
 3134     t_comptype p2)
 3135 {
 3136     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3137     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3138 
 3139     if (s2_last > s1_last)
 3140         return -1;
 3141 
 3142     if (s2_last < s1_last)
 3143         return 1;
 3144 
 3145     return 0;
 3146 }
 3147 
 3148 
 3149 static time_t
 3150 get_last_posting_date(
 3151     long n)
 3152 {
 3153     long i;
 3154     time_t last = (time_t) 0;
 3155 
 3156     for (i = n; i >= 0; i = arts[i].thread) {
 3157         if (arts[i].date > last)
 3158             last = arts[i].date;
 3159     }
 3160 
 3161     return last;
 3162 }
 3163 
 3164 
 3165 void
 3166 set_article(
 3167     struct t_article *art)
 3168 {
 3169     art->subject = NULL;
 3170     art->from = NULL;
 3171     art->name = NULL;
 3172     art->date = (time_t) 0;
 3173     art->xref = NULL;
 3174     art->msgid = NULL;
 3175     art->refs = NULL;
 3176     art->refptr = NULL;
 3177     art->line_count = -1;
 3178     art->tagged = 0;
 3179     art->thread = ART_EXPIRED;
 3180     art->prev = ART_NORMAL;
 3181     art->score = 0;
 3182     art->status = ART_UNREAD;
 3183     art->killed = ART_NOTKILLED;
 3184     art->zombie = FALSE;
 3185     art->delete_it = FALSE;
 3186     art->selected = FALSE;
 3187     art->inrange = FALSE;
 3188     art->matched = FALSE;
 3189     art->keep_in_base = FALSE;
 3190     art->multipart_subj = FALSE;
 3191 }
 3192 
 3193 
 3194 /*
 3195  * Do a binary chop to see if 'art' (an article number) exists in arts[]
 3196  * Naturally arts[] must be sorted on artnum
 3197  * Return index into arts[] or -1
 3198  */
 3199 static int
 3200 valid_artnum(
 3201     t_artnum art)
 3202 {
 3203     int prev, range;
 3204     int dctop = top_art;
 3205     int cur = 1;
 3206 
 3207     while ((dctop >>= 1))
 3208         cur <<= 1;
 3209 
 3210     range = cur >> 1;
 3211     cur--;
 3212 
 3213     forever {
 3214         if (arts[cur].artnum == art)
 3215             return cur;
 3216 
 3217         prev = cur;
 3218         cur += ((arts[cur].artnum < art) ? range : -range);
 3219         if (prev == cur)
 3220             break;
 3221 
 3222         if (cur >= top_art)
 3223             cur = top_art - 1;
 3224 
 3225         range >>= 1;
 3226     }
 3227     return -1;
 3228 }
 3229 
 3230 
 3231 /*
 3232  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
 3233  * Needed if arts[] is not sorted on artnum
 3234  * Return index into arts[] or -1
 3235  */
 3236 int
 3237 find_artnum(
 3238     t_artnum art)
 3239 {
 3240     int i;
 3241 
 3242     for_each_art(i) {
 3243         if (arts[i].artnum == art)
 3244             return i;
 3245     }
 3246     return -1;
 3247 }
 3248 
 3249 
 3250 /*----------------------------- Overview handling -----------------------*/
 3251 /* TODO: use
 3252  *           setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
 3253  *           my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
 3254  *       instead?
 3255  */
 3256 static char *
 3257 print_date(
 3258     time_t secs)
 3259 {
 3260     static char date[25];
 3261     struct tm *tm;
 3262     static const char *const months_a[] = {
 3263         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 3264         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
 3265     };
 3266 
 3267     if ((tm = gmtime(&secs)) != NULL)
 3268         snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
 3269             tm->tm_mday,
 3270             months_a[tm->tm_mon],
 3271             tm->tm_year + 1900,
 3272             tm->tm_hour, tm->tm_min, tm->tm_sec);
 3273     else
 3274         snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
 3275 
 3276     return date;
 3277 }
 3278 
 3279 
 3280 static char *
 3281 print_from(
 3282     struct t_group *group,
 3283     struct t_article *article,
 3284     int charset)
 3285 {
 3286     char *p, *q;
 3287     static char from[PATH_LEN];
 3288 
 3289     *from = '\0';
 3290 
 3291     if (article->name != NULL) {
 3292         q = my_strdup(article->name);
 3293 #ifdef CHARSET_CONVERSION
 3294         if (charset != -1)
 3295             buffer_to_network(q, charset);
 3296 #endif /* CHARSET_CONVERSION */
 3297         p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
 3298         unfold_header(p);
 3299         if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
 3300             snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3301         else
 3302             snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3303 
 3304         free(p);
 3305         free(q);
 3306     } else
 3307         snprintf(from, sizeof(from), "<%s>", article->from);
 3308 
 3309     return from;
 3310 }
 3311 
 3312 
 3313 /*
 3314  * Open a group news overview file
 3315  * Use NNTP XOVER where possible unless 'local' is set
 3316  */
 3317 static FILE *
 3318 open_xover_fp(
 3319     struct t_group *group,
 3320     const char *mode,
 3321     t_artnum min,
 3322     t_artnum max,
 3323     t_bool local)
 3324 {
 3325 #ifdef NNTP_ABLE
 3326     if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
 3327         char line[NNTP_STRLEN];
 3328 
 3329         if (!max)
 3330             return NULL;
 3331         if (min == max)
 3332             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
 3333         else
 3334             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
 3335         return (nntp_command(line, OK_XOVER, NULL, 0));
 3336     }
 3337 #endif /* NNTP_ABLE */
 3338     {
 3339         FILE *fp;
 3340         char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
 3341 
 3342         if (nov_file != NULL) {
 3343             if ((fp = fopen(nov_file, mode)) != NULL)
 3344                 return fp;
 3345 
 3346             if (*mode != 'r')
 3347                 error_message(2, _(txt_cannot_open), nov_file);
 3348         }
 3349     }
 3350     return NULL;
 3351 }
 3352 
 3353 
 3354 #ifdef USE_HEAPSORT
 3355 int
 3356 tin_sort(
 3357     void *sbase,
 3358     size_t nel,
 3359     size_t width,
 3360     t_compfunc compar)
 3361 {
 3362     int rc;
 3363 
 3364     switch (tinrc.sort_function) {
 3365         case 0:
 3366             qsort(sbase, nel, width, compar);
 3367             rc = 0;
 3368             break;
 3369 
 3370         case 1:
 3371             rc = heapsort(sbase, nel, width, compar);
 3372             break;
 3373 
 3374         default:
 3375             rc = -1;
 3376             break;
 3377     }
 3378     return rc;
 3379 }
 3380 #endif /* USE_HEAPSORT */