"Fossies" - the Fresh Open Source Software Archive

Member "tin-2.4.5/src/art.c" (1 Dec 2020, 90939 Bytes) of package /linux/misc/tin-2.4.5.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "art.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.4.4_vs_2.4.5.

    1 /*
    2  *  Project   : tin - a Usenet reader
    3  *  Module    : art.c
    4  *  Author    : I.Lea & R.Skrenta
    5  *  Created   : 1991-04-01
    6  *  Updated   : 2020-07-08
    7  *  Notes     :
    8  *
    9  * Copyright (c) 1991-2021 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
   10  * All rights reserved.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  *
   16  * 1. Redistributions of source code must retain the above copyright notice,
   17  *    this list of conditions and the following disclaimer.
   18  *
   19  * 2. Redistributions in binary form must reproduce the above copyright
   20  *    notice, this list of conditions and the following disclaimer in the
   21  *    documentation and/or other materials provided with the distribution.
   22  *
   23  * 3. Neither the name of the copyright holder nor the names of its
   24  *    contributors may be used to endorse or promote products derived from
   25  *    this software without specific prior written permission.
   26  *
   27  * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   28  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   31  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   32  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   33  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   34  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   35  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   36  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   37  * POSSIBILITY OF SUCH DAMAGE.
   38  */
   39 
   40 
   41 #ifndef TIN_H
   42 #   include "tin.h"
   43 #endif /* !TIN_H */
   44 #ifndef TCURSES_H
   45 #   include "tcurses.h"
   46 #endif /* !TCURSES_H */
   47 #ifndef NEWSRC_H
   48 #   include "newsrc.h"
   49 #endif /* !NEWSRC_H */
   50 
   51 #ifndef STPWATCH_H
   52 #   include "stpwatch.h"
   53 #endif /* !STPWATCH_H */
   54 
   55 /*
   56  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
   57  */
   58 #define SortBy(func)    tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
   59 
   60 int top_art = 0;                /* # of articles in arts[] */
   61 
   62 /*
   63  * Local prototypes
   64  */
   65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
   66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
   67 static char *find_nov_file(struct t_group *group, int mode);
   68 static char *print_date(time_t secs);
   69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
   70 static int artnum_comp(t_comptype p1, t_comptype p2);
   71 static int base_comp(t_comptype p1, t_comptype p2);
   72 static int date_comp_asc(t_comptype p1, t_comptype p2);
   73 static int date_comp_desc(t_comptype p1, t_comptype p2);
   74 static int from_comp_asc(t_comptype p1, t_comptype p2);
   75 static int from_comp_desc(t_comptype p1, t_comptype p2);
   76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
   77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
   78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
   79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
   80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
   81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
   82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
   83 static int score_comp_asc(t_comptype p1, t_comptype p2);
   84 static int score_comp_desc(t_comptype p1, t_comptype p2);
   85 static int score_comp_base(t_comptype p1, t_comptype p2);
   86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
   87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
   88 static int valid_artnum(t_artnum art);
   89 static t_artnum find_first_unread(struct t_group *group);
   90 static t_artnum setup_hard_base(struct t_group *group);
   91 static t_bool parse_headers(FILE *fp, struct t_article *h);
   92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
   93 static time_t get_last_posting_date(long n);
   94 static void sort_base(unsigned int sort_threads_type);
   95 static void thread_by_multipart(void);
   96 static void thread_by_percentage(unsigned int percentage);
   97 static void thread_by_subject(void);
   98 static void write_overview(struct t_group *group);
   99 #ifdef NNTP_ABLE
  100     static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
  101     static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
  102 #endif /* NNTP_ABLE */
  103 
  104 
  105 /*
  106  * Display a suitable 'entering group' message if screen needs redrawing
  107  * Allow for the non-printing %s, and the %-age counter
  108  */
  109 void
  110 show_art_msg(
  111     const char *group)
  112 {
  113     wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
  114 }
  115 
  116 
  117 /*
  118  * Construct the pointers to the first (base) article in each thread.
  119  * If we are showing only unread, then point to the first unread. I have
  120  * no idea why this should be so, it causes problems elsewhere [which_response]
  121  */
  122 void
  123 find_base(
  124     struct t_group *group)
  125 {
  126     int i, j;
  127 
  128     grpmenu.max = 0;
  129 
  130 #ifdef DEBUG
  131     if (debug & DEBUG_FILTER)
  132         debug_print_arts();
  133 #endif /* DEBUG */
  134 
  135     for_each_art(i) {
  136         /*
  137          * .prev will be set on each article that is after the first article in
  138          * the thread. Invalid articles which have been expired will have
  139          * .thread set to ART_EXPIRED
  140          */
  141         if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
  142             continue;
  143 
  144         if (grpmenu.max >= max_base)
  145             expand_base();
  146 
  147         if (group->attribute->show_only_unread_arts) {
  148             if (arts[i].status != ART_READ || arts[i].keep_in_base)
  149                 base[grpmenu.max++] = i;
  150             else {
  151                 /* Find 1st unread art in thread */
  152                 for (j = i; j >= 0; j = arts[j].thread) {
  153                     if (arts[j].status != ART_READ || arts[j].keep_in_base) {
  154                         base[grpmenu.max++] = i;
  155                         break;
  156                     }
  157                 }
  158             }
  159         } else
  160             base[grpmenu.max++] = i;
  161     }
  162     /* sort base[] */
  163     if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
  164         sort_base(group->attribute->sort_threads_type);
  165 }
  166 
  167 
  168 /*
  169  * Longword comparison routine for the tin_sort()
  170  */
  171 static int
  172 base_comp(
  173     t_comptype p1,
  174     t_comptype p2)
  175 {
  176     const t_artnum *a = (const t_artnum *) p1;
  177     const t_artnum *b = (const t_artnum *) p2;
  178 
  179     if (*a < *b)
  180         return -1;
  181 
  182     if (*a > *b)
  183         return 1;
  184 
  185     return 0;
  186 }
  187 
  188 
  189 /*
  190  * via NNTP:
  191  *   Issue a LISTGROUP command
  192  *   Read the article numbers existing in the group into base[]
  193  *   If the LISTGROUP failed, issue a GROUP command. Use the results to
  194  *   create a less accurate version of base[]
  195  *   This data will already be sorted
  196  *
  197  * on local spool:
  198  *   Read the spool dir to populate base[] as above. Sort it.
  199  *
  200  * Grow the arts[] and bitmaps as needed.
  201  * NB: the output will be sorted on artnum
  202  *
  203  * grpmenu.max is one past top.
  204  * Returns total number of articles in group, or -1 on error
  205  */
  206 static t_artnum
  207 setup_hard_base(
  208     struct t_group *group)
  209 {
  210     t_artnum total = 0;
  211 
  212     grpmenu.max = 0;
  213 
  214     /*
  215      * If reading with NNTP, issue a LISTGROUP
  216      */
  217     if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
  218 #ifdef NNTP_ABLE
  219         char buf[NNTP_STRLEN];
  220         char line[NNTP_STRLEN];
  221         int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  222         FILE *fp;
  223         t_artnum last, start, count = 0, j = 0;
  224         static t_bool skip_listgroup = FALSE;
  225 
  226         /*
  227          * Some nntp servers are broken and need an extra GROUP command
  228          * (reported by reorx@irc.pl). This affects (old?) versions of
  229          * nntpcache, leafnode and SurgeNews. Usually this should not be
  230          * needed.
  231          *
  232          * For getart_limit recheck lowwatermark as at least giganews gives
  233          * very different results for LIST ACTIVE (3 year retention for all)
  234          * and GROUP (based on the clients contract).
  235          * Calculate range and prepare base[] not to lose unread arts.
  236          */
  237         if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
  238             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  239             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  240                 return -1;
  241 
  242             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
  243                 return -1;
  244 
  245             if (getart_limit > 0) {
  246                 j = group->xmax - getart_limit;
  247                 count = MAX(find_first_unread(group), start);
  248             }
  249             if (getart_limit < 0) {
  250                 j = getart_limit + find_first_unread(group);
  251                 count = group->xmin;
  252             }
  253             if (j < group->xmin)
  254                 j = group->xmin;
  255 
  256             for (; count < j; count++) {
  257                 if (grpmenu.max >= max_base)
  258                     expand_base();
  259                 base[grpmenu.max++] = count;
  260             }
  261         }
  262 
  263         /*
  264          * See if LISTGROUP works
  265          */
  266         if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
  267             if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
  268                 /* RFC 3977 allows ranges in LISTGROUP */
  269                 if (getart_limit > 0)
  270                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
  271                 else /* getart_limit < 0; fetch till newest art */
  272                     snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
  273 
  274             } else /* for RFC 977 just use GROUP */
  275                 skip_listgroup = TRUE;
  276 
  277         } else /* get all article numbers */
  278             snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
  279 
  280         if (!skip_listgroup) {
  281             if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
  282                 char *ptr;
  283 
  284 #   ifdef DEBUG
  285                 if ((debug & DEBUG_NNTP) && verbose > 1)
  286                     debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  287 #   endif /* DEBUG */
  288 
  289                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
  290                     if (grpmenu.max >= max_base)
  291                         expand_base();
  292                     base[grpmenu.max++] = atoartnum(ptr);
  293                     total++;
  294                 }
  295 
  296                 if (tin_errno)
  297                     return -1;
  298             } else
  299                 skip_listgroup = TRUE;
  300         }
  301 
  302         if (skip_listgroup) { /* LISTGROUP was skipped or failed */
  303             /*
  304              * Handle the obscure case that the user aborted before the LISTGROUP
  305              * had a chance to respond
  306              */
  307             if (tin_errno)
  308                 return -1;
  309 
  310             snprintf(buf, sizeof(buf), "GROUP %s", group->name);
  311             if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
  312                 return -1;
  313 
  314             if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
  315                 return -1;
  316 
  317 #   ifdef DEBUG
  318             if ((debug & DEBUG_NNTP) && verbose > 1)
  319                 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
  320 #   endif /* DEBUG */
  321             total = count;
  322             grpmenu.max = 0;
  323             if (getart_limit > 0) {
  324                 if ((j = find_first_unread(group)) > start) {
  325                     if (group->xmax > getart_limit) {
  326                         start = MIN(j, group->xmax - getart_limit);
  327                         total = getart_limit;
  328                     } else
  329                         start = j;
  330                 }
  331             }
  332             if (getart_limit < 0) {
  333                 if ((j = (getart_limit + find_first_unread(group))) > start)
  334                     start = j;
  335             }
  336             while (start <= last) {
  337                 if (grpmenu.max >= max_base)
  338                     expand_base();
  339                 base[grpmenu.max++] = start++;
  340             }
  341         }
  342 #endif /* NNTP_ABLE */
  343     /*
  344      * Reading off local spool, read the directory files
  345      */
  346     } else {
  347         DIR *d;
  348         DIR_BUF *e;
  349         char group_path[PATH_LEN];
  350         t_artnum art;
  351 
  352         make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
  353 
  354         if ((d = opendir(group_path)) != NULL) {
  355             while ((e = readdir(d)) != NULL) {
  356                 art = atoartnum(e->d_name);
  357                 if (art >= 1) {
  358                     total++;
  359                     if (grpmenu.max >= max_base)
  360                         expand_base();
  361                     base[grpmenu.max++] = art;
  362                 }
  363             }
  364             CLOSEDIR(d);
  365             tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
  366         } else {
  367             perror_message(_(txt_cannot_open), group_path);
  368 #if 0
  369             if (access(group_path, R_OK) != 0)
  370                 error_message(2, _(txt_not_exist));
  371 #endif /* 0 */
  372             return -1;
  373         }
  374     }
  375 
  376     if (grpmenu.max) {
  377         if (base[grpmenu.max - 1] > group->xmax)
  378             group->xmax = base[grpmenu.max - 1];
  379         expand_bitmap(group, base[0]);
  380     }
  381 
  382     return total;
  383 }
  384 
  385 
  386 /*
  387  * Main group indexing routine.
  388  *
  389  * Will read any existing index, create or incrementally update
  390  * the index by looking at the articles in the spool directory,
  391  * and attempt to write a new index if necessary.
  392  *
  393  * Returns FALSE if the user aborted the indexing, otherwise TRUE
  394  */
  395 t_bool
  396 index_group(
  397     struct t_group *group)
  398 {
  399     int i;
  400     int changed;                /* Count of articles whose overview has changed */
  401     int getart_limit;
  402     int respnum;
  403     int total;
  404     t_artnum last_read_article;
  405     t_artnum min, new_min, max;
  406     t_bool caching_xover;
  407     t_bool filtered;
  408     t_bool path_in_nov = FALSE;
  409     t_bool rebuild_cache = FALSE;
  410 
  411     if (group == NULL)
  412         return TRUE;
  413 
  414     if (!batch_mode)
  415         show_art_msg(group->name);
  416     else {
  417         if (verbose) /* -> lang.c */
  418             wait_message(0, _("Reading %s\n"), group->name);
  419     }
  420 
  421     signal_context = cArt;          /* Set this only once curr_group is valid */
  422 
  423     hash_reclaim();
  424     free_art_array();
  425     free_msgids();
  426 
  427     BegStopWatch("setup_hard_base()");
  428 
  429     /*
  430      * Get list of valid article numbers
  431      */
  432     if (setup_hard_base(group) < 0)
  433         return FALSE;
  434 
  435     EndStopWatch();
  436     PrintStopWatch();
  437 
  438 #ifdef DEBUG
  439     if (debug & DEBUG_NEWSRC) {
  440         debug_print_comment("Before read_overview");
  441         debug_print_bitmap(group, NULL);
  442     }
  443 #endif /* DEBUG */
  444 
  445     min = grpmenu.max ? base[0] : group->xmin;
  446     max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
  447 
  448     getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
  449 
  450     if (getart_limit > 0) {
  451         if (grpmenu.max && (grpmenu.max > getart_limit))
  452             min = base[grpmenu.max - getart_limit];
  453         else
  454             getart_limit = 0;
  455     } else if (getart_limit < 0) {
  456         t_artnum first_unread = find_first_unread(group);
  457 
  458         if (min - first_unread < getart_limit)
  459             min = first_unread + getart_limit;
  460         else
  461             getart_limit = 0;
  462     }
  463 
  464     /*
  465      * Quit now if no articles
  466      */
  467     if (max < 0)
  468         return FALSE;
  469 
  470     top_art = 0;
  471     last_read_article = T_ARTNUM_CONST(0);
  472 
  473     /*
  474      * Read in the existing overview data for min..max
  475      * This read has local=TRUE set if locally caching XOVER records to ensure
  476      * we pull in any private overview caches in preference to using OVER
  477      *
  478      * When reading local spool, this will pull in the system wide overview
  479      * cache (if found) otherwise the private overview cache will be read
  480      */
  481     caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
  482     if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
  483         return FALSE;   /* user aborted indexing */
  484 
  485     /*
  486      * Fill in the range last_read_article...max using XOVER
  487      * Only do this if the previous read_overview() was against private cache
  488      */
  489     if ((last_read_article < max) && caching_xover) {
  490         new_min = (last_read_article >= min) ? last_read_article + 1 : min;
  491 
  492         if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
  493             return FALSE;   /* user aborted indexing */
  494         else
  495             changed += i;
  496     } else
  497         caching_xover = FALSE;
  498 
  499     /*
  500      * Mark as UNTHREADED all articles that have been verified as valid
  501      * Get num of new arts to index so the user will have an idea of index time
  502      */
  503     for (i = 0, total = 0; i < grpmenu.max; i++) {
  504         if ((respnum = valid_artnum(base[i])) >= 0) {
  505             arts[respnum].thread = ART_UNTHREADED;
  506             continue;
  507         }
  508         if (base[i] <= last_read_article)       /* It is vital this test be done last */
  509             continue;
  510         total++;
  511     }
  512 
  513     /*
  514      * Add any articles to arts[] that are new or were killed
  515      */
  516     if (total > 0) {
  517         new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
  518 
  519         if ((i = read_art_headers(group, total, new_min)) == -1)
  520             return FALSE;       /* user aborted indexing */
  521         else
  522             changed += i;
  523     }
  524 
  525 #ifdef DEBUG
  526     if (debug & DEBUG_NEWSRC) {
  527         debug_print_comment("Before parse_unread_arts()");
  528         debug_print_bitmap(group, NULL);
  529     }
  530 #endif /* DEBUG */
  531     /*
  532      * Do this before calling art_mark(,, ART_READ) if you want
  533      * the unread count to be correct.
  534      */
  535     min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
  536     parse_unread_arts(group, min);
  537 #ifdef DEBUG
  538     if (debug & DEBUG_NEWSRC) {
  539         debug_print_comment("After parse_unread_arts()");
  540         debug_print_bitmap(group, NULL);
  541     }
  542 #endif /* DEBUG */
  543 
  544     /*
  545      * Stat all articles to see if any have expired
  546      */
  547     for_each_art(i) {
  548         if (arts[i].thread == ART_EXPIRED) {
  549             changed++;
  550 #ifdef DEBUG
  551             if (debug & DEBUG_NEWSRC)
  552                 debug_print_comment("art.c: index_group() purging...");
  553 #endif /* DEBUG */
  554             art_mark(group, &arts[i], ART_READ);
  555             if (group->attribute->show_only_unread_arts)
  556                 arts[i].keep_in_base = FALSE;
  557         }
  558         if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
  559             path_in_nov = TRUE;
  560     }
  561 
  562     /*
  563      * Only rewrite the index if it has changed
  564      * TODO review the exact logic behind "|| caching_xover"
  565      */
  566     if (changed || caching_xover || rebuild_cache)
  567         write_overview(group);
  568 
  569     /*
  570      * Create the reference tree. The msgid and ref ptrs will
  571      * be free()d now that the NovFile has been written.
  572      */
  573     build_references(group);
  574 
  575     /*
  576      * Needs access to the reference tree
  577      */
  578     filtered = filter_articles(group);
  579 
  580     BegStopWatch("make_threads()");
  581 
  582     /*
  583      * Thread the group
  584      */
  585     make_threads(group, FALSE);
  586 
  587     EndStopWatch();
  588     PrintStopWatch();
  589 
  590     if ((changed > 0 || filtered) && !batch_mode)
  591         clear_message();
  592 
  593     return TRUE;
  594 }
  595 
  596 
  597 /*
  598  * Returns number of first unread article
  599  */
  600 static t_artnum
  601 find_first_unread(
  602     struct t_group *group)
  603 {
  604     unsigned char *p;
  605     unsigned char *end = group->newsrc.xbitmap;
  606     t_artnum first = group->newsrc.xmin; /* initial value */
  607 
  608     if ((p = group->newsrc.xbitmap)) {
  609         end += group->newsrc.xbitlen / NBITS;
  610         for (; *p == '\0' && p < end; p++, first += NBITS)
  611             ;
  612     }
  613     return first;
  614 }
  615 
  616 
  617 /*
  618  * Open an article for reading just the header
  619  * 'next' is used/updated with the next article number
  620  * to optimise the number of 'HEAD' commands issued on
  621  * groups with holes.
  622  */
  623 static FILE *
  624 open_art_header(
  625     char *groupname,
  626     t_artnum art,
  627     t_artnum *next)
  628 {
  629     char buf[NNTP_STRLEN];
  630 #ifdef NNTP_ABLE
  631     FILE *fp;
  632     int i;
  633 
  634     if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
  635         /*
  636          * Don't bother requesting if we have not got there yet.
  637          * This is a big win if the group has got holes in it (ie. if 000's
  638          * of articles have expired between active files min & max values).
  639          */
  640         if (art < *next)
  641             return NULL;
  642 
  643         snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  644         if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  645             return fp;
  646 
  647         /*
  648          * TODO:
  649          * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
  650          * "503 Access denied" instead of 480 but NEXT still works,
  651          * so tin loops over all articles without getting useful data
  652          */
  653 
  654         /*
  655          * HEAD failed, try to find NEXT
  656          * Should return "223 artno message-id more text...."
  657          */
  658         i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
  659         switch (i) {
  660             case OK_NOTEXT:
  661                 *next = atoartnum(buf);     /* Set next art number */
  662                 break;
  663 
  664 #   ifndef BROKEN_LISTGROUP
  665             /*
  666              * might happen if LISTGROUP doesn't select group, but
  667              * we are not -DBROKEN_LISTGROUP
  668              */
  669             case ERR_NCING:
  670                 nntp_caps.broken_listgroup = TRUE;
  671                 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
  672                 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
  673                     return NULL;
  674                 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
  675                 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
  676                     return fp;
  677                 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
  678                     *next = atoartnum(buf);
  679                 break;
  680 #   endif /* !BROKEN_LISTGROUP */
  681 
  682             default:
  683                 /*
  684                  * TODO: abort loop over all arts on ERR_NONEXT
  685                  */
  686 #   ifndef BROKEN_LISTGROUP
  687                 /*
  688                  * to avoid out of sync responses
  689                  * (listgroup seems to work, but didn't select new group,
  690                  *  so xover seems to work but returns old data)
  691                  * we set listgroup_broken = TRUE; once we saw a
  692                  * ERR_NOARTIG / ERR_NONEXT or the like - even if
  693                  * ERR_NOARTIG may occur on servers where listgroup
  694                  * isn't broken...
  695                  */
  696                 nntp_caps.broken_listgroup = TRUE;
  697 #   endif /* !BROKEN_LISTGROUP */
  698                 break;
  699         }
  700 
  701         return NULL;
  702     }
  703 #endif /* NNTP_ABLE */
  704 
  705     snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
  706     return (fopen(buf, "r"));
  707 }
  708 
  709 
  710 /*
  711  * Called after XOVER/local/private overview databases have been loaded
  712  * Read and parse in headers for any arts not already found (usually
  713  * new articles that have not been indexed yet)
  714  * Any new articles that are added have ->thread set to ART_UNTHREADED
  715  * 'top' is the current highest artnum read
  716  *
  717  * Return values are:
  718  *   >0   Number of additional articles read in
  719  *    0   No additional (new) articles were found
  720  *   -1   user aborted during read
  721  */
  722 static int
  723 read_art_headers(
  724     struct t_group *group,
  725     int total,
  726     t_artnum top)
  727 {
  728     FILE *fp;
  729     char dir[PATH_LEN];
  730     char group_msg[LEN];
  731     int i;
  732     int modified = 0;
  733     t_artnum art;
  734     t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
  735     t_bool res;
  736 
  737     /*
  738      * Change to groups spooldir to optimize fopen()'s on local articles
  739      * NB open_art_header() requires this
  740      */
  741     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
  742         char buf[PATH_LEN];
  743 
  744         get_cwd(dir);
  745         make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
  746         if (chdir(buf) != 0) {
  747 #ifdef DEBUG
  748             if (debug & DEBUG_MISC)
  749                 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
  750 #endif /* DEBUG */
  751             return -1;
  752         }
  753     }
  754 
  755     snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
  756 
  757     for (i = 0; i < grpmenu.max; i++) { /* for each article number */
  758         art = base[i];
  759 
  760         /*
  761          * Skip articles that are below the low water mark or are
  762          * already present
  763          */
  764         if (valid_artnum(art) >= 0)
  765             continue;
  766         if (art <= top)
  767             continue;
  768 
  769         /*
  770          * Try and open the article
  771          */
  772         if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
  773             continue;
  774 
  775         /*
  776          * Add article to arts[]
  777          */
  778         if (top_art >= max_art)
  779             expand_art();
  780 
  781         set_article(&arts[top_art]);
  782         arts[top_art].artnum = art;
  783         arts[top_art].thread = ART_UNTHREADED;
  784 
  785         res = parse_headers(fp, &arts[top_art]);
  786 
  787         TIN_FCLOSE(fp);
  788         if (tin_errno) {
  789             modified = -1;
  790             break;
  791         }
  792 
  793         if (!res) {
  794 #ifdef DEBUG
  795             if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
  796                 char buf[PATH_LEN];
  797 
  798                 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
  799                 debug_print_file("ARTS", "read_art_headers() %s", buf);
  800             }
  801 #endif /* DEBUG */
  802             arts[top_art].artnum = T_ARTNUM_CONST(0);
  803             arts[top_art].date = (time_t) 0;
  804             FreeAndNull(arts[top_art].xref);
  805             FreeAndNull(arts[top_art].path);
  806             FreeAndNull(arts[top_art].refs);
  807             FreeAndNull(arts[top_art].msgid);
  808             if (arts[top_art].archive) {
  809                 FreeAndNull(arts[top_art].archive->partnum);
  810                 FreeAndNull(arts[top_art].archive);
  811             }
  812             arts[top_art].tagged = 0;
  813             arts[top_art].thread = ART_EXPIRED;
  814             arts[top_art].prev = ART_NORMAL;
  815             arts[top_art].status = ART_UNREAD;
  816             arts[top_art].killed = ART_NOTKILLED;
  817             arts[top_art].selected = FALSE;
  818             continue;
  819         }
  820 
  821         top = arts[top_art].artnum; /* used if arts are killed */
  822         top_art++;
  823 
  824         if (++modified % (MODULO_COUNT_NUM * 20) == 0)
  825             show_progress(group_msg, modified, total);
  826     }
  827 
  828     /*
  829      * Change back to previous dir before indexing started
  830      */
  831     if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
  832         chdir(dir);
  833 
  834     return modified;
  835 }
  836 
  837 
  838 /*
  839  * The algorithm is elegant, using the fact that identical Subject lines
  840  * are hashed to the same node in table[] (see hashstr.c)
  841  *
  842  * Mark i as being in j's thread list if
  843  * . The article is _not_ being ignored
  844  * . The article is not already threaded
  845  * . One of the following is true:
  846  *    1) The subject lines are the same
  847  *    2) Both are part of the same archive (name's match and arch bit set)
  848  * IMHO the tests for archive name are redundant and have been for years
  849  */
  850 static void
  851 thread_by_subject(
  852     void)
  853 {
  854     int i, j;
  855     struct t_hashnode *h;
  856 
  857     for_each_art(i) {
  858         if (IGNORE_ART_THREAD(i))
  859             continue;
  860 
  861         /*
  862          * Get the contents of the magic marker in the hashnode
  863          */
  864         h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
  865 
  866         j = h->aptr;
  867 
  868         if (j != -1 && j < i) {
  869 #if 1
  870             if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
  871 #else
  872             /* see also refs.c:collate_subjects() */
  873             if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
  874 #endif /* 1 */
  875             {
  876                 arts[j].thread = i;
  877                 arts[i].prev = j;
  878             }
  879         }
  880 
  881         /*
  882          * Update the magic marker with the highest numbered mesg in
  883          * arts[] that has been used in this thread so far
  884          */
  885         h->aptr = i;
  886     }
  887 
  888 #if 0
  889     fprintf(stderr, "Subj dump\n");
  890     fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
  891     for_each_art(i) {
  892         fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
  893             (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
  894             (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
  895             (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
  896             arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
  897     }
  898 #endif /* 0 */
  899 }
  900 
  901 
  902 /*
  903  * This Threading algorithm threads articles into 'buckets' where each bucket
  904  * contains all the articles which match the root article's subject line to
  905  * the configured percentage. Eg, if the root article had the subject "asdf"
  906  * and the match percentage was configured to be 75% then any article would
  907  * match if its subject was no different in more than a single character.
  908  */
  909 static void
  910 thread_by_percentage(
  911     unsigned int percentage)
  912 {
  913     int i, j, k;
  914     int root_num = 0; /* The index number of the root we are currently working on. */
  915     unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
  916     size_t slen;
  917 
  918     /* First we need to sort art[] to simplify and speed up the matching. */
  919     SortBy(subj_comp_asc);
  920 
  921     /*
  922      * Now we put all the articles which match enough into the thread. If
  923      * an article doesn't match enough we create a new thread and then add
  924      * to that and so on.
  925      */
  926     base[0] = 0;
  927     arts[0].prev = ART_NORMAL;
  928     for_each_art(i) {
  929         if (i == 0)
  930             continue;
  931 
  932         /* Check each character to see if it matched enough */
  933         k = 0;
  934         unmatched = 0;
  935         for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
  936             if (arts[base[root_num]].subject[j] != arts[i].subject[k])
  937                 unmatched++;
  938         }
  939 
  940         /*
  941          * By getting here we have a number of unmatched characters
  942          * between the two strings. We also have the length of the
  943          * strings available to us easily.
  944          * All we need to do is see if the match is good enough, but
  945          * we count differences in the length of the strings against
  946          * them matching.
  947          */
  948         if (!(slen = strlen(arts[base[root_num]].subject)))
  949             slen++;
  950         unmatched += slen - strlen(arts[i].subject);
  951         if (unmatched * 100 / slen > percentage) {
  952             /*
  953              * If there is less greater than percentage% different start a
  954              * new thread.
  955              */
  956             base[++root_num] = i;
  957             arts[i].prev = ART_NORMAL;
  958             continue;
  959         } else {
  960             /*
  961              * The subject lines match enough to consider them part of a single
  962              * thread, so add the current article to the thread.
  963              */
  964             if (arts[base[root_num]].thread < 0)
  965                 arts[base[root_num]].thread = i;
  966             arts[i].prev = i - 1;
  967             arts[i - 1].thread = i;
  968             continue;
  969         }
  970     }
  971 }
  972 
  973 
  974 /*
  975  * Parses a subject header of the type "multipart message subject (01/42)"
  976  * into a MultiPartInfo struct, or fails if the message subject isn't in the
  977  * right form.
  978  *
  979  * @return nonzero on success
  980  */
  981 int
  982 global_get_multipart_info(
  983     int aindex,
  984     MultiPartInfo *setme)
  985 {
  986     int i, j, offi, offj;
  987     MultiPartInfo setmei, setmej;
  988 
  989     i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
  990     j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
  991 
  992     /* Ok i hits first */
  993     if (offi > offj) {
  994         *setme = setmei;
  995         return i;
  996     }
  997 
  998     /* Its j or they are both the same (which must be zero!) so we don't care */
  999     *setme = setmej;
 1000     return j;
 1001 }
 1002 
 1003 
 1004 static int
 1005 global_look_for_multipart_info(
 1006     int aindex,
 1007     MultiPartInfo* setme,
 1008     char start,
 1009     char stop,
 1010     int *offset)
 1011 {
 1012     char *subj;
 1013     char *pch;
 1014     MultiPartInfo tmp;
 1015 
 1016     *offset = 0;
 1017 
 1018     /* entry assertions */
 1019     assert(0 <= aindex && aindex < top_art && "invalid index");
 1020     assert(setme != NULL && "setme must not be NULL");
 1021 
 1022     /* parse the message */
 1023     subj = arts[aindex].subject;
 1024     pch = strrchr(subj, start);
 1025     if (!pch || !isdigit((int) pch[1]))
 1026         return 0;
 1027 
 1028     tmp.arts_index = aindex;
 1029     tmp.subject_compare_len = pch - subj;
 1030     tmp.part_number = (int) strtol(pch + 1, &pch, 10);
 1031     if (*pch != '/' && *pch != '|')
 1032         return 0;
 1033 
 1034     if (!isdigit((int) pch[1]))
 1035         return 0;
 1036 
 1037     tmp.total = (int) strtol(pch + 1, &pch, 10);
 1038     if (*pch != stop)
 1039         return 0;
 1040 
 1041     /*
 1042      * skip "blah (00/102)" or "blah (103/102)" subjects
 1043      */
 1044     if (tmp.part_number < 1 || tmp.part_number > tmp.total)
 1045         return 0;
 1046 
 1047     tmp.subject = subj;
 1048     *setme = tmp;
 1049     *offset = pch - subj;
 1050     return 1;
 1051 }
 1052 
 1053 
 1054 t_bool
 1055 global_look_for_multipart(
 1056     int aindex,
 1057     char start,
 1058     char stop)
 1059 {
 1060     char *pch;
 1061 
 1062     pch = strrchr(arts[aindex].subject, start);
 1063     if (!pch || !isdigit((int) pch[1]))
 1064         return FALSE;
 1065 
 1066     strtol(pch + 1, &pch, 10);
 1067     if (*pch != '/' && *pch != '|')
 1068         return FALSE;
 1069 
 1070     if (!isdigit((int) pch[1]))
 1071         return FALSE;
 1072 
 1073     strtol(pch + 1, &pch, 10);
 1074     if (*pch != stop)
 1075         return FALSE;
 1076 
 1077     arts[aindex].multipart_subj = TRUE;
 1078     return TRUE;
 1079 }
 1080 
 1081 
 1082 /*
 1083  * Tries to find all the parts to the multipart message pointed to by
 1084  * aindex.
 1085  *
 1086  * @return on success, the number of parts found. On failure, zero if not
 1087  * a multipart or the negative value of the first missing part in case of
 1088  * tagging.
 1089  * @param aindex index pointing to one of the messages in a multipart
 1090  * message.
 1091  * @param malloc_and_setme_info on success, set to a malloced array the
 1092  * parts found.
 1093  */
 1094 int
 1095 global_get_multiparts(
 1096     int aindex,
 1097     MultiPartInfo **malloc_and_setme_info,
 1098     t_bool tagging)
 1099 {
 1100     int i, part_index, part_cnt = 0;
 1101     MultiPartInfo tmp, tmp2;
 1102     MultiPartInfo *info = NULL;
 1103 
 1104     /* entry assertions */
 1105     assert(0 <= aindex && aindex < top_art && "Invalid index");
 1106     assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
 1107 
 1108     /* make sure this is a multipart message... */
 1109     if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
 1110         return 0;
 1111 
 1112     /* make a temporary buffer to hold the multipart info... */
 1113     info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
 1114 
 1115     /* zero out part-number for the repost check below */
 1116     for (i = 0; i < tmp.total; ++i) {
 1117         info[i].total = tmp.total; /* Added this for thread_by_multipart */
 1118         info[i].part_number = -1;
 1119     }
 1120 
 1121     /* try to find all the multiparts... */
 1122     for (i = (tagging ? 0 : aindex); i < top_art; i++) {
 1123         if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
 1124             continue;
 1125 
 1126         if (!global_get_multipart_info(i, &tmp2))
 1127             continue;
 1128 
 1129         /* 'test (1/5)' is not the same as 'test (1/15)' */
 1130         if (tmp.total != tmp2.total)
 1131             continue;
 1132 
 1133         part_index = tmp2.part_number - 1;
 1134 
 1135         /* repost check: do we already have this part? */
 1136         if (info[part_index].part_number != -1) {
 1137             assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
 1138             continue;
 1139         }
 1140 
 1141         /* we have a match, hooray! */
 1142         info[part_index] = tmp2;
 1143 
 1144         arts[i].multipart_subj = FALSE;
 1145 
 1146         /* all parts found? */
 1147         if (++part_cnt == tmp.total)
 1148             break;
 1149     }
 1150 
 1151     /* see if we got them all. */
 1152     if (tagging) {
 1153         for (i = 0; i < tmp.total; ++i) {
 1154             if (info[i].part_number != i + 1) {
 1155                 free(info);
 1156                 return -(i + 1); /* missing part #(i+1) */
 1157             }
 1158         }
 1159     }
 1160 
 1161     /* looks like a success .. */
 1162     *malloc_and_setme_info = info;
 1163     return tmp.total;
 1164 }
 1165 
 1166 
 1167 /*
 1168  *  The algorithm uses the tag multipart searches to thread articles together.
 1169  */
 1170 static void
 1171 thread_by_multipart(
 1172     void)
 1173 {
 1174     int i, j, threadNum;
 1175     MultiPartInfo *minfo = NULL;
 1176 
 1177     for_each_art(i) {
 1178         if (!global_look_for_multipart(i, '[', ']'))
 1179             global_look_for_multipart(i, '(', ')');
 1180     }
 1181 
 1182     for_each_art(i) {
 1183         if (!arts[i].multipart_subj)
 1184             continue;
 1185 
 1186         if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
 1187             FreeAndNull(minfo);
 1188             arts[i].multipart_subj = FALSE;
 1189             continue;
 1190         }
 1191 
 1192         threadNum = -1;
 1193         for (j = minfo[0].total - 1; j >= 0; j--) {
 1194             if (minfo[j].part_number != -1) {
 1195                 if (threadNum != -1) {
 1196                     arts[minfo[j].arts_index].thread = threadNum;
 1197                     arts[threadNum].prev = minfo[j].arts_index;
 1198                 }
 1199                 threadNum = minfo[j].arts_index;
 1200             }
 1201         }
 1202         FreeAndNull(minfo);
 1203         arts[i].multipart_subj = FALSE;
 1204         if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
 1205             show_progress(_("Threading by multipart"), i, top_art);
 1206     }
 1207 }
 1208 
 1209 
 1210 /*
 1211  * Go through the articles in arts[] and create threads. There are
 1212  * 5 strategies currently defined :
 1213  *
 1214  *  THREAD_NONE     No threading
 1215  *  THREAD_SUBJ     Threads are created using like Subject lines
 1216  *  THREAD_REFS     Threads are created using the References headers
 1217  *  THREAD_BOTH     Threads created using References and then Subject
 1218  *  THREAD_MULTI    Threads created using Subject to search for Multiparts
 1219  *  THREAD_PERC     Threads based upon a char for char match of greater than x%
 1220  *
 1221  * .thread and .prev are used to hold the threading information, see tin.h for
 1222  * more information
 1223  * Only process valid (unexpired) articles we haven't visited yet
 1224  * (ie arts[].thread == ART_UNTHREADED)
 1225  *
 1226  * The rethread parameter is used to force the deletion of existing threading
 1227  * information before threading which happens anyway expect when using
 1228  * THREAD_NONE (I don't immediately see how this is useful)
 1229  */
 1230 /* TODO: rewrite that user can easily combine different 'threading'
 1231  *       methods, i.e:
 1232  *       - thread_by_multipart() + collate_subjects()
 1233  */
 1234 void
 1235 make_threads(
 1236     struct t_group *group,
 1237     t_bool rethread)
 1238 {
 1239     if (!cmd_line && !batch_mode) {
 1240         info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
 1241         my_flush();
 1242     }
 1243 
 1244 #ifdef DEBUG
 1245     if (debug & DEBUG_MISC)
 1246         error_message(2, "rethread=[%d]  thread_articles=[%d]  attr_thread_articles=[%d]",
 1247             rethread, tinrc.thread_articles, group->attribute->thread_articles);
 1248 #endif /* DEBUG */
 1249 
 1250     /*
 1251      * Sort all the articles using the preferred method
 1252      * When find_base() is called, the bases are created ordered
 1253      * on arts[] and so the base messages under all threading systems
 1254      * will be sorted in this way.
 1255      */
 1256     sort_arts(group->attribute->sort_article_type);
 1257 
 1258     /*
 1259      * Reset all the ptrs to articles following the above sort
 1260      */
 1261     clear_art_ptrs();
 1262 
 1263     /*
 1264      * The threading pointers need to be reset if re-threading
 1265      * If using ref threading, revector the links back to the articles
 1266      */
 1267     if (rethread || group->attribute->thread_articles) {
 1268         int i;
 1269 
 1270         for_each_art(i) {
 1271             if (arts[i].thread >= 0)
 1272                 arts[i].thread = ART_UNTHREADED;
 1273 
 1274             arts[i].prev = ART_NORMAL;
 1275 
 1276             /* Should never happen if tree is built properly */
 1277             if (arts[i].refptr == NULL) {
 1278 #ifdef DEBUG
 1279                 if (debug & DEBUG_REFS) {
 1280                     my_fprintf(stderr, "\nError  : art->refptr is NULL\n");
 1281                     my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
 1282                     my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
 1283                     my_fprintf(stderr, "From   : %s\n", arts[i].from);
 1284                     assert(arts[i].refptr != NULL);
 1285                 } else
 1286 #endif /* DEBUG */
 1287                     continue;
 1288             }
 1289             arts[i].refptr->article = i;
 1290         }
 1291     }
 1292 
 1293     /*
 1294      * Do the right thing according to the threading strategy
 1295      */
 1296     switch (group->attribute->thread_articles) {
 1297         case THREAD_NONE:
 1298             break;
 1299 
 1300         case THREAD_SUBJ:
 1301             thread_by_subject();
 1302             break;
 1303 
 1304         case THREAD_REFS:
 1305             thread_by_reference();
 1306             break;
 1307 
 1308         case THREAD_BOTH:
 1309             thread_by_reference();
 1310             collate_subjects();
 1311             break;
 1312 
 1313         case THREAD_MULTI:
 1314             thread_by_multipart();
 1315             break;
 1316 
 1317         case THREAD_PERC:
 1318             thread_by_percentage(100 - group->attribute->thread_perc);
 1319             break;
 1320 
 1321         default: /* not reached */
 1322             break;
 1323     }
 1324 
 1325     /*
 1326      * Rebuild base[]
 1327      */
 1328     find_base(group);
 1329 }
 1330 
 1331 
 1332 static t_compfunc
 1333 eval_sort_arts_func(
 1334     unsigned int sort_art_type)
 1335 {
 1336     switch (sort_art_type) {
 1337         case SORT_ARTICLES_BY_NOTHING:      /* don't sort at all */
 1338             return artnum_comp;
 1339 
 1340         case SORT_ARTICLES_BY_SUBJ_DESCEND:
 1341             return subj_comp_desc;
 1342 
 1343         case SORT_ARTICLES_BY_SUBJ_ASCEND:
 1344             return subj_comp_asc;
 1345 
 1346         case SORT_ARTICLES_BY_FROM_DESCEND:
 1347             return from_comp_desc;
 1348 
 1349         case SORT_ARTICLES_BY_FROM_ASCEND:
 1350             return from_comp_asc;
 1351 
 1352         case SORT_ARTICLES_BY_DATE_DESCEND:
 1353             return date_comp_desc;
 1354 
 1355         case SORT_ARTICLES_BY_DATE_ASCEND:
 1356             return date_comp_asc;
 1357 
 1358         case SORT_ARTICLES_BY_SCORE_DESCEND:
 1359             return score_comp_desc;
 1360 
 1361         case SORT_ARTICLES_BY_SCORE_ASCEND:
 1362             return score_comp_asc;
 1363 
 1364         case SORT_ARTICLES_BY_LINES_DESCEND:
 1365             return lines_comp_desc;
 1366 
 1367         case SORT_ARTICLES_BY_LINES_ASCEND:
 1368             return lines_comp_asc;
 1369 
 1370         default:
 1371             break;
 1372     }
 1373     return NULL;
 1374 }
 1375 
 1376 
 1377 void
 1378 sort_arts(
 1379     unsigned int sort_art_type)
 1380 {
 1381     t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
 1382 
 1383     if (comp_func)
 1384         SortBy(comp_func);
 1385 }
 1386 
 1387 
 1388 static void
 1389 sort_base(
 1390     unsigned int sort_threads_type)
 1391 {
 1392     switch (sort_threads_type) {
 1393         case SORT_THREADS_BY_SCORE_DESCEND:
 1394         case SORT_THREADS_BY_SCORE_ASCEND:
 1395             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
 1396             break;
 1397 
 1398         case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
 1399             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
 1400             break;
 1401 
 1402         case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
 1403             tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
 1404             break;
 1405     }
 1406 }
 1407 
 1408 
 1409 /*
 1410  * This is called to get header info for articles not already found in the
 1411  * overview files.
 1412  * Code reads (max_lineno) lines of article to catch headers like Archive-name:
 1413  * which are not normally included in XOVER or even the normal block of headers.
 1414  * How this is supposed to be useful when 99% of the time we'll have overview
 1415  * data I don't know...
 1416  * TODO: move Archive-name: parsing to article body parsing, remove the
 1417  * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
 1418  * TODO: currgrp->auto_save is set, otherwise it is redundant info
 1419  */
 1420 static t_bool
 1421 parse_headers(
 1422     FILE *fp,
 1423     struct t_article *h)
 1424 {
 1425     char art_from_addr[HEADER_LEN];
 1426     char art_full_name[HEADER_LEN];
 1427     char *s, *hdr, *ptr;
 1428     unsigned int lineno = 0;
 1429     unsigned int max_lineno = 25;
 1430     t_bool got_from, got_lines, got_received;
 1431 
 1432     got_from = got_lines = got_received = FALSE;
 1433 
 1434     while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
 1435         /*
 1436          * Look for the end of information which tin wants to get.
 1437          * Applies when reading local spool and via NNTP.
 1438          */
 1439 
 1440         /*
 1441          * as Archive-name: is placed in the body it's safe to exit
 1442          * the loop if it was found. Don't mix up with Archive: from
 1443          * RFC 5536.
 1444          */
 1445         if (lineno++ > max_lineno || h->archive)
 1446             break;
 1447 
 1448         unfold_header(ptr);
 1449         switch (my_toupper((unsigned char) *ptr)) {
 1450             case 'A':   /* Archive-name:  optional */
 1451                 /*
 1452                  * Archive-name: {name}/{part|patch}{number}
 1453                  * eg, acorn/faq/part01
 1454                  */
 1455                 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
 1456                     if ((s = strrchr(hdr, '/')) != NULL) {
 1457                         struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
 1458 
 1459                         if (STRNCASECMPEQ(s + 1, "part", 4)) {
 1460                             archptr->partnum = my_strdup(s + 5);
 1461                             archptr->ispart = TRUE;
 1462                         } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
 1463                             archptr->partnum = my_strdup(s + 6);
 1464                             archptr->ispart = FALSE;
 1465                         } else {        /* part or patch must be present */
 1466                             free(archptr);
 1467                             continue;
 1468                         }
 1469                         strtok(archptr->partnum, "\n");
 1470                         *s = '\0';
 1471                         archptr->name = hash_str(hdr);
 1472                         h->archive = archptr;
 1473                     }
 1474                 }
 1475                 break;
 1476 
 1477             case 'D':   /* Date:  mandatory */
 1478                 if (!h->date) {
 1479                     if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
 1480                         h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
 1481                 }
 1482                 break;
 1483 
 1484             case 'F':   /* From:  mandatory */
 1485                 if (!got_from) {
 1486                     if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
 1487                         h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
 1488                         h->from = hash_str(buffer_to_ascii(art_from_addr));
 1489                         if (*art_full_name)
 1490                             h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 1491                         got_from = TRUE;
 1492                     }
 1493                 }
 1494                 break;
 1495 
 1496             case 'L':   /* Lines:  optional */
 1497                 if (!got_lines) {
 1498                     if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
 1499                         h->line_count = atoi(hdr);
 1500                         got_lines = TRUE;
 1501                     }
 1502                 }
 1503                 break;
 1504 
 1505             case 'M':   /* Message-ID:  mandatory */
 1506                 if (!h->msgid) {
 1507                     if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
 1508                         h->msgid = my_strdup(hdr);
 1509                 }
 1510                 break;
 1511 
 1512             /* for Path:-filter when reading from local spool */
 1513             case 'P':   /* Path: */
 1514                 if (!h->path) {
 1515                     if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
 1516                         h->path = my_strdup(hdr);
 1517                 }
 1518                 break;
 1519 
 1520             case 'R':   /* References:  optional */
 1521                 if (!h->refs) {
 1522                     if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
 1523                         h->refs = my_strdup(hdr);
 1524                 }
 1525 
 1526                 /* Received:  If found it's probably a mail article */
 1527                 if (!got_received) {
 1528                     if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
 1529                         max_lineno <<= 1;       /* double the max number of line to read for mails */
 1530                         got_received = TRUE;
 1531                     }
 1532                 }
 1533                 break;
 1534 
 1535             case 'S':   /* Subject:  mandatory */
 1536                 if (!h->subject) {
 1537                     if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
 1538 #ifdef HAVE_UNICODE_NORMALIZATION
 1539                         if (IS_LOCAL_CHARSET("UTF-8"))
 1540                             s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1541                         else
 1542 #endif /* HAVE_UNICODE_NORMALIZATION */
 1543                             s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
 1544 
 1545                         h->subject = hash_str(s);
 1546                         free(s);
 1547                     }
 1548                 }
 1549                 break;
 1550 
 1551             case 'X':   /* Xref:  optional */
 1552                 if (!h->xref) {
 1553                     if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
 1554                         h->xref = my_strdup(hdr);
 1555                 }
 1556                 break;
 1557 
 1558             default:
 1559                 break;
 1560         } /* switch */
 1561 
 1562     } /* while */
 1563 
 1564 #ifdef NNTP_ABLE
 1565     if (ptr)
 1566         drain_buffer(fp);
 1567 #endif /* NNTP_ABLE */
 1568 
 1569     if (tin_errno)
 1570         return FALSE;
 1571 
 1572     /*
 1573      * The son of RFC 1036 states that the following hdrs are mandatory. It
 1574      * also states that Subject, Newsgroups and Path are too. Ho hum.
 1575      *
 1576      * What about reading mail from local spool via ~/.tin/active.mail,
 1577      * they might not have a Message-ID but got_received is very likely to
 1578      * be true.
 1579      */
 1580     if (got_from && h->date && h->msgid) {
 1581         if (!h->subject)
 1582             h->subject = hash_str("<No subject>");
 1583 
 1584 #ifdef DEBUG
 1585         if (debug & DEBUG_FILTER)
 1586             debug_print_header(h);
 1587 #endif /* DEBUG */
 1588         return TRUE;
 1589     }
 1590 
 1591     return FALSE;
 1592 }
 1593 
 1594 #ifdef NNTP_ABLE
 1595 /*
 1596  * Loop over arts[] and find ranges without Path: header
 1597  * If there are any try to optimize the ranges regarding traffic consumption
 1598  * Start optimization if at least MIN_CNT ranges exist
 1599  * If there are more than MAX_CNT ranges after optimization, fetch all in one
 1600  * big range
 1601  */
 1602 #define MIN_CNT 10
 1603 #define MAX_CNT 50
 1604 static struct t_article_range *
 1605 build_range_list(
 1606     t_artnum min,
 1607     t_artnum max,
 1608     int *range_cnt)
 1609 {
 1610     int i, gap_cnt = 0;
 1611     struct t_article_range *res = NULL, *gap_list, *curr, *from;
 1612     t_artnum new_end;
 1613 
 1614     new_end = T_ARTNUM_CONST(0);
 1615     gap_list = my_malloc(sizeof(struct t_article_range));
 1616     curr = gap_list;
 1617     curr->start = min;
 1618     curr->end = max;
 1619     curr->cnt = T_ARTNUM_CONST(0);
 1620     curr->next = NULL;
 1621 
 1622     for_each_art(i) {
 1623         if (arts[i].artnum < min)
 1624             continue;
 1625         if (arts[i].artnum > max)
 1626             break;
 1627         if (arts[i].path) {
 1628             for (; i < top_art && arts[i].path; i++)
 1629                 ;
 1630             /*
 1631              * the current art has no path -> we use this one
 1632              * if we reached top_art all arts have path
 1633              * so we use max
 1634              */
 1635             curr->start = i == top_art ? max : arts[i--].artnum;
 1636         } else {
 1637             for (; i < top_art && !arts[i].path; i++)
 1638                 ;
 1639             /* the current art has path -> we use the last one */
 1640             new_end = curr->end = arts[--i].artnum;
 1641         }
 1642         if (new_end) {
 1643             curr->cnt = curr->end - curr->start + 1;
 1644             curr->next = my_malloc(sizeof(struct t_article_range));
 1645             curr = curr->next;
 1646             curr->start = new_end;
 1647             curr->end = max;
 1648             curr->cnt = T_ARTNUM_CONST(0);
 1649             curr->next = NULL;
 1650             new_end = T_ARTNUM_CONST(0);
 1651         }
 1652     }
 1653 
 1654     curr = gap_list;
 1655     while (curr && curr->cnt) {
 1656         ++gap_cnt;
 1657 #   ifdef DEBUG
 1658         if ((debug & DEBUG_NNTP) && verbose > 1)
 1659             debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1660 #   endif /* DEBUG */
 1661         curr = curr->next;
 1662     }
 1663 
 1664     /*
 1665      * Optimize only if there are at least MIN_CNT ranges
 1666      */
 1667     if (gap_cnt >= MIN_CNT) {
 1668         res = my_malloc(sizeof(struct t_article_range));
 1669         res->start = T_ARTNUM_CONST(0);
 1670         res->end = T_ARTNUM_CONST(0);
 1671         res->cnt = T_ARTNUM_CONST(0);
 1672         res->next = NULL;
 1673 
 1674         from = gap_list;
 1675         curr = res;
 1676         while (from) {
 1677             curr->start = from->start;
 1678             curr->end = from->end;
 1679             curr->cnt = from->cnt;
 1680             if ((from = from->next)) {
 1681                 /*
 1682                  * If the next range is grater then the gap between the current
 1683                  * one and the next one we build a new range including the
 1684                  * current one, the next one and the gap between
 1685                  */
 1686                 while (from && from->cnt >= from->start - curr->end - 1) {
 1687                     curr->end = from->end;
 1688                     from = from->next;
 1689                 }
 1690                 curr->cnt = curr->end - curr->start + 1;
 1691                 curr->next = my_malloc(sizeof(struct t_article_range));
 1692                 curr = curr->next;
 1693                 curr->start = T_ARTNUM_CONST(0);
 1694                 curr->end = T_ARTNUM_CONST(0);
 1695                 curr->cnt = T_ARTNUM_CONST(0);
 1696                 curr->next = NULL;
 1697             }
 1698         }
 1699     }
 1700 
 1701     /*
 1702      * If there are less then MIN_CNT ranges
 1703      * no res is build -> return the original list
 1704      */
 1705     if (res) {
 1706         while (gap_list) {
 1707             curr = gap_list;
 1708             gap_list = curr->next;
 1709             free(curr);
 1710         }
 1711     } else
 1712         res = gap_list;
 1713 
 1714     curr = res;
 1715     gap_cnt = 0;
 1716     while (curr && curr->cnt) {
 1717         ++gap_cnt;
 1718 #   ifdef DEBUG
 1719         if ((debug & DEBUG_NNTP) && verbose > 1)
 1720             debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
 1721 #   endif /* DEBUG */
 1722         curr = curr->next;
 1723     }
 1724 
 1725     if (gap_cnt >= MAX_CNT) {
 1726         curr = res;
 1727         while (curr->next && curr->next->cnt) {
 1728             res->end = curr->next->end;
 1729             curr->next->cnt = 0;
 1730             curr = curr->next;
 1731         }
 1732         res->cnt = res->end - res->start + 1;
 1733         gap_cnt = 1;
 1734 #   ifdef DEBUG
 1735         if ((debug & DEBUG_NNTP) && verbose > 1)
 1736             debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
 1737 #   endif /* DEBUG */
 1738     }
 1739     *range_cnt = gap_cnt;
 1740 
 1741     return res;
 1742 }
 1743 
 1744 
 1745 /*
 1746  * Fetch the Path header in case we want to filter on that in the given group
 1747  *
 1748  * Try [X]HDR first, then XPAT
 1749  */
 1750 static t_bool
 1751 get_path_header(
 1752     int cur,
 1753     int cnt,
 1754     struct t_group *group,
 1755     t_artnum min,
 1756     t_artnum max)
 1757 {
 1758     FILE *fp = NULL;
 1759     char *prep_msg;
 1760     char *buf, *ptr;
 1761     char cmd[NNTP_STRLEN];
 1762     t_artnum artnum, i;
 1763     t_bool found = FALSE;
 1764     static t_bool supported = TRUE; /* assume HDR || XPAT works */
 1765 
 1766     if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
 1767         return FALSE;
 1768 
 1769 #   ifdef DEBUG
 1770     if ((debug & DEBUG_NNTP) && verbose > 1)
 1771         debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
 1772 #   endif /* DEBUG */
 1773 
 1774     if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
 1775         int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
 1776         switch (j) {
 1777             case 215:
 1778                 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 1779 #   ifdef DEBUG
 1780                     if (debug & DEBUG_NNTP)
 1781                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 1782 #   endif /* DEBUG */
 1783                     nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 1784                     strcat(nntp_caps.headers_range, ptr);
 1785                     strcat(nntp_caps.headers_range, "\n");
 1786                 }
 1787                 break;
 1788 
 1789             default:
 1790                 break;
 1791         }
 1792     }
 1793 
 1794     /* does HDR return Path? */
 1795     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
 1796         do {
 1797             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
 1798                 found = TRUE;
 1799         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 1800     }
 1801 
 1802     if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
 1803         if (min == max)
 1804             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
 1805         else
 1806             snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
 1807         fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
 1808         if (!nntp_caps.hdr && fp)
 1809             nntp_caps.hdr = TRUE;
 1810     } else if (nntp_caps.xpat) {
 1811         if (min == max)
 1812             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
 1813         else
 1814             snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
 1815         fp = nntp_command(cmd, OK_XPAT, NULL, 0);
 1816     }
 1817 
 1818     if (fp) {
 1819         int j = 0;
 1820 
 1821         prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
 1822         while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
 1823 #   ifdef DEBUG
 1824             if ((debug & DEBUG_NNTP) && verbose)
 1825                 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1826 #   endif /* DEBUG */
 1827             if ((ptr = tin_strtok(buf, " ")) == NULL)
 1828                 continue;
 1829             artnum = atoartnum(ptr);
 1830             if ((ptr = tin_strtok(NULL, " ")) == NULL)
 1831                 continue;
 1832             for (i = j; i < top_art; i++) {
 1833                 if (arts[i].artnum == artnum) {
 1834                     FreeIfNeeded(arts[i].path);
 1835                     arts[i].path = my_strdup(ptr);
 1836                     j = i;
 1837                     break;
 1838                 }
 1839             }
 1840             if (++artnum % MODULO_COUNT_NUM == 0)
 1841                 show_progress(prep_msg, artnum - min, max - min);
 1842         }
 1843         free(prep_msg);
 1844         return supported;
 1845     }
 1846 
 1847     /* !fp */
 1848     supported = FALSE;
 1849     if (nntp_caps.xpat)
 1850         nntp_caps.xpat = FALSE;
 1851     /* as nntp_caps.hdr may work with other headers we don't disable it */
 1852 
 1853 #   ifdef DEBUG
 1854     if ((debug & DEBUG_NNTP) && verbose > 1)
 1855         debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
 1856 #   endif /* DEBUG */
 1857     return supported;
 1858 }
 1859 #endif /* NNTP_ABLE */
 1860 
 1861 
 1862 /*
 1863  * Read in an overview index file. Fields are separated by TAB.
 1864  * return the number of expired articles encountered or -1 if the user aborted
 1865  * the read
 1866  * 'top' is set to the highest artnum read
 1867  * If 'local' is set then always open local overview cache in preference to
 1868  * using NNTP XOVER
 1869  *
 1870  * Format (mandatory as far as line count [RFC2980]):
 1871  *  1. article number (ie. 183)                [mandatory]
 1872  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 1873  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 1874  *  4. Date: line     (rfc822 format)          [mandatory]
 1875  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 1876  *  6. References:    (ie. <message-id> ....)  [optional]
 1877  *  7. Byte count     (Skipped - not used)     [mandatory]
 1878  *  8. Line count     (ie. 23)                 [mandatory]
 1879  *  9. Xref: line     (ie. alt.test:389)       [optional]
 1880  */
 1881 static int
 1882 read_overview(
 1883     struct t_group *group,
 1884     t_artnum min,
 1885     t_artnum max,
 1886     t_artnum *top,
 1887     t_bool local,
 1888     t_bool *rebuild_cache)
 1889 {
 1890     FILE *fp;
 1891     char *ptr;
 1892     char *q;
 1893     char *buf;
 1894     char *group_msg;
 1895     char art_full_name[HEADER_LEN];
 1896     char art_from_addr[HEADER_LEN];
 1897     unsigned int count;
 1898     int expired = 0;
 1899     t_artnum artnum;
 1900     t_bool path_found = FALSE, path_in_ofmt = FALSE;
 1901     struct t_article *art;
 1902     size_t over_fields = 1;
 1903 
 1904     /*
 1905      * open the overview file (whether it be local or via nntp)
 1906      */
 1907     if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
 1908         return expired;
 1909 
 1910     if (group->xmax > max)
 1911         group->xmax = max;
 1912 
 1913     group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
 1914 
 1915     /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
 1916     if (ofmt) {
 1917         for (; ofmt[over_fields].name; over_fields++) {
 1918             if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
 1919                 path_in_ofmt = TRUE;
 1920         }
 1921     }
 1922 
 1923     if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
 1924         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
 1925         ofmt[0].type = OVER_T_INT;
 1926         ofmt[0].name = my_strdup("Artnum:");
 1927         ofmt[1].type = OVER_T_STRING;
 1928         ofmt[1].name = my_strdup("Subject:");
 1929         ofmt[2].type = OVER_T_STRING;
 1930         ofmt[2].name = my_strdup("From:");
 1931         ofmt[3].type = OVER_T_STRING;
 1932         ofmt[3].name = my_strdup("Date:");
 1933         ofmt[4].type = OVER_T_STRING;
 1934         ofmt[4].name = my_strdup("Message-ID:");
 1935         ofmt[5].type = OVER_T_STRING;
 1936         ofmt[5].name = my_strdup("References:");
 1937         ofmt[6].type = OVER_T_INT;
 1938         ofmt[6].name = my_strdup("Bytes:");
 1939         ofmt[7].type = OVER_T_INT;
 1940         ofmt[7].name = my_strdup("Lines:");
 1941         ofmt[8].type = OVER_T_ERROR;
 1942         ofmt[8].name = NULL;
 1943         over_fields = 7;
 1944     }
 1945 
 1946     while ((buf = tin_fgets(fp, FALSE)) != NULL) {
 1947 #ifdef DEBUG
 1948         if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
 1949             debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
 1950 #endif /* DEBUG */
 1951 
 1952         if (need_resize) {
 1953             handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
 1954             need_resize = cNo;
 1955         }
 1956 
 1957         /*
 1958          * Read artnum
 1959          */
 1960         if ((ptr = tin_strtok(buf, "\t")) == NULL)
 1961             continue;
 1962 
 1963         /*
 1964          * read the article number, guaranteed to be the first field
 1965          */
 1966         artnum = atoartnum(ptr);
 1967 
 1968         /*
 1969          * artnum field invalid/corrupt or is 1st line of local cached overview
 1970          * (group name)
 1971          */
 1972         if (artnum <= 0)
 1973             continue;
 1974 
 1975         /*
 1976          * skip artnums below the given minimum (getart_limit)
 1977          */
 1978         if (artnum < min)
 1979             continue;
 1980 
 1981         /*
 1982          * Check to make sure article in nov file has not expired in group
 1983          */
 1984         if (artnum < group->xmin) {
 1985             expired++;
 1986             continue;
 1987         }
 1988 
 1989         /*
 1990          * artnum in overview data higher than groups high mark
 1991          *
 1992          * TODO: - warn user about broken overviews?
 1993          *       - try to parse the Xref:-line to get the correct artnum
 1994          *       - see also parse_unread_arts()
 1995          */
 1996         if (artnum > group->xmax)
 1997             continue;
 1998 
 1999         if (top_art >= max_art)
 2000             expand_art();
 2001 
 2002         art = &arts[top_art];
 2003         set_article(art);
 2004         art->artnum = *top = artnum;
 2005 
 2006         /*
 2007          * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
 2008          *       to check for additions like we do with xref_supported
 2009          */
 2010         for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
 2011             /* skip unexpected tailing fields */
 2012             if (count > over_fields) {
 2013 #ifdef DEBUG
 2014                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2015                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
 2016 #endif /* DEBUG */
 2017 
 2018                 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
 2019                 if (count == over_fields + 1) {
 2020                     if (!strncasecmp(ptr, "Xref: ", 6)) {
 2021 #ifdef DEBUG
 2022                         if ((debug & DEBUG_NNTP) && verbose > 1)
 2023                             debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
 2024 #endif /* DEBUG */
 2025                         over_fields++;
 2026                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 2027                         ofmt[over_fields].type = OVER_T_FSTRING;
 2028                         ofmt[over_fields].name = my_strdup("Xref:");
 2029                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 2030                         ofmt[over_fields + 1].name = NULL;
 2031                         xref_supported = TRUE;
 2032                     } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
 2033 #ifdef DEBUG
 2034                         if ((debug & DEBUG_NNTP) && verbose > 1)
 2035                             debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
 2036 #endif /* DEBUG */
 2037                         over_fields++;
 2038                         ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
 2039                         ofmt[over_fields].type = OVER_T_FSTRING;
 2040                         ofmt[over_fields].name = my_strdup("Path:");
 2041                         ofmt[over_fields + 1].type = OVER_T_ERROR;
 2042                         ofmt[over_fields + 1].name = NULL;
 2043                         xref_supported = TRUE;
 2044                     } else
 2045                         continue;
 2046                 } else
 2047                     continue;
 2048             }
 2049 
 2050             /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
 2051             if (expensive_over_parse) { /* strange order */
 2052                 /* mandatory fields */
 2053                 if (ofmt[count].type == OVER_T_STRING) {
 2054                     if (!strcasecmp(ofmt[count].name, "Subject:")) {
 2055                         if (*ptr) {
 2056 #ifdef HAVE_UNICODE_NORMALIZATION
 2057                             if (IS_LOCAL_CHARSET("UTF-8"))
 2058                                 q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2059                             else
 2060 #endif /* HAVE_UNICODE_NORMALIZATION */
 2061                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2062 
 2063                             art->subject = hash_str(q);
 2064                             free(q);
 2065                         } else {
 2066                             art->subject = hash_str("");
 2067 #ifdef DEBUG
 2068                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2069                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2070 #endif /* DEBUG */
 2071                         }
 2072                         continue;
 2073                     }
 2074 
 2075                     if (!strcasecmp(ofmt[count].name, "From:")) {
 2076                         if (*ptr) {
 2077                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2078                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2079                             if (*art_full_name)
 2080                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2081                         } else {
 2082                             art->from = hash_str("");
 2083 #ifdef DEBUG
 2084                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2085                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2086 #endif /* DEBUG */
 2087                         }
 2088                         continue;
 2089                     }
 2090 
 2091                     if (!strcasecmp(ofmt[count].name, "Date:")) {
 2092                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2093 #ifdef DEBUG
 2094                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2095                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2096 #endif /* DEBUG */
 2097                         continue;
 2098                     }
 2099 
 2100                     if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
 2101                         if (*ptr) {
 2102                             FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
 2103                             art->msgid = my_strdup(ptr);
 2104                         } else {
 2105                             art->msgid = NULL;
 2106 #ifdef DEBUG
 2107                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2108                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2109 #endif /* DEBUG */
 2110                         }
 2111                         continue;
 2112                     }
 2113 
 2114                     if (!strcasecmp(ofmt[count].name, "References:")) {
 2115                         if (*ptr) {
 2116                             FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
 2117                             art->refs = my_strdup(ptr);
 2118                         } else
 2119                             art->refs = NULL;
 2120                         continue;
 2121                     }
 2122 
 2123                     /*
 2124                      * non std. fields when doing
 2125                      * expensive overview parsing (very
 2126                      * rare, just happens if RFC 3977
 2127                      * 8.4.2 is violated) go here
 2128                      */
 2129                     /* for Path:-filter */
 2130                     if (!strcasecmp(ofmt[count].name, "Path:")) {
 2131                         if (!path_found)
 2132                             path_found = TRUE;
 2133                         if (*ptr) {
 2134                             FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
 2135                             art->path = my_strdup(ptr);
 2136                         } else
 2137                             art->path = NULL;
 2138                         continue;
 2139                     }
 2140                 }
 2141                 /* metadata fields */
 2142                 if (ofmt[count].type == OVER_T_INT) {
 2143                     if (!strcasecmp(ofmt[count].name, "Bytes:")) {
 2144                         if (*ptr) {
 2145 #ifdef DEBUG
 2146                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2147                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2148 #endif /* DEBUG */
 2149                         }
 2150                         continue;
 2151                     }
 2152 
 2153                     if (!strcasecmp(ofmt[count].name, "Lines:")) {
 2154                         if (*ptr) {
 2155                             if (isdigit((unsigned char) *ptr))
 2156                                 art->line_count = atoi(ptr);
 2157                             else {
 2158                                 art->line_count = 0;
 2159 #ifdef DEBUG
 2160                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2161                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2162 #endif /* DEBUG */
 2163                             }
 2164                         } else
 2165                             art->line_count = 0;
 2166                         continue;
 2167                     }
 2168                 }
 2169             } else { /* first 7 fields are in RFC 3977 order */
 2170                 switch (count) {
 2171                     case 1: /* Subject: */
 2172                         if (*ptr) {
 2173 #ifdef HAVE_UNICODE_NORMALIZATION
 2174                             if (IS_LOCAL_CHARSET("UTF-8"))
 2175                                 q =  normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2176                             else
 2177 #endif /* HAVE_UNICODE_NORMALIZATION */
 2178                                 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
 2179 
 2180                             art->subject = hash_str(q);
 2181                             free(q);
 2182                         } else {
 2183                             art->subject = hash_str("");
 2184 #ifdef DEBUG
 2185                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2186                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2187 #endif /* DEBUG */
 2188                         }
 2189                         break;
 2190 
 2191                     case 2: /* From: */
 2192                         if (*ptr) {
 2193                             art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
 2194                             art->from = hash_str(buffer_to_ascii(art_from_addr));
 2195                             if (*art_full_name)
 2196                                 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
 2197                         } else {
 2198                             art->from = hash_str("");
 2199 #ifdef DEBUG
 2200                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2201                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2202 #endif /* DEBUG */
 2203                         }
 2204                         break;
 2205 
 2206                     case 3: /* Date: */
 2207                         art->date = parsedate(ptr, (TIMEINFO *) 0);
 2208 #ifdef DEBUG
 2209                         if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
 2210                             debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2211 #endif /* DEBUG */
 2212                         break;
 2213 
 2214                     case 4: /* Message-ID: */
 2215                         if (*ptr)
 2216                             art->msgid = my_strdup(ptr);
 2217                         else {
 2218                             art->msgid = NULL;
 2219 #ifdef DEBUG
 2220                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2221                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
 2222 #endif /* DEBUG */
 2223                         }
 2224                         break;
 2225 
 2226                     case 5: /* References: */
 2227                         if (*ptr)
 2228                             art->refs = my_strdup(ptr);
 2229                         else
 2230                             art->refs = NULL;
 2231                         break;
 2232 
 2233                     case 6: /* :bytes || Bytes: */
 2234                         if (*ptr) {
 2235 #ifdef DEBUG
 2236                             if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
 2237                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2238 #endif /* DEBUG */
 2239                         }
 2240                         break;
 2241 
 2242                     case 7: /* :lines || Lines: */
 2243                         if (*ptr) {
 2244                             if (isdigit((unsigned char) *ptr))
 2245                                 art->line_count = atoi(ptr);
 2246                             else {
 2247                                 art->line_count = 0;
 2248 #ifdef DEBUG
 2249                                 if ((debug & DEBUG_NNTP) && verbose > 1)
 2250                                     debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
 2251 #endif /* DEBUG */
 2252                             }
 2253                         } else
 2254                             art->line_count = 0;
 2255                         break;
 2256 
 2257                     default:
 2258                         break;
 2259                 }
 2260             }
 2261 
 2262             /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
 2263             if (ofmt[count].type == OVER_T_FSTRING) {
 2264                 if (*ptr) {
 2265                     if (!strcasecmp(ofmt[count].name, "Xref:")) {
 2266                         if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
 2267                             FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
 2268                             art->xref = my_strdup(q);
 2269                         }
 2270 #ifdef DEBUG
 2271                         else {
 2272                             if ((debug & DEBUG_NNTP) && verbose > 1)
 2273                                 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
 2274                         }
 2275 #endif /* DEBUG */
 2276                         continue;
 2277                     }
 2278                     /*
 2279                      * handling of addition overview fields
 2280                      * goes here
 2281                      */
 2282 #ifdef DEBUG
 2283                     if ((debug & DEBUG_NNTP) && verbose > 1)
 2284                         debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
 2285 #endif /* DEBUG */
 2286                     /* if we're lucky we've Path in NOV */
 2287                     /*
 2288                      * if reading locally cached overview data try
 2289                      * path regardless of the server OVERVIEW.FMT
 2290                      */
 2291                     if (local || !strcasecmp(ofmt[count].name, "Path:")) {
 2292                         if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
 2293                             if (!path_found)
 2294                                 path_found = TRUE;
 2295                             FreeIfNeeded(art->path);
 2296                             art->path = my_strdup(q);
 2297 #ifdef DEBUG
 2298                             if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
 2299                                 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
 2300 #endif /* DEBUG */
 2301 
 2302                         }
 2303                         continue;
 2304                     }
 2305                 }
 2306                 continue;
 2307             }
 2308         }
 2309 
 2310         /*
 2311          * RFC says Message-ID is mandatory in newsgroups (but not in
 2312          * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
 2313          * threading in mailgroups
 2314          */
 2315         if (!art->msgid && group->type == GROUP_TYPE_NEWS)
 2316             continue;
 2317 
 2318         /* we might lose accuracy here, but that shouldn't hurt */
 2319         if (artnum % (MODULO_COUNT_NUM * 20) == 0)
 2320             show_progress(group_msg, artnum - min, max - min);
 2321 
 2322         top_art++;              /* Basically this statement commits the article */
 2323     }
 2324 
 2325     free(group_msg);
 2326     TIN_FCLOSE(fp);
 2327 
 2328     if (tin_errno)
 2329         return -1;
 2330 
 2331 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
 2332     if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
 2333         char cbuf[HEADER_LEN];
 2334         int i;
 2335         static t_bool found;
 2336         static t_bool first = TRUE;
 2337 
 2338         if (first) {
 2339             found = TRUE;
 2340             /*
 2341              * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
 2342              */
 2343             if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
 2344                 if (!*nntp_caps.headers_range) {
 2345                     i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
 2346 
 2347                     found = FALSE;
 2348                     switch (i) {
 2349                         case 215:
 2350                             while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
 2351 #   ifdef DEBUG
 2352                                 if (debug & DEBUG_NNTP)
 2353                                     debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2354 #   endif /* DEBUG */
 2355                                 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
 2356                                     found = TRUE;
 2357                                 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
 2358                                 strcat(nntp_caps.headers_range, ptr);
 2359                                 strcat(nntp_caps.headers_range, "\n");
 2360                             }
 2361                             break;
 2362 
 2363                         default:
 2364                             break;
 2365                     }
 2366                     first = FALSE;
 2367                 } else {
 2368                     found = FALSE;
 2369                     if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
 2370                         do {
 2371                             if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
 2372                                 found = TRUE;
 2373                         } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
 2374                     }
 2375                 }
 2376             }
 2377         }
 2378 
 2379         if (found) {
 2380             snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
 2381             group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
 2382             if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
 2383                 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
 2384 #   ifdef DEBUG
 2385                     if ((debug & DEBUG_NNTP) && verbose)
 2386                         debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
 2387 #   endif /* DEBUG */
 2388 
 2389                     artnum = atoartnum(ptr);
 2390                     if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
 2391                         continue;
 2392                     art = &arts[top_art];
 2393                     set_article(art);
 2394                     if (!art->xref && !strstr(ptr, "(none)")) {
 2395                         if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
 2396                             continue;
 2397                         ptr = q;
 2398                         while (*ptr && isspace((int) *ptr))
 2399                             ptr++;
 2400                         q = strchr(ptr, '\n');
 2401                         if (q)
 2402                             *q = '\0';
 2403                         art->xref = my_strdup(ptr);
 2404                     }
 2405                     /* we might lose accuracy here, but that shouldn't hurt */
 2406                     if (artnum % (MODULO_COUNT_NUM * 20) == 0)
 2407                         show_progress(group_msg, artnum - min, max - min);
 2408                 }
 2409             }
 2410             free(group_msg);
 2411         }
 2412     }
 2413 #endif /* NNTP_ABLE && XHDR_XREF */
 2414 
 2415     if (local) {
 2416 #ifdef NNTP_ABLE
 2417         if (filter_on_path(group)) {
 2418             struct t_article_range *ranges, *curr;
 2419             t_bool supported = TRUE;
 2420             int curr_range, range_cnt;
 2421 
 2422             /*
 2423              * Get the ranges without Path: header and try to fetch the
 2424              * headers
 2425              */
 2426             if ((ranges = build_range_list(min, *top, &range_cnt))) {
 2427                 curr = ranges;
 2428                 curr_range = 1;
 2429                 while (curr && supported) {
 2430                     if (curr->cnt)
 2431                         supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
 2432                     curr = curr->next;
 2433                 }
 2434                 if (!supported && path_in_ofmt) {
 2435                     /*
 2436                      * fetching Path: headers via [X]HDR or XPAT has failed
 2437                      * Path: is in the servers overview so let the next
 2438                      * read_overview() fetch them
 2439                      */
 2440                     free_art_array();
 2441                     free_msgids();
 2442                     top_art = 0;
 2443                     *top = T_ARTNUM_CONST(0);
 2444                     expired = 0;
 2445                 }
 2446                 *rebuild_cache = TRUE;
 2447                 while (ranges) {
 2448                     curr = ranges;
 2449                     ranges = curr->next;
 2450                     free(curr);
 2451                 }
 2452             }
 2453         }
 2454 #endif /* NNTP_ABLE */
 2455     } else
 2456         if (!path_found && filter_on_path(group)) {
 2457 #ifdef NNTP_ABLE
 2458             if (!get_path_header(1, 1, group, min, *top))
 2459 #endif /* NNTP_ABLE */
 2460                 wait_message(2, _(txt_cannot_filter_on_path));
 2461         }
 2462     return expired;
 2463 }
 2464 
 2465 
 2466 /*
 2467  * Write an Nov/Xover index file. Fields are separated by '\t'.
 2468  *
 2469  * Format:
 2470  *  1. article number (ie. 183)                [mandatory]
 2471  *  2. Subject: line  (ie. Which newsreader?)  [mandatory]
 2472  *  3. From: line     (ie. iain@ecrc.de)       [mandatory]
 2473  *  4. Date: line     (rfc822 format)          [mandatory]
 2474  *  5. MessageID:     (ie. <123@ether.net>)    [mandatory]
 2475  *  6. References:    (ie. <message-id> ....)  [optional]
 2476  *  7. Byte count     (Skipped - not used)     [mandatory]
 2477  *  8. Line count     (ie. 23)                 [mandatory]
 2478  *  9. Xref: line     (ie. alt.test:389)       [optional]
 2479  *
 2480  * TODO: as we don't use the original data, we currently can't store
 2481  *       the data (from/subject) in the original charset (we don't store
 2482  *       that info). this has the advantage that we can avoid raw 8bit data
 2483  *       in our overviews, but the disadvantage that we might store the data
 2484  *       with a wrong charset and thus lose information. a similar problem
 2485  *       exists with the data for the from:-line, we don't store it in the
 2486  *       original format, whenever our from-parser (partially) fails we'll
 2487  *       lose information in our overviews (but those couldn't be handled
 2488  *       by tin anyway, so this is not a real problem).
 2489  *       long-term solution: store the original data in the overview
 2490  *       (tin has to handle raw 8bit data and other ugly stuff in the
 2491  *       overviews anyway and thus we preserver as much info as possible)
 2492  *       this would require some changes in read_overview() and
 2493  *       parse_headers(): don't do the decoding/unfolding there, but in a
 2494  *       second pass right after write_overview(), or two additional fields
 2495  *       which hold the raw data for from/subject. the latter has the
 2496  *       disadvantage that it costs (much) more memory.
 2497  */
 2498 static void
 2499 write_overview(
 2500     struct t_group *group)
 2501 {
 2502     FILE *fp;
 2503     int i;
 2504     struct t_article *article;
 2505 #ifdef CHARSET_CONVERSION
 2506     int c = -1;
 2507 #endif /* CHARSET_CONVERSION */
 2508 
 2509     /*
 2510      * Can't write or caching is off or getart_limit is set
 2511      */
 2512     if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
 2513         return;
 2514 
 2515     if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
 2516         return;
 2517 
 2518     if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
 2519         SortBy(artnum_comp);
 2520 
 2521     /*
 2522      * Needed to preserve uniqueness in hashed private overview files
 2523      */
 2524     fprintf(fp, "%s\n", group->name);
 2525 
 2526 #ifdef CHARSET_CONVERSION
 2527     /* get undeclared_charset number if required */
 2528     if (group->attribute->undeclared_charset) {
 2529         for (i = 0; txt_mime_charsets[i] != NULL; i++) {
 2530             if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
 2531                 c = i;
 2532                 break;
 2533             }
 2534         }
 2535     }
 2536 #endif /* CHARSET_CONVERSION */
 2537 
 2538     if (verbose && batch_mode) /* -> lang.c */
 2539         wait_message(0, _("Writing %s\n"), group->name);
 2540 
 2541     for_each_art(i) {
 2542         char *p;
 2543         char *q, *ref;
 2544 
 2545         article = &arts[i];
 2546 
 2547         if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
 2548             ref = NULL;
 2549 
 2550             if (!group->attribute->post_8bit_header) { /* write encoded data */
 2551                 /*
 2552                  * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
 2553                  *       here and in print_from() in the CHARSET_CONVERSION case.
 2554                  *       note that this requires something like
 2555                  *          buffer_to_network(article->subject, "UTF-8");
 2556                  *       right before the rfc1522_encode() call.
 2557                  *
 2558                  *       if we would cache the original undecoded data, we could
 2559                  *       ignore stuff like this.
 2560                  */
 2561                 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
 2562                 /* as the subject might now be folded we have to unfold it */
 2563                 unfold_header(p);
 2564             } else { /* raw data */
 2565                 p = my_strdup(article->subject);
 2566 #ifdef CHARSET_CONVERSION
 2567                 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
 2568                     buffer_to_network(p, c);
 2569 #endif /* CHARSET_CONVERSION */
 2570             }
 2571 
 2572             /*
 2573              * replace any '\t's with ' ' in the references-data
 2574              *
 2575              * TODO: nntpext-draft might come up with a new scheme:
 2576              *       For all fields, the value is processed by first
 2577              *       removing all US-ASCII CRLF pairs and then replacing
 2578              *       each remaining US-ASCII NUL, TAB, CR, or LF character
 2579              *       with a single US-ASCII space (for example, CR LF LF TAB
 2580              *       will become two spaces).
 2581              */
 2582             if (article->refs) {
 2583                 ref = q = my_strdup(article->refs);
 2584                 while (*q) {
 2585                     if (*q == '\t')
 2586                         *q = ' ';
 2587                     q++;
 2588                 }
 2589             }
 2590 
 2591             fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
 2592                 article->artnum,
 2593                 p,
 2594 #ifdef CHARSET_CONVERSION
 2595                 print_from(group, article, c),
 2596 #else
 2597                 print_from(group, article, -1),
 2598 #endif /* CHARSET_CONVERSION */
 2599                 print_date(article->date),
 2600                 BlankIfNull(article->msgid),
 2601                 BlankIfNull(ref),
 2602                 0,  /* bytes */
 2603                 article->line_count);
 2604 
 2605             if (article->xref)
 2606                 fprintf(fp, "\tXref: %s", article->xref);
 2607 
 2608             if (article->path)
 2609                 fprintf(fp, "\tPath: %s", article->path);
 2610 
 2611             fprintf(fp, "\n");
 2612             free(p);
 2613             if (article->refs) {
 2614                 FreeIfNeeded(ref);
 2615                 q = NULL;
 2616             }
 2617         }
 2618         if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
 2619             show_progress(_("Writing overview cache..."), i, top_art);
 2620     }
 2621 #ifdef HAVE_FCHMOD
 2622     fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
 2623     /*
 2624      * TODO:
 2625      * add code for !HAVE_FCHMOD && HAVE_CHMOD
 2626      */
 2627 #endif /* HAVE_FCHMOD */
 2628     fclose(fp);
 2629 }
 2630 
 2631 
 2632 /*
 2633  * A complex little function to determine the correct overview index file
 2634  * according to 'mode' (read or write)
 2635  * NULL is returned if the current setup dictates otherwise
 2636  *
 2637  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
 2638  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
 2639  *
 2640  * Both of these are hashed
 2641  *
 2642  * GROUP_TYPE_NEWS index files are a little bit more complex
 2643  *
 2644  * When hashing the index filename will be in format number.number.
 2645  * Hashing the groupname gets a number. See if that #.1 file exists;
 2646  * if so, read first line. Is this the group we want? If no, try #.2.
 2647  * Repeat until no such file or we find an existing file that matches
 2648  * our group. Return pointer to path or NULL if not found.
 2649  */
 2650 static char *
 2651 find_nov_file(
 2652     struct t_group *group,
 2653     int mode)
 2654 {
 2655     FILE *fp;
 2656     const char *dir;
 2657     char buf[PATH_LEN];
 2658     int i;
 2659     struct stat sb;
 2660     unsigned long hash;
 2661     static char nov_file[PATH_LEN];
 2662     static t_bool once_only = FALSE;    /* Trap things that are done only 1 time */
 2663 
 2664     if (group == NULL || (mode != R_OK && mode != W_OK))
 2665         return NULL;
 2666 
 2667     switch (group->type) {
 2668         case GROUP_TYPE_MAIL:
 2669             dir = index_maildir;
 2670             break;
 2671 
 2672         case GROUP_TYPE_SAVE:
 2673             dir = index_savedir;
 2674             break;
 2675 
 2676         case GROUP_TYPE_NEWS:
 2677             /*
 2678              * nntp.caps.over_cmd is not an issue here, any gripes and warnings
 2679              * about [X]OVER are handled in nntp_open()
 2680              */
 2681 
 2682             /*
 2683              * When reading via NNTP, system wide overviews are irrelevant, of
 2684              * course, and the private overview filename will be the same for
 2685              * both reading and writing.
 2686              *
 2687              * When working locally, we only use a private cache for reading
 2688              * if requested and when system wide overviews don't already exist.
 2689              * When writing then only private overviews can be used since
 2690              * updating system wide overviews is not safe wrt locking etc.
 2691              *
 2692              * See if local overview file $SPOOLDIR/<groupname>/.overview exists
 2693              */
 2694 #ifndef NNTP_ONLY
 2695             if (!read_news_via_nntp) {
 2696                 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
 2697                 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
 2698                 if (access(nov_file, R_OK) == 0) {
 2699                     if (mode == R_OK)
 2700                         return nov_file;        /* Use system wide overviews */
 2701                     else
 2702                         return NULL;            /* Don't write cache in this case */
 2703                 }
 2704             }
 2705 #endif /* !NNTP_ONLY */
 2706 
 2707             /*
 2708              * We only get here when private overviews are going to be used
 2709              * Go no further if they are explicitly turned off
 2710              */
 2711             if (!tinrc.cache_overview_files)
 2712                 return NULL;
 2713 
 2714             /*
 2715              * Append -<nntpserver> to private cache dir
 2716              */
 2717             if (!once_only && nntp_server) {
 2718                 size_t sp, ln = strlen(index_newsdir);
 2719 
 2720                 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
 2721                     char *srv = my_strdup(nntp_server);
 2722 
 2723                     strcat(index_newsdir, "-");
 2724                     sp--;
 2725                     ln++;
 2726                     str_lwr(srv);
 2727                     my_strncpy(index_newsdir + ln, srv, sp);
 2728                     free(srv);
 2729                 }
 2730                 once_only = TRUE;
 2731             }
 2732 
 2733             /*
 2734              * Only try to set up the private cache when writing. If it
 2735              * doesn't exist yet, then ergo we can't read from it.
 2736              * The cache will be checked/created on every write; a previous
 2737              * bug report complained that this was not the case
 2738              */
 2739             if (stat(index_newsdir, &sb) == -1) {           /* Private cache doesn't exist */
 2740                 if (mode == R_OK)
 2741                     return NULL;
 2742                 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
 2743                     return NULL;
 2744             } else {
 2745                 if (!S_ISDIR(sb.st_mode))
 2746                     return NULL;
 2747             }
 2748 
 2749             /*
 2750              * Update the newsgroups cache to point to the new location
 2751              * now that we know it is valid
 2752              */
 2753             if (!once_only)
 2754                 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
 2755 
 2756             dir = index_newsdir;
 2757             break;
 2758 
 2759         default: /* not reached */
 2760             return NULL;
 2761     }
 2762 
 2763     /*
 2764      * We only get here if writing to a private overview.
 2765      * These always have hashed filenames.
 2766      * Try <hash>.<seqno> and check the group name tagline until
 2767      * matching index file is found. If not found return next unused
 2768      * filename
 2769      */
 2770     hash = hash_groupname(group->name);
 2771 
 2772     for (i = 1; ; i++) {
 2773         char *ptr;
 2774 
 2775         snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
 2776         joinpath(nov_file, sizeof(nov_file), dir, buf);
 2777 
 2778         if ((fp = fopen(nov_file, "r")) == NULL)
 2779             break;
 2780 
 2781         /*
 2782          * No group name header, so not a valid index file => overwrite it
 2783          */
 2784         if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
 2785             fclose(fp);
 2786             break;
 2787         }
 2788         fclose(fp);
 2789 
 2790         if ((ptr = strrchr(buf, '\n')) != NULL)
 2791             *ptr = '\0';
 2792 
 2793         if (strcmp(buf, group->name) == 0)
 2794             break;
 2795     }
 2796 
 2797     return nov_file;
 2798 }
 2799 
 2800 
 2801 /*
 2802  * Run the index file updater only for the groups we've loaded.
 2803  */
 2804 void
 2805 do_update(
 2806     t_bool catchup)
 2807 {
 2808     int i, j, k = 0;
 2809     time_t beg_epoch = 0;
 2810     struct t_article *art;
 2811     struct t_group *group;
 2812 
 2813     if (verbose)
 2814         (void) time(&beg_epoch);
 2815 
 2816     /*
 2817      * loop through groups and update any required index files
 2818      */
 2819     for (i = 0; i < selmenu.max; i++) {
 2820         group = &active[my_group[i]];
 2821         /*
 2822          * FIXME: workaround to get a valid CURR_GROUP
 2823          * it also points to the currently processed group so that
 2824          * the correct attributes are used
 2825          * The correct fix is to get rid of CURR_GROUP
 2826          */
 2827         selmenu.curr = i;
 2828 
 2829         if (group->bogus || !group->subscribed)
 2830             continue;
 2831 
 2832         if (!index_group(group)) {
 2833             for_each_art(j) {
 2834                 art = &arts[j];
 2835                 FreeAndNull(art->refs);
 2836                 FreeAndNull(art->msgid);
 2837             }
 2838             continue;
 2839         }
 2840 
 2841         k++;
 2842 
 2843         if (verbose) {
 2844             my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
 2845             my_flush();
 2846         }
 2847 
 2848         if (catchup) {
 2849             for_each_art(j)
 2850                 art_mark(group, &arts[j], ART_READ);
 2851         }
 2852     }
 2853 
 2854     if (verbose) {
 2855         wait_message(0, _(txt_catchup_update_info),
 2856             (catchup ? _(txt_caughtup) : _(txt_updated)), k,
 2857             PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
 2858     }
 2859 }
 2860 
 2861 
 2862 static int
 2863 artnum_comp(
 2864     t_comptype p1,
 2865     t_comptype p2)
 2866 {
 2867     const struct t_article *s1 = (const struct t_article *) p1;
 2868     const struct t_article *s2 = (const struct t_article *) p2;
 2869 
 2870     /*
 2871      * s1->artnum less than s2->artnum
 2872      */
 2873     if (s1->artnum < s2->artnum)
 2874         return -1;
 2875 
 2876     /*
 2877      * s1->artnum greater than s2->artnum
 2878      */
 2879     if (s1->artnum > s2->artnum)
 2880         return 1;
 2881 
 2882     return 0;
 2883 }
 2884 
 2885 
 2886 /*
 2887  * return result of strcmp (reversed for descending)
 2888  */
 2889 static int
 2890 subj_comp_asc(
 2891     t_comptype p1,
 2892     t_comptype p2)
 2893 {
 2894     int retval;
 2895     const struct t_article *s1 = (const struct t_article *) p1;
 2896     const struct t_article *s2 = (const struct t_article *) p2;
 2897 
 2898     if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
 2899         return retval;
 2900 
 2901     return s1->date - s2->date > 0 ? 1 : -1;
 2902 }
 2903 
 2904 
 2905 static int
 2906 subj_comp_desc(
 2907     t_comptype p1,
 2908     t_comptype p2)
 2909 {
 2910     int retval;
 2911     const struct t_article *s1 = (const struct t_article *) p1;
 2912     const struct t_article *s2 = (const struct t_article *) p2;
 2913 
 2914     if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
 2915         return retval;
 2916 
 2917     return s1->date - s2->date > 0 ? 1 : -1;
 2918 }
 2919 
 2920 
 2921 /*
 2922  * return result of strcmp (reversed for descending)
 2923  */
 2924 static int
 2925 from_comp_asc(
 2926     t_comptype p1,
 2927     t_comptype p2)
 2928 {
 2929     int retval;
 2930     const struct t_article *s1 = (const struct t_article *) p1;
 2931     const struct t_article *s2 = (const struct t_article *) p2;
 2932 
 2933     if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
 2934         return retval;
 2935 
 2936     return s1->date - s2->date > 0 ? 1 : -1;
 2937 }
 2938 
 2939 
 2940 static int
 2941 from_comp_desc(
 2942     t_comptype p1,
 2943     t_comptype p2)
 2944 {
 2945     int retval;
 2946     const struct t_article *s1 = (const struct t_article *) p1;
 2947     const struct t_article *s2 = (const struct t_article *) p2;
 2948 
 2949     if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
 2950         return retval;
 2951 
 2952     return s1->date - s2->date > 0 ? 1 : -1;
 2953 }
 2954 
 2955 
 2956 /*
 2957  * Works like strcmp() for comparing time_t type values
 2958  * Return codes:
 2959  *  -1:     If p1 is before p2
 2960  *   0:     If they are the same time
 2961  *   1:     If p1 is after p2
 2962  * If the sort order is _not_ DATE_ASCEND then the sense of the above
 2963  * is reversed.
 2964  */
 2965 static int
 2966 date_comp_asc(
 2967     t_comptype p1,
 2968     t_comptype p2)
 2969 {
 2970     const struct t_article *s1 = (const struct t_article *) p1;
 2971     const struct t_article *s2 = (const struct t_article *) p2;
 2972 
 2973     /*
 2974      * s1->date less than s2->date
 2975      */
 2976     if (s1->date < s2->date)
 2977         return -1;
 2978 
 2979     /*
 2980      * s1->date greater than s2->date
 2981      */
 2982     if (s1->date > s2->date)
 2983         return 1;
 2984 
 2985     return 0;
 2986 }
 2987 
 2988 
 2989 static int
 2990 date_comp_desc(
 2991     t_comptype p1,
 2992     t_comptype p2)
 2993 {
 2994     const struct t_article *s1 = (const struct t_article *) p1;
 2995     const struct t_article *s2 = (const struct t_article *) p2;
 2996 
 2997     /*
 2998      * s2->date less than s1->date
 2999      */
 3000     if (s2->date < s1->date)
 3001         return -1;
 3002 
 3003     /*
 3004      * s2->date greater than s1->date
 3005      */
 3006     if (s2->date > s1->date)
 3007         return 1;
 3008 
 3009     return 0;
 3010 }
 3011 
 3012 
 3013 /*
 3014  * Same again, but for art[].score
 3015  */
 3016 static int
 3017 score_comp_asc(
 3018     t_comptype p1,
 3019     t_comptype p2)
 3020 {
 3021     const struct t_article *s1 = (const struct t_article *) p1;
 3022     const struct t_article *s2 = (const struct t_article *) p2;
 3023 
 3024     if (s1->score < s2->score)
 3025         return -1;
 3026 
 3027     if (s1->score > s2->score)
 3028         return 1;
 3029 
 3030     return s1->date - s2->date > 0 ? 1 : -1;
 3031 }
 3032 
 3033 
 3034 static int
 3035 score_comp_desc(
 3036     t_comptype p1,
 3037     t_comptype p2)
 3038 {
 3039     const struct t_article *s1 = (const struct t_article *) p1;
 3040     const struct t_article *s2 = (const struct t_article *) p2;
 3041 
 3042     if (s2->score < s1->score)
 3043         return -1;
 3044 
 3045     if (s2->score > s1->score)
 3046         return 1;
 3047 
 3048     return s1->date - s2->date > 0 ? 1 : -1;
 3049 }
 3050 
 3051 
 3052 /*
 3053  * Same again, but for art[].line_count
 3054  */
 3055 static int
 3056 lines_comp_asc(
 3057     t_comptype p1,
 3058     t_comptype p2)
 3059 {
 3060     const struct t_article *s1 = (const struct t_article *) p1;
 3061     const struct t_article *s2 = (const struct t_article *) p2;
 3062 
 3063     if (s1->line_count < s2->line_count)
 3064         return -1;
 3065 
 3066     if (s1->line_count > s2->line_count)
 3067         return 1;
 3068 
 3069     return s1->date - s2->date > 0 ? 1 : -1;
 3070 }
 3071 
 3072 
 3073 static int
 3074 lines_comp_desc(
 3075     t_comptype p1,
 3076     t_comptype p2)
 3077 {
 3078     const struct t_article *s1 = (const struct t_article *) p1;
 3079     const struct t_article *s2 = (const struct t_article *) p2;
 3080 
 3081     if (s2->line_count < s1->line_count)
 3082         return -1;
 3083 
 3084     if (s2->line_count > s1->line_count)
 3085         return 1;
 3086 
 3087     return s1->date - s2->date > 0 ? 1 : -1;
 3088 }
 3089 
 3090 
 3091 /*
 3092  * Compares the total score of two threads. Used for sorting base[].
 3093  */
 3094 static int
 3095 score_comp_base(
 3096     t_comptype p1,
 3097     t_comptype p2)
 3098 {
 3099     int a = get_score_of_thread(*(const long *) p1);
 3100     int b = get_score_of_thread(*(const long *) p2);
 3101 
 3102     /* If scores are equal, compare using the article sort order.
 3103      * This determines the order in a group of equally scored threads.
 3104      */
 3105     if (a == b) {
 3106         const struct t_article *s1 = &arts[*(const long *) p1];
 3107         const struct t_article *s2 = &arts[*(const long *) p2];
 3108         t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
 3109 
 3110         if (comp_func)
 3111             return (*comp_func)(s1, s2);
 3112         return 0;
 3113     }
 3114 
 3115     if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
 3116         return a > b ? 1 : -1;
 3117     return a < b ? 1 : -1;
 3118 }
 3119 
 3120 
 3121 /*
 3122  * Compare the date of the last posted article of two threads.
 3123  * Used for sorting base[].
 3124  */
 3125 static int
 3126 last_date_comp_base_desc(
 3127     t_comptype p1,
 3128     t_comptype p2)
 3129 {
 3130     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3131     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3132 
 3133     if (s2_last < s1_last)
 3134         return -1;
 3135 
 3136     if (s2_last > s1_last)
 3137         return 1;
 3138 
 3139     return 0;
 3140 }
 3141 
 3142 
 3143 static int
 3144 last_date_comp_base_asc(
 3145     t_comptype p1,
 3146     t_comptype p2)
 3147 {
 3148     time_t s1_last = get_last_posting_date(*(const long *) p1);
 3149     time_t s2_last = get_last_posting_date(*(const long *) p2);
 3150 
 3151     if (s2_last > s1_last)
 3152         return -1;
 3153 
 3154     if (s2_last < s1_last)
 3155         return 1;
 3156 
 3157     return 0;
 3158 }
 3159 
 3160 
 3161 static time_t
 3162 get_last_posting_date(
 3163     long n)
 3164 {
 3165     long i;
 3166     time_t last = (time_t) 0;
 3167 
 3168     for (i = n; i >= 0; i = arts[i].thread) {
 3169         if (arts[i].date > last)
 3170             last = arts[i].date;
 3171     }
 3172 
 3173     return last;
 3174 }
 3175 
 3176 
 3177 void
 3178 set_article(
 3179     struct t_article *art)
 3180 {
 3181     art->subject = NULL;
 3182     art->from = NULL;
 3183     art->name = NULL;
 3184     art->date = (time_t) 0;
 3185     art->xref = NULL;
 3186     art->msgid = NULL;
 3187     art->refs = NULL;
 3188     art->refptr = NULL;
 3189     art->line_count = -1;
 3190     art->archive = NULL;
 3191     art->tagged = 0;
 3192     art->thread = ART_EXPIRED;
 3193     art->prev = ART_NORMAL;
 3194     art->score = 0;
 3195     art->status = ART_UNREAD;
 3196     art->killed = ART_NOTKILLED;
 3197     art->zombie = FALSE;
 3198     art->delete_it = FALSE;
 3199     art->selected = FALSE;
 3200     art->inrange = FALSE;
 3201     art->matched = FALSE;
 3202     art->keep_in_base = FALSE;
 3203     art->multipart_subj = FALSE;
 3204 }
 3205 
 3206 
 3207 /*
 3208  * Do a binary chop to see if 'art' (an article number) exists in arts[]
 3209  * Naturally arts[] must be sorted on artnum
 3210  * Return index into arts[] or -1
 3211  */
 3212 static int
 3213 valid_artnum(
 3214     t_artnum art)
 3215 {
 3216     int prev, range;
 3217     int dctop = top_art;
 3218     int cur = 1;
 3219 
 3220     while ((dctop >>= 1))
 3221         cur <<= 1;
 3222 
 3223     range = cur >> 1;
 3224     cur--;
 3225 
 3226     forever {
 3227         if (arts[cur].artnum == art)
 3228             return cur;
 3229 
 3230         prev = cur;
 3231         cur += ((arts[cur].artnum < art) ? range : -range);
 3232         if (prev == cur)
 3233             break;
 3234 
 3235         if (cur >= top_art)
 3236             cur = top_art - 1;
 3237 
 3238         range >>= 1;
 3239     }
 3240     return -1;
 3241 }
 3242 
 3243 
 3244 /*
 3245  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
 3246  * Needed if arts[] is not sorted on artnum
 3247  * Return index into arts[] or -1
 3248  */
 3249 int
 3250 find_artnum(
 3251     t_artnum art)
 3252 {
 3253     int i;
 3254 
 3255     for_each_art(i) {
 3256         if (arts[i].artnum == art)
 3257             return i;
 3258     }
 3259     return -1;
 3260 }
 3261 
 3262 
 3263 /*----------------------------- Overview handling -----------------------*/
 3264 /* TODO: use
 3265  *           setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
 3266  *           my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
 3267  *       instead?
 3268  */
 3269 static char *
 3270 print_date(
 3271     time_t secs)
 3272 {
 3273     static char date[25];
 3274     struct tm *tm;
 3275     static const char *const months_a[] = {
 3276         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 3277         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
 3278     };
 3279 
 3280     if ((tm = gmtime(&secs)) != NULL)
 3281         snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
 3282             tm->tm_mday,
 3283             months_a[tm->tm_mon],
 3284             tm->tm_year + 1900,
 3285             tm->tm_hour, tm->tm_min, tm->tm_sec);
 3286     else
 3287         snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
 3288 
 3289     return date;
 3290 }
 3291 
 3292 
 3293 static char *
 3294 print_from(
 3295     struct t_group *group,
 3296     struct t_article *article,
 3297     int charset)
 3298 {
 3299     char *p, *q;
 3300     static char from[PATH_LEN];
 3301 
 3302     *from = '\0';
 3303 
 3304     if (article->name != NULL) {
 3305         q = my_strdup(article->name);
 3306 #ifdef CHARSET_CONVERSION
 3307         if (charset != -1)
 3308             buffer_to_network(q, charset);
 3309 #endif /* CHARSET_CONVERSION */
 3310         p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
 3311         unfold_header(p);
 3312         if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
 3313             snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3314         else
 3315             snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
 3316 
 3317         free(p);
 3318         free(q);
 3319     } else
 3320         snprintf(from, sizeof(from), "<%s>", article->from);
 3321 
 3322     return from;
 3323 }
 3324 
 3325 
 3326 /*
 3327  * Open a group news overview file
 3328  * Use NNTP XOVER where possible unless 'local' is set
 3329  */
 3330 static FILE *
 3331 open_xover_fp(
 3332     struct t_group *group,
 3333     const char *mode,
 3334     t_artnum min,
 3335     t_artnum max,
 3336     t_bool local)
 3337 {
 3338 #ifdef NNTP_ABLE
 3339     if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
 3340         char line[NNTP_STRLEN];
 3341 
 3342         if (!max)
 3343             return NULL;
 3344         if (min == max)
 3345             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
 3346         else
 3347             snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
 3348         return (nntp_command(line, OK_XOVER, NULL, 0));
 3349     }
 3350 #endif /* NNTP_ABLE */
 3351     {
 3352         FILE *fp;
 3353         char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
 3354 
 3355         if (nov_file != NULL) {
 3356             if ((fp = fopen(nov_file, mode)) != NULL)
 3357                 return fp;
 3358 
 3359             if (*mode != 'r')
 3360                 error_message(2, _(txt_cannot_open), nov_file);
 3361         }
 3362     }
 3363     return NULL;
 3364 }
 3365 
 3366 
 3367 #ifdef USE_HEAPSORT
 3368 int
 3369 tin_sort(
 3370     void *sbase,
 3371     size_t nel,
 3372     size_t width,
 3373     t_compfunc compar)
 3374 {
 3375     int rc;
 3376 
 3377     switch (tinrc.sort_function) {
 3378         case 0:
 3379             qsort(sbase, nel, width, compar);
 3380             rc = 0;
 3381             break;
 3382 
 3383         case 1:
 3384             rc = heapsort(sbase, nel, width, compar);
 3385             break;
 3386 
 3387         default:
 3388             rc = -1;
 3389             break;
 3390     }
 3391     return rc;
 3392 }
 3393 #endif /* USE_HEAPSORT */