tin  2.4.5
About: TIN is a threaded NNTP and spool based UseNet newsreader.
  Fossies Dox: tin-2.4.5.tar.xz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

art.c
Go to the documentation of this file.
1 /*
2  * Project : tin - a Usenet reader
3  * Module : art.c
4  * Author : I.Lea & R.Skrenta
5  * Created : 1991-04-01
6  * Updated : 2020-07-08
7  * Notes :
8  *
9  * Copyright (c) 1991-2021 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10  * All rights reserved.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  *
16  * 1. Redistributions of source code must retain the above copyright notice,
17  * this list of conditions and the following disclaimer.
18  *
19  * 2. Redistributions in binary form must reproduce the above copyright
20  * notice, this list of conditions and the following disclaimer in the
21  * documentation and/or other materials provided with the distribution.
22  *
23  * 3. Neither the name of the copyright holder nor the names of its
24  * contributors may be used to endorse or promote products derived from
25  * this software without specific prior written permission.
26  *
27  * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
31  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37  * POSSIBILITY OF SUCH DAMAGE.
38  */
39 
40 
41 #ifndef TIN_H
42 # include "tin.h"
43 #endif /* !TIN_H */
44 #ifndef TCURSES_H
45 # include "tcurses.h"
46 #endif /* !TCURSES_H */
47 #ifndef NEWSRC_H
48 # include "newsrc.h"
49 #endif /* !NEWSRC_H */
50 
51 #ifndef STPWATCH_H
52 # include "stpwatch.h"
53 #endif /* !STPWATCH_H */
54 
55 /*
56  * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
57  */
58 #define SortBy(func) tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
59 
60 int top_art = 0; /* # of articles in arts[] */
61 
62 /*
63  * Local prototypes
64  */
65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
67 static char *find_nov_file(struct t_group *group, int mode);
68 static char *print_date(time_t secs);
69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
70 static int artnum_comp(t_comptype p1, t_comptype p2);
71 static int base_comp(t_comptype p1, t_comptype p2);
72 static int date_comp_asc(t_comptype p1, t_comptype p2);
73 static int date_comp_desc(t_comptype p1, t_comptype p2);
74 static int from_comp_asc(t_comptype p1, t_comptype p2);
75 static int from_comp_desc(t_comptype p1, t_comptype p2);
76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
83 static int score_comp_asc(t_comptype p1, t_comptype p2);
84 static int score_comp_desc(t_comptype p1, t_comptype p2);
85 static int score_comp_base(t_comptype p1, t_comptype p2);
86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
88 static int valid_artnum(t_artnum art);
89 static t_artnum find_first_unread(struct t_group *group);
90 static t_artnum setup_hard_base(struct t_group *group);
91 static t_bool parse_headers(FILE *fp, struct t_article *h);
92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
93 static time_t get_last_posting_date(long n);
94 static void sort_base(unsigned int sort_threads_type);
95 static void thread_by_multipart(void);
96 static void thread_by_percentage(unsigned int percentage);
97 static void thread_by_subject(void);
98 static void write_overview(struct t_group *group);
99 #ifdef NNTP_ABLE
100  static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
101  static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
102 #endif /* NNTP_ABLE */
103 
104 
105 /*
106  * Display a suitable 'entering group' message if screen needs redrawing
107  * Allow for the non-printing %s, and the %-age counter
108  */
109 void
111  const char *group)
112 {
113  wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
114 }
115 
116 
117 /*
118  * Construct the pointers to the first (base) article in each thread.
119  * If we are showing only unread, then point to the first unread. I have
120  * no idea why this should be so, it causes problems elsewhere [which_response]
121  */
122 void
124  struct t_group *group)
125 {
126  int i, j;
127 
128  grpmenu.max = 0;
129 
130 #ifdef DEBUG
131  if (debug & DEBUG_FILTER)
132  debug_print_arts();
133 #endif /* DEBUG */
134 
135  for_each_art(i) {
136  /*
137  * .prev will be set on each article that is after the first article in
138  * the thread. Invalid articles which have been expired will have
139  * .thread set to ART_EXPIRED
140  */
141  if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
142  continue;
143 
144  if (grpmenu.max >= max_base)
145  expand_base();
146 
147  if (group->attribute->show_only_unread_arts) {
148  if (arts[i].status != ART_READ || arts[i].keep_in_base)
149  base[grpmenu.max++] = i;
150  else {
151  /* Find 1st unread art in thread */
152  for (j = i; j >= 0; j = arts[j].thread) {
153  if (arts[j].status != ART_READ || arts[j].keep_in_base) {
154  base[grpmenu.max++] = i;
155  break;
156  }
157  }
158  }
159  } else
160  base[grpmenu.max++] = i;
161  }
162  /* sort base[] */
165 }
166 
167 
168 /*
169  * Longword comparison routine for the tin_sort()
170  */
171 static int
173  t_comptype p1,
174  t_comptype p2)
175 {
176  const t_artnum *a = (const t_artnum *) p1;
177  const t_artnum *b = (const t_artnum *) p2;
178 
179  if (*a < *b)
180  return -1;
181 
182  if (*a > *b)
183  return 1;
184 
185  return 0;
186 }
187 
188 
189 /*
190  * via NNTP:
191  * Issue a LISTGROUP command
192  * Read the article numbers existing in the group into base[]
193  * If the LISTGROUP failed, issue a GROUP command. Use the results to
194  * create a less accurate version of base[]
195  * This data will already be sorted
196  *
197  * on local spool:
198  * Read the spool dir to populate base[] as above. Sort it.
199  *
200  * Grow the arts[] and bitmaps as needed.
201  * NB: the output will be sorted on artnum
202  *
203  * grpmenu.max is one past top.
204  * Returns total number of articles in group, or -1 on error
205  */
206 static t_artnum
208  struct t_group *group)
209 {
210  t_artnum total = 0;
211 
212  grpmenu.max = 0;
213 
214  /*
215  * If reading with NNTP, issue a LISTGROUP
216  */
218 #ifdef NNTP_ABLE
219  char buf[NNTP_STRLEN];
220  char line[NNTP_STRLEN];
222  FILE *fp;
223  t_artnum last, start, count = 0, j = 0;
224  static t_bool skip_listgroup = FALSE;
225 
226  /*
227  * Some nntp servers are broken and need an extra GROUP command
228  * (reported by reorx@irc.pl). This affects (old?) versions of
229  * nntpcache, leafnode and SurgeNews. Usually this should not be
230  * needed.
231  *
232  * For getart_limit recheck lowwatermark as at least giganews gives
233  * very different results for LIST ACTIVE (3 year retention for all)
234  * and GROUP (based on the clients contract).
235  * Calculate range and prepare base[] not to lose unread arts.
236  */
237  if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
238  snprintf(buf, sizeof(buf), "GROUP %s", group->name);
239  if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
240  return -1;
241 
242  if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
243  return -1;
244 
245  if (getart_limit > 0) {
246  j = group->xmax - getart_limit;
247  count = MAX(find_first_unread(group), start);
248  }
249  if (getart_limit < 0) {
250  j = getart_limit + find_first_unread(group);
251  count = group->xmin;
252  }
253  if (j < group->xmin)
254  j = group->xmin;
255 
256  for (; count < j; count++) {
257  if (grpmenu.max >= max_base)
258  expand_base();
259  base[grpmenu.max++] = count;
260  }
261  }
262 
263  /*
264  * See if LISTGROUP works
265  */
266  if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
268  /* RFC 3977 allows ranges in LISTGROUP */
269  if (getart_limit > 0)
270  snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
271  else /* getart_limit < 0; fetch till newest art */
272  snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
273 
274  } else /* for RFC 977 just use GROUP */
275  skip_listgroup = TRUE;
276 
277  } else /* get all article numbers */
278  snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
279 
280  if (!skip_listgroup) {
281  if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
282  char *ptr;
283 
284 # ifdef DEBUG
285  if ((debug & DEBUG_NNTP) && verbose > 1)
286  debug_print_file("NNTP", "setup_hard_base(%s)", buf);
287 # endif /* DEBUG */
288 
289  while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
290  if (grpmenu.max >= max_base)
291  expand_base();
292  base[grpmenu.max++] = atoartnum(ptr);
293  total++;
294  }
295 
296  if (tin_errno)
297  return -1;
298  } else
299  skip_listgroup = TRUE;
300  }
301 
302  if (skip_listgroup) { /* LISTGROUP was skipped or failed */
303  /*
304  * Handle the obscure case that the user aborted before the LISTGROUP
305  * had a chance to respond
306  */
307  if (tin_errno)
308  return -1;
309 
310  snprintf(buf, sizeof(buf), "GROUP %s", group->name);
311  if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
312  return -1;
313 
314  if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
315  return -1;
316 
317 # ifdef DEBUG
318  if ((debug & DEBUG_NNTP) && verbose > 1)
319  debug_print_file("NNTP", "setup_hard_base(%s)", buf);
320 # endif /* DEBUG */
321  total = count;
322  grpmenu.max = 0;
323  if (getart_limit > 0) {
324  if ((j = find_first_unread(group)) > start) {
325  if (group->xmax > getart_limit) {
326  start = MIN(j, group->xmax - getart_limit);
327  total = getart_limit;
328  } else
329  start = j;
330  }
331  }
332  if (getart_limit < 0) {
333  if ((j = (getart_limit + find_first_unread(group))) > start)
334  start = j;
335  }
336  while (start <= last) {
337  if (grpmenu.max >= max_base)
338  expand_base();
339  base[grpmenu.max++] = start++;
340  }
341  }
342 #endif /* NNTP_ABLE */
343  /*
344  * Reading off local spool, read the directory files
345  */
346  } else {
347  DIR *d;
348  DIR_BUF *e;
349  char group_path[PATH_LEN];
350  t_artnum art;
351 
352  make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
353 
354  if ((d = opendir(group_path)) != NULL) {
355  while ((e = readdir(d)) != NULL) {
356  art = atoartnum(e->d_name);
357  if (art >= 1) {
358  total++;
359  if (grpmenu.max >= max_base)
360  expand_base();
361  base[grpmenu.max++] = art;
362  }
363  }
364  CLOSEDIR(d);
365  tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
366  } else {
367  perror_message(_(txt_cannot_open), group_path);
368 #if 0
369  if (access(group_path, R_OK) != 0)
371 #endif /* 0 */
372  return -1;
373  }
374  }
375 
376  if (grpmenu.max) {
377  if (base[grpmenu.max - 1] > group->xmax)
378  group->xmax = base[grpmenu.max - 1];
379  expand_bitmap(group, base[0]);
380  }
381 
382  return total;
383 }
384 
385 
386 /*
387  * Main group indexing routine.
388  *
389  * Will read any existing index, create or incrementally update
390  * the index by looking at the articles in the spool directory,
391  * and attempt to write a new index if necessary.
392  *
393  * Returns FALSE if the user aborted the indexing, otherwise TRUE
394  */
395 t_bool
397  struct t_group *group)
398 {
399  int i;
400  int changed; /* Count of articles whose overview has changed */
401  int getart_limit;
402  int respnum;
403  int total;
404  t_artnum last_read_article;
405  t_artnum min, new_min, max;
406  t_bool caching_xover;
407  t_bool filtered;
408  t_bool path_in_nov = FALSE;
409  t_bool rebuild_cache = FALSE;
410 
411  if (group == NULL)
412  return TRUE;
413 
414  if (!batch_mode)
415  show_art_msg(group->name);
416  else {
417  if (verbose) /* -> lang.c */
418  wait_message(0, _("Reading %s\n"), group->name);
419  }
420 
421  signal_context = cArt; /* Set this only once curr_group is valid */
422 
423  hash_reclaim();
424  free_art_array();
425  free_msgids();
426 
427  BegStopWatch("setup_hard_base()");
428 
429  /*
430  * Get list of valid article numbers
431  */
432  if (setup_hard_base(group) < 0)
433  return FALSE;
434 
435  EndStopWatch();
436  PrintStopWatch();
437 
438 #ifdef DEBUG
439  if (debug & DEBUG_NEWSRC) {
440  debug_print_comment("Before read_overview");
441  debug_print_bitmap(group, NULL);
442  }
443 #endif /* DEBUG */
444 
445  min = grpmenu.max ? base[0] : group->xmin;
446  max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
447 
449 
450  if (getart_limit > 0) {
451  if (grpmenu.max && (grpmenu.max > getart_limit))
452  min = base[grpmenu.max - getart_limit];
453  else
454  getart_limit = 0;
455  } else if (getart_limit < 0) {
456  t_artnum first_unread = find_first_unread(group);
457 
458  if (min - first_unread < getart_limit)
459  min = first_unread + getart_limit;
460  else
461  getart_limit = 0;
462  }
463 
464  /*
465  * Quit now if no articles
466  */
467  if (max < 0)
468  return FALSE;
469 
470  top_art = 0;
471  last_read_article = T_ARTNUM_CONST(0);
472 
473  /*
474  * Read in the existing overview data for min..max
475  * This read has local=TRUE set if locally caching XOVER records to ensure
476  * we pull in any private overview caches in preference to using OVER
477  *
478  * When reading local spool, this will pull in the system wide overview
479  * cache (if found) otherwise the private overview cache will be read
480  */
481  caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
482  if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
483  return FALSE; /* user aborted indexing */
484 
485  /*
486  * Fill in the range last_read_article...max using XOVER
487  * Only do this if the previous read_overview() was against private cache
488  */
489  if ((last_read_article < max) && caching_xover) {
490  new_min = (last_read_article >= min) ? last_read_article + 1 : min;
491 
492  if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
493  return FALSE; /* user aborted indexing */
494  else
495  changed += i;
496  } else
497  caching_xover = FALSE;
498 
499  /*
500  * Mark as UNTHREADED all articles that have been verified as valid
501  * Get num of new arts to index so the user will have an idea of index time
502  */
503  for (i = 0, total = 0; i < grpmenu.max; i++) {
504  if ((respnum = valid_artnum(base[i])) >= 0) {
505  arts[respnum].thread = ART_UNTHREADED;
506  continue;
507  }
508  if (base[i] <= last_read_article) /* It is vital this test be done last */
509  continue;
510  total++;
511  }
512 
513  /*
514  * Add any articles to arts[] that are new or were killed
515  */
516  if (total > 0) {
517  new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
518 
519  if ((i = read_art_headers(group, total, new_min)) == -1)
520  return FALSE; /* user aborted indexing */
521  else
522  changed += i;
523  }
524 
525 #ifdef DEBUG
526  if (debug & DEBUG_NEWSRC) {
527  debug_print_comment("Before parse_unread_arts()");
528  debug_print_bitmap(group, NULL);
529  }
530 #endif /* DEBUG */
531  /*
532  * Do this before calling art_mark(,, ART_READ) if you want
533  * the unread count to be correct.
534  */
535  min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
536  parse_unread_arts(group, min);
537 #ifdef DEBUG
538  if (debug & DEBUG_NEWSRC) {
539  debug_print_comment("After parse_unread_arts()");
540  debug_print_bitmap(group, NULL);
541  }
542 #endif /* DEBUG */
543 
544  /*
545  * Stat all articles to see if any have expired
546  */
547  for_each_art(i) {
548  if (arts[i].thread == ART_EXPIRED) {
549  changed++;
550 #ifdef DEBUG
551  if (debug & DEBUG_NEWSRC)
552  debug_print_comment("art.c: index_group() purging...");
553 #endif /* DEBUG */
554  art_mark(group, &arts[i], ART_READ);
555  if (group->attribute->show_only_unread_arts)
556  arts[i].keep_in_base = FALSE;
557  }
558  if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
559  path_in_nov = TRUE;
560  }
561 
562  /*
563  * Only rewrite the index if it has changed
564  * TODO review the exact logic behind "|| caching_xover"
565  */
566  if (changed || caching_xover || rebuild_cache)
567  write_overview(group);
568 
569  /*
570  * Create the reference tree. The msgid and ref ptrs will
571  * be free()d now that the NovFile has been written.
572  */
573  build_references(group);
574 
575  /*
576  * Needs access to the reference tree
577  */
578  filtered = filter_articles(group);
579 
580  BegStopWatch("make_threads()");
581 
582  /*
583  * Thread the group
584  */
585  make_threads(group, FALSE);
586 
587  EndStopWatch();
588  PrintStopWatch();
589 
590  if ((changed > 0 || filtered) && !batch_mode)
591  clear_message();
592 
593  return TRUE;
594 }
595 
596 
597 /*
598  * Returns number of first unread article
599  */
600 static t_artnum
602  struct t_group *group)
603 {
604  unsigned char *p;
605  unsigned char *end = group->newsrc.xbitmap;
606  t_artnum first = group->newsrc.xmin; /* initial value */
607 
608  if ((p = group->newsrc.xbitmap)) {
609  end += group->newsrc.xbitlen / NBITS;
610  for (; *p == '\0' && p < end; p++, first += NBITS)
611  ;
612  }
613  return first;
614 }
615 
616 
617 /*
618  * Open an article for reading just the header
619  * 'next' is used/updated with the next article number
620  * to optimise the number of 'HEAD' commands issued on
621  * groups with holes.
622  */
623 static FILE *
625  char *groupname,
626  t_artnum art,
627  t_artnum *next)
628 {
629  char buf[NNTP_STRLEN];
630 #ifdef NNTP_ABLE
631  FILE *fp;
632  int i;
633 
635  /*
636  * Don't bother requesting if we have not got there yet.
637  * This is a big win if the group has got holes in it (ie. if 000's
638  * of articles have expired between active files min & max values).
639  */
640  if (art < *next)
641  return NULL;
642 
643  snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
644  if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
645  return fp;
646 
647  /*
648  * TODO:
649  * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
650  * "503 Access denied" instead of 480 but NEXT still works,
651  * so tin loops over all articles without getting useful data
652  */
653 
654  /*
655  * HEAD failed, try to find NEXT
656  * Should return "223 artno message-id more text...."
657  */
658  i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
659  switch (i) {
660  case OK_NOTEXT:
661  *next = atoartnum(buf); /* Set next art number */
662  break;
663 
664 # ifndef BROKEN_LISTGROUP
665  /*
666  * might happen if LISTGROUP doesn't select group, but
667  * we are not -DBROKEN_LISTGROUP
668  */
669  case ERR_NCING:
671  snprintf(buf, sizeof(buf), "GROUP %s", groupname);
672  if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
673  return NULL;
674  snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
675  if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
676  return fp;
677  if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
678  *next = atoartnum(buf);
679  break;
680 # endif /* !BROKEN_LISTGROUP */
681 
682  default:
683  /*
684  * TODO: abort loop over all arts on ERR_NONEXT
685  */
686 # ifndef BROKEN_LISTGROUP
687  /*
688  * to avoid out of sync responses
689  * (listgroup seems to work, but didn't select new group,
690  * so xover seems to work but returns old data)
691  * we set listgroup_broken = TRUE; once we saw a
692  * ERR_NOARTIG / ERR_NONEXT or the like - even if
693  * ERR_NOARTIG may occur on servers where listgroup
694  * isn't broken...
695  */
697 # endif /* !BROKEN_LISTGROUP */
698  break;
699  }
700 
701  return NULL;
702  }
703 #endif /* NNTP_ABLE */
704 
705  snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
706  return (fopen(buf, "r"));
707 }
708 
709 
710 /*
711  * Called after XOVER/local/private overview databases have been loaded
712  * Read and parse in headers for any arts not already found (usually
713  * new articles that have not been indexed yet)
714  * Any new articles that are added have ->thread set to ART_UNTHREADED
715  * 'top' is the current highest artnum read
716  *
717  * Return values are:
718  * >0 Number of additional articles read in
719  * 0 No additional (new) articles were found
720  * -1 user aborted during read
721  */
722 static int
724  struct t_group *group,
725  int total,
726  t_artnum top)
727 {
728  FILE *fp;
729  char dir[PATH_LEN];
730  char group_msg[LEN];
731  int i;
732  int modified = 0;
733  t_artnum art;
734  t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
735  t_bool res;
736 
737  /*
738  * Change to groups spooldir to optimize fopen()'s on local articles
739  * NB open_art_header() requires this
740  */
741  if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
742  char buf[PATH_LEN];
743 
744  get_cwd(dir);
745  make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
746  if (chdir(buf) != 0) {
747 #ifdef DEBUG
748  if (debug & DEBUG_MISC)
749  fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
750 #endif /* DEBUG */
751  return -1;
752  }
753  }
754 
755  snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
756 
757  for (i = 0; i < grpmenu.max; i++) { /* for each article number */
758  art = base[i];
759 
760  /*
761  * Skip articles that are below the low water mark or are
762  * already present
763  */
764  if (valid_artnum(art) >= 0)
765  continue;
766  if (art <= top)
767  continue;
768 
769  /*
770  * Try and open the article
771  */
772  if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
773  continue;
774 
775  /*
776  * Add article to arts[]
777  */
778  if (top_art >= max_art)
779  expand_art();
780 
782  arts[top_art].artnum = art;
784 
785  res = parse_headers(fp, &arts[top_art]);
786 
787  TIN_FCLOSE(fp);
788  if (tin_errno) {
789  modified = -1;
790  break;
791  }
792 
793  if (!res) {
794 #ifdef DEBUG
795  if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
796  char buf[PATH_LEN];
797 
798  snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
799  debug_print_file("ARTS", "read_art_headers() %s", buf);
800  }
801 #endif /* DEBUG */
803  arts[top_art].date = (time_t) 0;
804  FreeAndNull(arts[top_art].xref);
805  FreeAndNull(arts[top_art].path);
806  FreeAndNull(arts[top_art].refs);
807  FreeAndNull(arts[top_art].msgid);
808  if (arts[top_art].archive) {
809  FreeAndNull(arts[top_art].archive->partnum);
810  FreeAndNull(arts[top_art].archive);
811  }
812  arts[top_art].tagged = 0;
818  continue;
819  }
820 
821  top = arts[top_art].artnum; /* used if arts are killed */
822  top_art++;
823 
824  if (++modified % (MODULO_COUNT_NUM * 20) == 0)
825  show_progress(group_msg, modified, total);
826  }
827 
828  /*
829  * Change back to previous dir before indexing started
830  */
831  if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
832  chdir(dir);
833 
834  return modified;
835 }
836 
837 
838 /*
839  * The algorithm is elegant, using the fact that identical Subject lines
840  * are hashed to the same node in table[] (see hashstr.c)
841  *
842  * Mark i as being in j's thread list if
843  * . The article is _not_ being ignored
844  * . The article is not already threaded
845  * . One of the following is true:
846  * 1) The subject lines are the same
847  * 2) Both are part of the same archive (name's match and arch bit set)
848  * IMHO the tests for archive name are redundant and have been for years
849  */
850 static void
852  void)
853 {
854  int i, j;
855  struct t_hashnode *h;
856 
857  for_each_art(i) {
858  if (IGNORE_ART_THREAD(i))
859  continue;
860 
861  /*
862  * Get the contents of the magic marker in the hashnode
863  */
864  h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
865 
866  j = h->aptr;
867 
868  if (j != -1 && j < i) {
869 #if 1
870  if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
871 #else
872  /* see also refs.c:collate_subjects() */
873  if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
874 #endif /* 1 */
875  {
876  arts[j].thread = i;
877  arts[i].prev = j;
878  }
879  }
880 
881  /*
882  * Update the magic marker with the highest numbered mesg in
883  * arts[] that has been used in this thread so far
884  */
885  h->aptr = i;
886  }
887 
888 #if 0
889  fprintf(stderr, "Subj dump\n");
890  fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
891  for_each_art(i) {
892  fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
893  (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
894  (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
895  (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
896  arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
897  }
898 #endif /* 0 */
899 }
900 
901 
902 /*
903  * This Threading algorithm threads articles into 'buckets' where each bucket
904  * contains all the articles which match the root article's subject line to
905  * the configured percentage. Eg, if the root article had the subject "asdf"
906  * and the match percentage was configured to be 75% then any article would
907  * match if its subject was no different in more than a single character.
908  */
909 static void
911  unsigned int percentage)
912 {
913  int i, j, k;
914  int root_num = 0; /* The index number of the root we are currently working on. */
915  unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
916  size_t slen;
917 
918  /* First we need to sort art[] to simplify and speed up the matching. */
920 
921  /*
922  * Now we put all the articles which match enough into the thread. If
923  * an article doesn't match enough we create a new thread and then add
924  * to that and so on.
925  */
926  base[0] = 0;
927  arts[0].prev = ART_NORMAL;
928  for_each_art(i) {
929  if (i == 0)
930  continue;
931 
932  /* Check each character to see if it matched enough */
933  k = 0;
934  unmatched = 0;
935  for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
936  if (arts[base[root_num]].subject[j] != arts[i].subject[k])
937  unmatched++;
938  }
939 
940  /*
941  * By getting here we have a number of unmatched characters
942  * between the two strings. We also have the length of the
943  * strings available to us easily.
944  * All we need to do is see if the match is good enough, but
945  * we count differences in the length of the strings against
946  * them matching.
947  */
948  if (!(slen = strlen(arts[base[root_num]].subject)))
949  slen++;
950  unmatched += slen - strlen(arts[i].subject);
951  if (unmatched * 100 / slen > percentage) {
952  /*
953  * If there is less greater than percentage% different start a
954  * new thread.
955  */
956  base[++root_num] = i;
957  arts[i].prev = ART_NORMAL;
958  continue;
959  } else {
960  /*
961  * The subject lines match enough to consider them part of a single
962  * thread, so add the current article to the thread.
963  */
964  if (arts[base[root_num]].thread < 0)
965  arts[base[root_num]].thread = i;
966  arts[i].prev = i - 1;
967  arts[i - 1].thread = i;
968  continue;
969  }
970  }
971 }
972 
973 
974 /*
975  * Parses a subject header of the type "multipart message subject (01/42)"
976  * into a MultiPartInfo struct, or fails if the message subject isn't in the
977  * right form.
978  *
979  * @return nonzero on success
980  */
981 int
983  int aindex,
984  MultiPartInfo *setme)
985 {
986  int i, j, offi, offj;
987  MultiPartInfo setmei, setmej;
988 
989  i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
990  j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
991 
992  /* Ok i hits first */
993  if (offi > offj) {
994  *setme = setmei;
995  return i;
996  }
997 
998  /* Its j or they are both the same (which must be zero!) so we don't care */
999  *setme = setmej;
1000  return j;
1001 }
1002 
1003 
1004 static int
1006  int aindex,
1007  MultiPartInfo* setme,
1008  char start,
1009  char stop,
1010  int *offset)
1011 {
1012  char *subj;
1013  char *pch;
1014  MultiPartInfo tmp;
1015 
1016  *offset = 0;
1017 
1018  /* entry assertions */
1019  assert(0 <= aindex && aindex < top_art && "invalid index");
1020  assert(setme != NULL && "setme must not be NULL");
1021 
1022  /* parse the message */
1023  subj = arts[aindex].subject;
1024  pch = strrchr(subj, start);
1025  if (!pch || !isdigit((int) pch[1]))
1026  return 0;
1027 
1028  tmp.arts_index = aindex;
1029  tmp.subject_compare_len = pch - subj;
1030  tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1031  if (*pch != '/' && *pch != '|')
1032  return 0;
1033 
1034  if (!isdigit((int) pch[1]))
1035  return 0;
1036 
1037  tmp.total = (int) strtol(pch + 1, &pch, 10);
1038  if (*pch != stop)
1039  return 0;
1040 
1041  /*
1042  * skip "blah (00/102)" or "blah (103/102)" subjects
1043  */
1044  if (tmp.part_number < 1 || tmp.part_number > tmp.total)
1045  return 0;
1046 
1047  tmp.subject = subj;
1048  *setme = tmp;
1049  *offset = pch - subj;
1050  return 1;
1051 }
1052 
1053 
1054 t_bool
1056  int aindex,
1057  char start,
1058  char stop)
1059 {
1060  char *pch;
1061 
1062  pch = strrchr(arts[aindex].subject, start);
1063  if (!pch || !isdigit((int) pch[1]))
1064  return FALSE;
1065 
1066  strtol(pch + 1, &pch, 10);
1067  if (*pch != '/' && *pch != '|')
1068  return FALSE;
1069 
1070  if (!isdigit((int) pch[1]))
1071  return FALSE;
1072 
1073  strtol(pch + 1, &pch, 10);
1074  if (*pch != stop)
1075  return FALSE;
1076 
1077  arts[aindex].multipart_subj = TRUE;
1078  return TRUE;
1079 }
1080 
1081 
1082 /*
1083  * Tries to find all the parts to the multipart message pointed to by
1084  * aindex.
1085  *
1086  * @return on success, the number of parts found. On failure, zero if not
1087  * a multipart or the negative value of the first missing part in case of
1088  * tagging.
1089  * @param aindex index pointing to one of the messages in a multipart
1090  * message.
1091  * @param malloc_and_setme_info on success, set to a malloced array the
1092  * parts found.
1093  */
1094 int
1096  int aindex,
1097  MultiPartInfo **malloc_and_setme_info,
1098  t_bool tagging)
1099 {
1100  int i, part_index, part_cnt = 0;
1101  MultiPartInfo tmp, tmp2;
1102  MultiPartInfo *info = NULL;
1103 
1104  /* entry assertions */
1105  assert(0 <= aindex && aindex < top_art && "Invalid index");
1106  assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1107 
1108  /* make sure this is a multipart message... */
1109  if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1110  return 0;
1111 
1112  /* make a temporary buffer to hold the multipart info... */
1113  info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
1114 
1115  /* zero out part-number for the repost check below */
1116  for (i = 0; i < tmp.total; ++i) {
1117  info[i].total = tmp.total; /* Added this for thread_by_multipart */
1118  info[i].part_number = -1;
1119  }
1120 
1121  /* try to find all the multiparts... */
1122  for (i = (tagging ? 0 : aindex); i < top_art; i++) {
1123  if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
1124  continue;
1125 
1126  if (!global_get_multipart_info(i, &tmp2))
1127  continue;
1128 
1129  /* 'test (1/5)' is not the same as 'test (1/15)' */
1130  if (tmp.total != tmp2.total)
1131  continue;
1132 
1133  part_index = tmp2.part_number - 1;
1134 
1135  /* repost check: do we already have this part? */
1136  if (info[part_index].part_number != -1) {
1137  assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1138  continue;
1139  }
1140 
1141  /* we have a match, hooray! */
1142  info[part_index] = tmp2;
1143 
1144  arts[i].multipart_subj = FALSE;
1145 
1146  /* all parts found? */
1147  if (++part_cnt == tmp.total)
1148  break;
1149  }
1150 
1151  /* see if we got them all. */
1152  if (tagging) {
1153  for (i = 0; i < tmp.total; ++i) {
1154  if (info[i].part_number != i + 1) {
1155  free(info);
1156  return -(i + 1); /* missing part #(i+1) */
1157  }
1158  }
1159  }
1160 
1161  /* looks like a success .. */
1162  *malloc_and_setme_info = info;
1163  return tmp.total;
1164 }
1165 
1166 
1167 /*
1168  * The algorithm uses the tag multipart searches to thread articles together.
1169  */
1170 static void
1172  void)
1173 {
1174  int i, j, threadNum;
1175  MultiPartInfo *minfo = NULL;
1176 
1177  for_each_art(i) {
1178  if (!global_look_for_multipart(i, '[', ']'))
1179  global_look_for_multipart(i, '(', ')');
1180  }
1181 
1182  for_each_art(i) {
1183  if (!arts[i].multipart_subj)
1184  continue;
1185 
1186  if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
1187  FreeAndNull(minfo);
1188  arts[i].multipart_subj = FALSE;
1189  continue;
1190  }
1191 
1192  threadNum = -1;
1193  for (j = minfo[0].total - 1; j >= 0; j--) {
1194  if (minfo[j].part_number != -1) {
1195  if (threadNum != -1) {
1196  arts[minfo[j].arts_index].thread = threadNum;
1197  arts[threadNum].prev = minfo[j].arts_index;
1198  }
1199  threadNum = minfo[j].arts_index;
1200  }
1201  }
1202  FreeAndNull(minfo);
1203  arts[i].multipart_subj = FALSE;
1204  if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
1205  show_progress(_("Threading by multipart"), i, top_art);
1206  }
1207 }
1208 
1209 
1210 /*
1211  * Go through the articles in arts[] and create threads. There are
1212  * 5 strategies currently defined :
1213  *
1214  * THREAD_NONE No threading
1215  * THREAD_SUBJ Threads are created using like Subject lines
1216  * THREAD_REFS Threads are created using the References headers
1217  * THREAD_BOTH Threads created using References and then Subject
1218  * THREAD_MULTI Threads created using Subject to search for Multiparts
1219  * THREAD_PERC Threads based upon a char for char match of greater than x%
1220  *
1221  * .thread and .prev are used to hold the threading information, see tin.h for
1222  * more information
1223  * Only process valid (unexpired) articles we haven't visited yet
1224  * (ie arts[].thread == ART_UNTHREADED)
1225  *
1226  * The rethread parameter is used to force the deletion of existing threading
1227  * information before threading which happens anyway expect when using
1228  * THREAD_NONE (I don't immediately see how this is useful)
1229  */
1230 /* TODO: rewrite that user can easily combine different 'threading'
1231  * methods, i.e:
1232  * - thread_by_multipart() + collate_subjects()
1233  */
1234 void
1236  struct t_group *group,
1237  t_bool rethread)
1238 {
1239  if (!cmd_line && !batch_mode) {
1241  my_flush();
1242  }
1243 
1244 #ifdef DEBUG
1245  if (debug & DEBUG_MISC)
1246  error_message(2, "rethread=[%d] thread_articles=[%d] attr_thread_articles=[%d]",
1247  rethread, tinrc.thread_articles, group->attribute->thread_articles);
1248 #endif /* DEBUG */
1249 
1250  /*
1251  * Sort all the articles using the preferred method
1252  * When find_base() is called, the bases are created ordered
1253  * on arts[] and so the base messages under all threading systems
1254  * will be sorted in this way.
1255  */
1257 
1258  /*
1259  * Reset all the ptrs to articles following the above sort
1260  */
1261  clear_art_ptrs();
1262 
1263  /*
1264  * The threading pointers need to be reset if re-threading
1265  * If using ref threading, revector the links back to the articles
1266  */
1267  if (rethread || group->attribute->thread_articles) {
1268  int i;
1269 
1270  for_each_art(i) {
1271  if (arts[i].thread >= 0)
1272  arts[i].thread = ART_UNTHREADED;
1273 
1274  arts[i].prev = ART_NORMAL;
1275 
1276  /* Should never happen if tree is built properly */
1277  if (arts[i].refptr == NULL) {
1278 #ifdef DEBUG
1279  if (debug & DEBUG_REFS) {
1280  my_fprintf(stderr, "\nError : art->refptr is NULL\n");
1281  my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1282  my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1283  my_fprintf(stderr, "From : %s\n", arts[i].from);
1284  assert(arts[i].refptr != NULL);
1285  } else
1286 #endif /* DEBUG */
1287  continue;
1288  }
1289  arts[i].refptr->article = i;
1290  }
1291  }
1292 
1293  /*
1294  * Do the right thing according to the threading strategy
1295  */
1296  switch (group->attribute->thread_articles) {
1297  case THREAD_NONE:
1298  break;
1299 
1300  case THREAD_SUBJ:
1302  break;
1303 
1304  case THREAD_REFS:
1306  break;
1307 
1308  case THREAD_BOTH:
1310  collate_subjects();
1311  break;
1312 
1313  case THREAD_MULTI:
1315  break;
1316 
1317  case THREAD_PERC:
1319  break;
1320 
1321  default: /* not reached */
1322  break;
1323  }
1324 
1325  /*
1326  * Rebuild base[]
1327  */
1328  find_base(group);
1329 }
1330 
1331 
1332 static t_compfunc
1334  unsigned int sort_art_type)
1335 {
1336  switch (sort_art_type) {
1337  case SORT_ARTICLES_BY_NOTHING: /* don't sort at all */
1338  return artnum_comp;
1339 
1341  return subj_comp_desc;
1342 
1344  return subj_comp_asc;
1345 
1347  return from_comp_desc;
1348 
1350  return from_comp_asc;
1351 
1353  return date_comp_desc;
1354 
1356  return date_comp_asc;
1357 
1359  return score_comp_desc;
1360 
1362  return score_comp_asc;
1363 
1365  return lines_comp_desc;
1366 
1368  return lines_comp_asc;
1369 
1370  default:
1371  break;
1372  }
1373  return NULL;
1374 }
1375 
1376 
1377 void
1379  unsigned int sort_art_type)
1380 {
1381  t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1382 
1383  if (comp_func)
1384  SortBy(comp_func);
1385 }
1386 
1387 
1388 static void
1390  unsigned int sort_threads_type)
1391 {
1392  switch (sort_threads_type) {
1395  tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1396  break;
1397 
1400  break;
1401 
1404  break;
1405  }
1406 }
1407 
1408 
1409 /*
1410  * This is called to get header info for articles not already found in the
1411  * overview files.
1412  * Code reads (max_lineno) lines of article to catch headers like Archive-name:
1413  * which are not normally included in XOVER or even the normal block of headers.
1414  * How this is supposed to be useful when 99% of the time we'll have overview
1415  * data I don't know...
1416  * TODO: move Archive-name: parsing to article body parsing, remove the
1417  * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
1418  * TODO: currgrp->auto_save is set, otherwise it is redundant info
1419  */
1420 static t_bool
1422  FILE *fp,
1423  struct t_article *h)
1424 {
1425  char art_from_addr[HEADER_LEN];
1426  char art_full_name[HEADER_LEN];
1427  char *s, *hdr, *ptr;
1428  unsigned int lineno = 0;
1429  unsigned int max_lineno = 25;
1430  t_bool got_from, got_lines, got_received;
1431 
1432  got_from = got_lines = got_received = FALSE;
1433 
1434  while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1435  /*
1436  * Look for the end of information which tin wants to get.
1437  * Applies when reading local spool and via NNTP.
1438  */
1439 
1440  /*
1441  * as Archive-name: is placed in the body it's safe to exit
1442  * the loop if it was found. Don't mix up with Archive: from
1443  * RFC 5536.
1444  */
1445  if (lineno++ > max_lineno || h->archive)
1446  break;
1447 
1448  unfold_header(ptr);
1449  switch (my_toupper((unsigned char) *ptr)) {
1450  case 'A': /* Archive-name: optional */
1451  /*
1452  * Archive-name: {name}/{part|patch}{number}
1453  * eg, acorn/faq/part01
1454  */
1455  if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
1456  if ((s = strrchr(hdr, '/')) != NULL) {
1457  struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
1458 
1459  if (STRNCASECMPEQ(s + 1, "part", 4)) {
1460  archptr->partnum = my_strdup(s + 5);
1461  archptr->ispart = TRUE;
1462  } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
1463  archptr->partnum = my_strdup(s + 6);
1464  archptr->ispart = FALSE;
1465  } else { /* part or patch must be present */
1466  free(archptr);
1467  continue;
1468  }
1469  strtok(archptr->partnum, "\n");
1470  *s = '\0';
1471  archptr->name = hash_str(hdr);
1472  h->archive = archptr;
1473  }
1474  }
1475  break;
1476 
1477  case 'D': /* Date: mandatory */
1478  if (!h->date) {
1479  if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1480  h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1481  }
1482  break;
1483 
1484  case 'F': /* From: mandatory */
1485  if (!got_from) {
1486  if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1487  h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1488  h->from = hash_str(buffer_to_ascii(art_from_addr));
1489  if (*art_full_name)
1491  got_from = TRUE;
1492  }
1493  }
1494  break;
1495 
1496  case 'L': /* Lines: optional */
1497  if (!got_lines) {
1498  if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1499  h->line_count = atoi(hdr);
1500  got_lines = TRUE;
1501  }
1502  }
1503  break;
1504 
1505  case 'M': /* Message-ID: mandatory */
1506  if (!h->msgid) {
1507  if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1508  h->msgid = my_strdup(hdr);
1509  }
1510  break;
1511 
1512  /* for Path:-filter when reading from local spool */
1513  case 'P': /* Path: */
1514  if (!h->path) {
1515  if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1516  h->path = my_strdup(hdr);
1517  }
1518  break;
1519 
1520  case 'R': /* References: optional */
1521  if (!h->refs) {
1522  if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1523  h->refs = my_strdup(hdr);
1524  }
1525 
1526  /* Received: If found it's probably a mail article */
1527  if (!got_received) {
1528  if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
1529  max_lineno <<= 1; /* double the max number of line to read for mails */
1530  got_received = TRUE;
1531  }
1532  }
1533  break;
1534 
1535  case 'S': /* Subject: mandatory */
1536  if (!h->subject) {
1537  if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
1538 #ifdef HAVE_UNICODE_NORMALIZATION
1539  if (IS_LOCAL_CHARSET("UTF-8"))
1540  s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1541  else
1542 #endif /* HAVE_UNICODE_NORMALIZATION */
1544 
1545  h->subject = hash_str(s);
1546  free(s);
1547  }
1548  }
1549  break;
1550 
1551  case 'X': /* Xref: optional */
1552  if (!h->xref) {
1553  if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1554  h->xref = my_strdup(hdr);
1555  }
1556  break;
1557 
1558  default:
1559  break;
1560  } /* switch */
1561 
1562  } /* while */
1563 
1564 #ifdef NNTP_ABLE
1565  if (ptr)
1566  drain_buffer(fp);
1567 #endif /* NNTP_ABLE */
1568 
1569  if (tin_errno)
1570  return FALSE;
1571 
1572  /*
1573  * The son of RFC 1036 states that the following hdrs are mandatory. It
1574  * also states that Subject, Newsgroups and Path are too. Ho hum.
1575  *
1576  * What about reading mail from local spool via ~/.tin/active.mail,
1577  * they might not have a Message-ID but got_received is very likely to
1578  * be true.
1579  */
1580  if (got_from && h->date && h->msgid) {
1581  if (!h->subject)
1582  h->subject = hash_str("<No subject>");
1583 
1584 #ifdef DEBUG
1585  if (debug & DEBUG_FILTER)
1586  debug_print_header(h);
1587 #endif /* DEBUG */
1588  return TRUE;
1589  }
1590 
1591  return FALSE;
1592 }
1593 
1594 #ifdef NNTP_ABLE
1595 /*
1596  * Loop over arts[] and find ranges without Path: header
1597  * If there are any try to optimize the ranges regarding traffic consumption
1598  * Start optimization if at least MIN_CNT ranges exist
1599  * If there are more than MAX_CNT ranges after optimization, fetch all in one
1600  * big range
1601  */
1602 #define MIN_CNT 10
1603 #define MAX_CNT 50
1604 static struct t_article_range *
1605 build_range_list(
1606  t_artnum min,
1607  t_artnum max,
1608  int *range_cnt)
1609 {
1610  int i, gap_cnt = 0;
1611  struct t_article_range *res = NULL, *gap_list, *curr, *from;
1612  t_artnum new_end;
1613 
1614  new_end = T_ARTNUM_CONST(0);
1615  gap_list = my_malloc(sizeof(struct t_article_range));
1616  curr = gap_list;
1617  curr->start = min;
1618  curr->end = max;
1619  curr->cnt = T_ARTNUM_CONST(0);
1620  curr->next = NULL;
1621 
1622  for_each_art(i) {
1623  if (arts[i].artnum < min)
1624  continue;
1625  if (arts[i].artnum > max)
1626  break;
1627  if (arts[i].path) {
1628  for (; i < top_art && arts[i].path; i++)
1629  ;
1630  /*
1631  * the current art has no path -> we use this one
1632  * if we reached top_art all arts have path
1633  * so we use max
1634  */
1635  curr->start = i == top_art ? max : arts[i--].artnum;
1636  } else {
1637  for (; i < top_art && !arts[i].path; i++)
1638  ;
1639  /* the current art has path -> we use the last one */
1640  new_end = curr->end = arts[--i].artnum;
1641  }
1642  if (new_end) {
1643  curr->cnt = curr->end - curr->start + 1;
1644  curr->next = my_malloc(sizeof(struct t_article_range));
1645  curr = curr->next;
1646  curr->start = new_end;
1647  curr->end = max;
1648  curr->cnt = T_ARTNUM_CONST(0);
1649  curr->next = NULL;
1650  new_end = T_ARTNUM_CONST(0);
1651  }
1652  }
1653 
1654  curr = gap_list;
1655  while (curr && curr->cnt) {
1656  ++gap_cnt;
1657 # ifdef DEBUG
1658  if ((debug & DEBUG_NNTP) && verbose > 1)
1659  debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1660 # endif /* DEBUG */
1661  curr = curr->next;
1662  }
1663 
1664  /*
1665  * Optimize only if there are at least MIN_CNT ranges
1666  */
1667  if (gap_cnt >= MIN_CNT) {
1668  res = my_malloc(sizeof(struct t_article_range));
1669  res->start = T_ARTNUM_CONST(0);
1670  res->end = T_ARTNUM_CONST(0);
1671  res->cnt = T_ARTNUM_CONST(0);
1672  res->next = NULL;
1673 
1674  from = gap_list;
1675  curr = res;
1676  while (from) {
1677  curr->start = from->start;
1678  curr->end = from->end;
1679  curr->cnt = from->cnt;
1680  if ((from = from->next)) {
1681  /*
1682  * If the next range is grater then the gap between the current
1683  * one and the next one we build a new range including the
1684  * current one, the next one and the gap between
1685  */
1686  while (from && from->cnt >= from->start - curr->end - 1) {
1687  curr->end = from->end;
1688  from = from->next;
1689  }
1690  curr->cnt = curr->end - curr->start + 1;
1691  curr->next = my_malloc(sizeof(struct t_article_range));
1692  curr = curr->next;
1693  curr->start = T_ARTNUM_CONST(0);
1694  curr->end = T_ARTNUM_CONST(0);
1695  curr->cnt = T_ARTNUM_CONST(0);
1696  curr->next = NULL;
1697  }
1698  }
1699  }
1700 
1701  /*
1702  * If there are less then MIN_CNT ranges
1703  * no res is build -> return the original list
1704  */
1705  if (res) {
1706  while (gap_list) {
1707  curr = gap_list;
1708  gap_list = curr->next;
1709  free(curr);
1710  }
1711  } else
1712  res = gap_list;
1713 
1714  curr = res;
1715  gap_cnt = 0;
1716  while (curr && curr->cnt) {
1717  ++gap_cnt;
1718 # ifdef DEBUG
1719  if ((debug & DEBUG_NNTP) && verbose > 1)
1720  debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1721 # endif /* DEBUG */
1722  curr = curr->next;
1723  }
1724 
1725  if (gap_cnt >= MAX_CNT) {
1726  curr = res;
1727  while (curr->next && curr->next->cnt) {
1728  res->end = curr->next->end;
1729  curr->next->cnt = 0;
1730  curr = curr->next;
1731  }
1732  res->cnt = res->end - res->start + 1;
1733  gap_cnt = 1;
1734 # ifdef DEBUG
1735  if ((debug & DEBUG_NNTP) && verbose > 1)
1736  debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1737 # endif /* DEBUG */
1738  }
1739  *range_cnt = gap_cnt;
1740 
1741  return res;
1742 }
1743 
1744 
1745 /*
1746  * Fetch the Path header in case we want to filter on that in the given group
1747  *
1748  * Try [X]HDR first, then XPAT
1749  */
1750 static t_bool
1751 get_path_header(
1752  int cur,
1753  int cnt,
1754  struct t_group *group,
1755  t_artnum min,
1756  t_artnum max)
1757 {
1758  FILE *fp = NULL;
1759  char *prep_msg;
1760  char *buf, *ptr;
1761  char cmd[NNTP_STRLEN];
1762  t_artnum artnum, i;
1763  t_bool found = FALSE;
1764  static t_bool supported = TRUE; /* assume HDR || XPAT works */
1765 
1766  if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1767  return FALSE;
1768 
1769 # ifdef DEBUG
1770  if ((debug & DEBUG_NNTP) && verbose > 1)
1771  debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1772 # endif /* DEBUG */
1773 
1775  int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1776  switch (j) {
1777  case 215:
1778  while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1779 # ifdef DEBUG
1780  if (debug & DEBUG_NNTP)
1781  debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1782 # endif /* DEBUG */
1784  strcat(nntp_caps.headers_range, ptr);
1785  strcat(nntp_caps.headers_range, "\n");
1786  }
1787  break;
1788 
1789  default:
1790  break;
1791  }
1792  }
1793 
1794  /* does HDR return Path? */
1795  if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1796  do {
1797  if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1798  found = TRUE;
1799  } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1800  }
1801 
1802  if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1803  if (min == max)
1804  snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1805  else
1806  snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1807  fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1808  if (!nntp_caps.hdr && fp)
1809  nntp_caps.hdr = TRUE;
1810  } else if (nntp_caps.xpat) {
1811  if (min == max)
1812  snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1813  else
1814  snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1815  fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1816  }
1817 
1818  if (fp) {
1819  int j = 0;
1820 
1821  prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1822  while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1823 # ifdef DEBUG
1824  if ((debug & DEBUG_NNTP) && verbose)
1825  debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1826 # endif /* DEBUG */
1827  if ((ptr = tin_strtok(buf, " ")) == NULL)
1828  continue;
1829  artnum = atoartnum(ptr);
1830  if ((ptr = tin_strtok(NULL, " ")) == NULL)
1831  continue;
1832  for (i = j; i < top_art; i++) {
1833  if (arts[i].artnum == artnum) {
1834  FreeIfNeeded(arts[i].path);
1835  arts[i].path = my_strdup(ptr);
1836  j = i;
1837  break;
1838  }
1839  }
1840  if (++artnum % MODULO_COUNT_NUM == 0)
1841  show_progress(prep_msg, artnum - min, max - min);
1842  }
1843  free(prep_msg);
1844  return supported;
1845  }
1846 
1847  /* !fp */
1848  supported = FALSE;
1849  if (nntp_caps.xpat)
1850  nntp_caps.xpat = FALSE;
1851  /* as nntp_caps.hdr may work with other headers we don't disable it */
1852 
1853 # ifdef DEBUG
1854  if ((debug & DEBUG_NNTP) && verbose > 1)
1855  debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
1856 # endif /* DEBUG */
1857  return supported;
1858 }
1859 #endif /* NNTP_ABLE */
1860 
1861 
1862 /*
1863  * Read in an overview index file. Fields are separated by TAB.
1864  * return the number of expired articles encountered or -1 if the user aborted
1865  * the read
1866  * 'top' is set to the highest artnum read
1867  * If 'local' is set then always open local overview cache in preference to
1868  * using NNTP XOVER
1869  *
1870  * Format (mandatory as far as line count [RFC2980]):
1871  * 1. article number (ie. 183) [mandatory]
1872  * 2. Subject: line (ie. Which newsreader?) [mandatory]
1873  * 3. From: line (ie. iain@ecrc.de) [mandatory]
1874  * 4. Date: line (rfc822 format) [mandatory]
1875  * 5. MessageID: (ie. <123@ether.net>) [mandatory]
1876  * 6. References: (ie. <message-id> ....) [optional]
1877  * 7. Byte count (Skipped - not used) [mandatory]
1878  * 8. Line count (ie. 23) [mandatory]
1879  * 9. Xref: line (ie. alt.test:389) [optional]
1880  */
1881 static int
1883  struct t_group *group,
1884  t_artnum min,
1885  t_artnum max,
1886  t_artnum *top,
1887  t_bool local,
1888  t_bool *rebuild_cache)
1889 {
1890  FILE *fp;
1891  char *ptr;
1892  char *q;
1893  char *buf;
1894  char *group_msg;
1895  char art_full_name[HEADER_LEN];
1896  char art_from_addr[HEADER_LEN];
1897  unsigned int count;
1898  int expired = 0;
1899  t_artnum artnum;
1900  t_bool path_found = FALSE, path_in_ofmt = FALSE;
1901  struct t_article *art;
1902  size_t over_fields = 1;
1903 
1904  /*
1905  * open the overview file (whether it be local or via nntp)
1906  */
1907  if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1908  return expired;
1909 
1910  if (group->xmax > max)
1911  group->xmax = max;
1912 
1913  group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1914 
1915  /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1916  if (ofmt) {
1917  for (; ofmt[over_fields].name; over_fields++) {
1918  if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1919  path_in_ofmt = TRUE;
1920  }
1921  }
1922 
1923  if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1924  ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1925  ofmt[0].type = OVER_T_INT;
1926  ofmt[0].name = my_strdup("Artnum:");
1927  ofmt[1].type = OVER_T_STRING;
1928  ofmt[1].name = my_strdup("Subject:");
1929  ofmt[2].type = OVER_T_STRING;
1930  ofmt[2].name = my_strdup("From:");
1931  ofmt[3].type = OVER_T_STRING;
1932  ofmt[3].name = my_strdup("Date:");
1933  ofmt[4].type = OVER_T_STRING;
1934  ofmt[4].name = my_strdup("Message-ID:");
1935  ofmt[5].type = OVER_T_STRING;
1936  ofmt[5].name = my_strdup("References:");
1937  ofmt[6].type = OVER_T_INT;
1938  ofmt[6].name = my_strdup("Bytes:");
1939  ofmt[7].type = OVER_T_INT;
1940  ofmt[7].name = my_strdup("Lines:");
1941  ofmt[8].type = OVER_T_ERROR;
1942  ofmt[8].name = NULL;
1943  over_fields = 7;
1944  }
1945 
1946  while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1947 #ifdef DEBUG
1948  if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1949  debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1950 #endif /* DEBUG */
1951 
1952  if (need_resize) {
1954  need_resize = cNo;
1955  }
1956 
1957  /*
1958  * Read artnum
1959  */
1960  if ((ptr = tin_strtok(buf, "\t")) == NULL)
1961  continue;
1962 
1963  /*
1964  * read the article number, guaranteed to be the first field
1965  */
1966  artnum = atoartnum(ptr);
1967 
1968  /*
1969  * artnum field invalid/corrupt or is 1st line of local cached overview
1970  * (group name)
1971  */
1972  if (artnum <= 0)
1973  continue;
1974 
1975  /*
1976  * skip artnums below the given minimum (getart_limit)
1977  */
1978  if (artnum < min)
1979  continue;
1980 
1981  /*
1982  * Check to make sure article in nov file has not expired in group
1983  */
1984  if (artnum < group->xmin) {
1985  expired++;
1986  continue;
1987  }
1988 
1989  /*
1990  * artnum in overview data higher than groups high mark
1991  *
1992  * TODO: - warn user about broken overviews?
1993  * - try to parse the Xref:-line to get the correct artnum
1994  * - see also parse_unread_arts()
1995  */
1996  if (artnum > group->xmax)
1997  continue;
1998 
1999  if (top_art >= max_art)
2000  expand_art();
2001 
2002  art = &arts[top_art];
2003  set_article(art);
2004  art->artnum = *top = artnum;
2005 
2006  /*
2007  * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
2008  * to check for additions like we do with xref_supported
2009  */
2010  for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
2011  /* skip unexpected tailing fields */
2012  if (count > over_fields) {
2013 #ifdef DEBUG
2014  if ((debug & DEBUG_NNTP) && verbose > 1)
2015  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
2016 #endif /* DEBUG */
2017 
2018  /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
2019  if (count == over_fields + 1) {
2020  if (!strncasecmp(ptr, "Xref: ", 6)) {
2021 #ifdef DEBUG
2022  if ((debug & DEBUG_NNTP) && verbose > 1)
2023  debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
2024 #endif /* DEBUG */
2025  over_fields++;
2026  ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2027  ofmt[over_fields].type = OVER_T_FSTRING;
2028  ofmt[over_fields].name = my_strdup("Xref:");
2029  ofmt[over_fields + 1].type = OVER_T_ERROR;
2030  ofmt[over_fields + 1].name = NULL;
2031  xref_supported = TRUE;
2032  } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
2033 #ifdef DEBUG
2034  if ((debug & DEBUG_NNTP) && verbose > 1)
2035  debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
2036 #endif /* DEBUG */
2037  over_fields++;
2038  ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2039  ofmt[over_fields].type = OVER_T_FSTRING;
2040  ofmt[over_fields].name = my_strdup("Path:");
2041  ofmt[over_fields + 1].type = OVER_T_ERROR;
2042  ofmt[over_fields + 1].name = NULL;
2043  xref_supported = TRUE;
2044  } else
2045  continue;
2046  } else
2047  continue;
2048  }
2049 
2050  /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
2051  if (expensive_over_parse) { /* strange order */
2052  /* mandatory fields */
2053  if (ofmt[count].type == OVER_T_STRING) {
2054  if (!strcasecmp(ofmt[count].name, "Subject:")) {
2055  if (*ptr) {
2056 #ifdef HAVE_UNICODE_NORMALIZATION
2057  if (IS_LOCAL_CHARSET("UTF-8"))
2058  q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2059  else
2060 #endif /* HAVE_UNICODE_NORMALIZATION */
2062 
2063  art->subject = hash_str(q);
2064  free(q);
2065  } else {
2066  art->subject = hash_str("");
2067 #ifdef DEBUG
2068  if ((debug & DEBUG_NNTP) && verbose > 1)
2069  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2070 #endif /* DEBUG */
2071  }
2072  continue;
2073  }
2074 
2075  if (!strcasecmp(ofmt[count].name, "From:")) {
2076  if (*ptr) {
2077  art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2078  art->from = hash_str(buffer_to_ascii(art_from_addr));
2079  if (*art_full_name)
2080  art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2081  } else {
2082  art->from = hash_str("");
2083 #ifdef DEBUG
2084  if ((debug & DEBUG_NNTP) && verbose > 1)
2085  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2086 #endif /* DEBUG */
2087  }
2088  continue;
2089  }
2090 
2091  if (!strcasecmp(ofmt[count].name, "Date:")) {
2092  art->date = parsedate(ptr, (TIMEINFO *) 0);
2093 #ifdef DEBUG
2094  if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2095  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2096 #endif /* DEBUG */
2097  continue;
2098  }
2099 
2100  if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2101  if (*ptr) {
2102  FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2103  art->msgid = my_strdup(ptr);
2104  } else {
2105  art->msgid = NULL;
2106 #ifdef DEBUG
2107  if ((debug & DEBUG_NNTP) && verbose > 1)
2108  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2109 #endif /* DEBUG */
2110  }
2111  continue;
2112  }
2113 
2114  if (!strcasecmp(ofmt[count].name, "References:")) {
2115  if (*ptr) {
2116  FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2117  art->refs = my_strdup(ptr);
2118  } else
2119  art->refs = NULL;
2120  continue;
2121  }
2122 
2123  /*
2124  * non std. fields when doing
2125  * expensive overview parsing (very
2126  * rare, just happens if RFC 3977
2127  * 8.4.2 is violated) go here
2128  */
2129  /* for Path:-filter */
2130  if (!strcasecmp(ofmt[count].name, "Path:")) {
2131  if (!path_found)
2132  path_found = TRUE;
2133  if (*ptr) {
2134  FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2135  art->path = my_strdup(ptr);
2136  } else
2137  art->path = NULL;
2138  continue;
2139  }
2140  }
2141  /* metadata fields */
2142  if (ofmt[count].type == OVER_T_INT) {
2143  if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2144  if (*ptr) {
2145 #ifdef DEBUG
2146  if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2147  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2148 #endif /* DEBUG */
2149  }
2150  continue;
2151  }
2152 
2153  if (!strcasecmp(ofmt[count].name, "Lines:")) {
2154  if (*ptr) {
2155  if (isdigit((unsigned char) *ptr))
2156  art->line_count = atoi(ptr);
2157  else {
2158  art->line_count = 0;
2159 #ifdef DEBUG
2160  if ((debug & DEBUG_NNTP) && verbose > 1)
2161  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2162 #endif /* DEBUG */
2163  }
2164  } else
2165  art->line_count = 0;
2166  continue;
2167  }
2168  }
2169  } else { /* first 7 fields are in RFC 3977 order */
2170  switch (count) {
2171  case 1: /* Subject: */
2172  if (*ptr) {
2173 #ifdef HAVE_UNICODE_NORMALIZATION
2174  if (IS_LOCAL_CHARSET("UTF-8"))
2175  q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2176  else
2177 #endif /* HAVE_UNICODE_NORMALIZATION */
2179 
2180  art->subject = hash_str(q);
2181  free(q);
2182  } else {
2183  art->subject = hash_str("");
2184 #ifdef DEBUG
2185  if ((debug & DEBUG_NNTP) && verbose > 1)
2186  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2187 #endif /* DEBUG */
2188  }
2189  break;
2190 
2191  case 2: /* From: */
2192  if (*ptr) {
2193  art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2194  art->from = hash_str(buffer_to_ascii(art_from_addr));
2195  if (*art_full_name)
2196  art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2197  } else {
2198  art->from = hash_str("");
2199 #ifdef DEBUG
2200  if ((debug & DEBUG_NNTP) && verbose > 1)
2201  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2202 #endif /* DEBUG */
2203  }
2204  break;
2205 
2206  case 3: /* Date: */
2207  art->date = parsedate(ptr, (TIMEINFO *) 0);
2208 #ifdef DEBUG
2209  if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2210  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2211 #endif /* DEBUG */
2212  break;
2213 
2214  case 4: /* Message-ID: */
2215  if (*ptr)
2216  art->msgid = my_strdup(ptr);
2217  else {
2218  art->msgid = NULL;
2219 #ifdef DEBUG
2220  if ((debug & DEBUG_NNTP) && verbose > 1)
2221  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2222 #endif /* DEBUG */
2223  }
2224  break;
2225 
2226  case 5: /* References: */
2227  if (*ptr)
2228  art->refs = my_strdup(ptr);
2229  else
2230  art->refs = NULL;
2231  break;
2232 
2233  case 6: /* :bytes || Bytes: */
2234  if (*ptr) {
2235 #ifdef DEBUG
2236  if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2237  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2238 #endif /* DEBUG */
2239  }
2240  break;
2241 
2242  case 7: /* :lines || Lines: */
2243  if (*ptr) {
2244  if (isdigit((unsigned char) *ptr))
2245  art->line_count = atoi(ptr);
2246  else {
2247  art->line_count = 0;
2248 #ifdef DEBUG
2249  if ((debug & DEBUG_NNTP) && verbose > 1)
2250  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2251 #endif /* DEBUG */
2252  }
2253  } else
2254  art->line_count = 0;
2255  break;
2256 
2257  default:
2258  break;
2259  }
2260  }
2261 
2262  /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2263  if (ofmt[count].type == OVER_T_FSTRING) {
2264  if (*ptr) {
2265  if (!strcasecmp(ofmt[count].name, "Xref:")) {
2266  if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2267  FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2268  art->xref = my_strdup(q);
2269  }
2270 #ifdef DEBUG
2271  else {
2272  if ((debug & DEBUG_NNTP) && verbose > 1)
2273  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2274  }
2275 #endif /* DEBUG */
2276  continue;
2277  }
2278  /*
2279  * handling of addition overview fields
2280  * goes here
2281  */
2282 #ifdef DEBUG
2283  if ((debug & DEBUG_NNTP) && verbose > 1)
2284  debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2285 #endif /* DEBUG */
2286  /* if we're lucky we've Path in NOV */
2287  /*
2288  * if reading locally cached overview data try
2289  * path regardless of the server OVERVIEW.FMT
2290  */
2291  if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2292  if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2293  if (!path_found)
2294  path_found = TRUE;
2295  FreeIfNeeded(art->path);
2296  art->path = my_strdup(q);
2297 #ifdef DEBUG
2298  if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2299  debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2300 #endif /* DEBUG */
2301 
2302  }
2303  continue;
2304  }
2305  }
2306  continue;
2307  }
2308  }
2309 
2310  /*
2311  * RFC says Message-ID is mandatory in newsgroups (but not in
2312  * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2313  * threading in mailgroups
2314  */
2315  if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2316  continue;
2317 
2318  /* we might lose accuracy here, but that shouldn't hurt */
2319  if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2320  show_progress(group_msg, artnum - min, max - min);
2321 
2322  top_art++; /* Basically this statement commits the article */
2323  }
2324 
2325  free(group_msg);
2326  TIN_FCLOSE(fp);
2327 
2328  if (tin_errno)
2329  return -1;
2330 
2331 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2333  char cbuf[HEADER_LEN];
2334  int i;
2335  static t_bool found;
2336  static t_bool first = TRUE;
2337 
2338  if (first) {
2339  found = TRUE;
2340  /*
2341  * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2342  */
2344  if (!*nntp_caps.headers_range) {
2345  i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2346 
2347  found = FALSE;
2348  switch (i) {
2349  case 215:
2350  while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2351 # ifdef DEBUG
2352  if (debug & DEBUG_NNTP)
2353  debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2354 # endif /* DEBUG */
2355  if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2356  found = TRUE;
2358  strcat(nntp_caps.headers_range, ptr);
2359  strcat(nntp_caps.headers_range, "\n");
2360  }
2361  break;
2362 
2363  default:
2364  break;
2365  }
2366  first = FALSE;
2367  } else {
2368  found = FALSE;
2369  if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
2370  do {
2371  if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2372  found = TRUE;
2373  } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2374  }
2375  }
2376  }
2377  }
2378 
2379  if (found) {
2380  snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2381  group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2382  if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2383  while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2384 # ifdef DEBUG
2385  if ((debug & DEBUG_NNTP) && verbose)
2386  debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2387 # endif /* DEBUG */
2388 
2389  artnum = atoartnum(ptr);
2390  if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2391  continue;
2392  art = &arts[top_art];
2393  set_article(art);
2394  if (!art->xref && !strstr(ptr, "(none)")) {
2395  if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2396  continue;
2397  ptr = q;
2398  while (*ptr && isspace((int) *ptr))
2399  ptr++;
2400  q = strchr(ptr, '\n');
2401  if (q)
2402  *q = '\0';
2403  art->xref = my_strdup(ptr);
2404  }
2405  /* we might lose accuracy here, but that shouldn't hurt */
2406  if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2407  show_progress(group_msg, artnum - min, max - min);
2408  }
2409  }
2410  free(group_msg);
2411  }
2412  }
2413 #endif /* NNTP_ABLE && XHDR_XREF */
2414 
2415  if (local) {
2416 #ifdef NNTP_ABLE
2417  if (filter_on_path(group)) {
2418  struct t_article_range *ranges, *curr;
2419  t_bool supported = TRUE;
2420  int curr_range, range_cnt;
2421 
2422  /*
2423  * Get the ranges without Path: header and try to fetch the
2424  * headers
2425  */
2426  if ((ranges = build_range_list(min, *top, &range_cnt))) {
2427  curr = ranges;
2428  curr_range = 1;
2429  while (curr && supported) {
2430  if (curr->cnt)
2431  supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2432  curr = curr->next;
2433  }
2434  if (!supported && path_in_ofmt) {
2435  /*
2436  * fetching Path: headers via [X]HDR or XPAT has failed
2437  * Path: is in the servers overview so let the next
2438  * read_overview() fetch them
2439  */
2440  free_art_array();
2441  free_msgids();
2442  top_art = 0;
2443  *top = T_ARTNUM_CONST(0);
2444  expired = 0;
2445  }
2446  *rebuild_cache = TRUE;
2447  while (ranges) {
2448  curr = ranges;
2449  ranges = curr->next;
2450  free(curr);
2451  }
2452  }
2453  }
2454 #endif /* NNTP_ABLE */
2455  } else
2456  if (!path_found && filter_on_path(group)) {
2457 #ifdef NNTP_ABLE
2458  if (!get_path_header(1, 1, group, min, *top))
2459 #endif /* NNTP_ABLE */
2461  }
2462  return expired;
2463 }
2464 
2465 
2466 /*
2467  * Write an Nov/Xover index file. Fields are separated by '\t'.
2468  *
2469  * Format:
2470  * 1. article number (ie. 183) [mandatory]
2471  * 2. Subject: line (ie. Which newsreader?) [mandatory]
2472  * 3. From: line (ie. iain@ecrc.de) [mandatory]
2473  * 4. Date: line (rfc822 format) [mandatory]
2474  * 5. MessageID: (ie. <123@ether.net>) [mandatory]
2475  * 6. References: (ie. <message-id> ....) [optional]
2476  * 7. Byte count (Skipped - not used) [mandatory]
2477  * 8. Line count (ie. 23) [mandatory]
2478  * 9. Xref: line (ie. alt.test:389) [optional]
2479  *
2480  * TODO: as we don't use the original data, we currently can't store
2481  * the data (from/subject) in the original charset (we don't store
2482  * that info). this has the advantage that we can avoid raw 8bit data
2483  * in our overviews, but the disadvantage that we might store the data
2484  * with a wrong charset and thus lose information. a similar problem
2485  * exists with the data for the from:-line, we don't store it in the
2486  * original format, whenever our from-parser (partially) fails we'll
2487  * lose information in our overviews (but those couldn't be handled
2488  * by tin anyway, so this is not a real problem).
2489  * long-term solution: store the original data in the overview
2490  * (tin has to handle raw 8bit data and other ugly stuff in the
2491  * overviews anyway and thus we preserver as much info as possible)
2492  * this would require some changes in read_overview() and
2493  * parse_headers(): don't do the decoding/unfolding there, but in a
2494  * second pass right after write_overview(), or two additional fields
2495  * which hold the raw data for from/subject. the latter has the
2496  * disadvantage that it costs (much) more memory.
2497  */
2498 static void
2500  struct t_group *group)
2501 {
2502  FILE *fp;
2503  int i;
2504  struct t_article *article;
2505 #ifdef CHARSET_CONVERSION
2506  int c = -1;
2507 #endif /* CHARSET_CONVERSION */
2508 
2509  /*
2510  * Can't write or caching is off or getart_limit is set
2511  */
2513  return;
2514 
2515  if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2516  return;
2517 
2520 
2521  /*
2522  * Needed to preserve uniqueness in hashed private overview files
2523  */
2524  fprintf(fp, "%s\n", group->name);
2525 
2526 #ifdef CHARSET_CONVERSION
2527  /* get undeclared_charset number if required */
2528  if (group->attribute->undeclared_charset) {
2529  for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2530  if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2531  c = i;
2532  break;
2533  }
2534  }
2535  }
2536 #endif /* CHARSET_CONVERSION */
2537 
2538  if (verbose && batch_mode) /* -> lang.c */
2539  wait_message(0, _("Writing %s\n"), group->name);
2540 
2541  for_each_art(i) {
2542  char *p;
2543  char *q, *ref;
2544 
2545  article = &arts[i];
2546 
2547  if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2548  ref = NULL;
2549 
2550  if (!group->attribute->post_8bit_header) { /* write encoded data */
2551  /*
2552  * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2553  * here and in print_from() in the CHARSET_CONVERSION case.
2554  * note that this requires something like
2555  * buffer_to_network(article->subject, "UTF-8");
2556  * right before the rfc1522_encode() call.
2557  *
2558  * if we would cache the original undecoded data, we could
2559  * ignore stuff like this.
2560  */
2562  /* as the subject might now be folded we have to unfold it */
2563  unfold_header(p);
2564  } else { /* raw data */
2565  p = my_strdup(article->subject);
2566 #ifdef CHARSET_CONVERSION
2567  if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2568  buffer_to_network(p, c);
2569 #endif /* CHARSET_CONVERSION */
2570  }
2571 
2572  /*
2573  * replace any '\t's with ' ' in the references-data
2574  *
2575  * TODO: nntpext-draft might come up with a new scheme:
2576  * For all fields, the value is processed by first
2577  * removing all US-ASCII CRLF pairs and then replacing
2578  * each remaining US-ASCII NUL, TAB, CR, or LF character
2579  * with a single US-ASCII space (for example, CR LF LF TAB
2580  * will become two spaces).
2581  */
2582  if (article->refs) {
2583  ref = q = my_strdup(article->refs);
2584  while (*q) {
2585  if (*q == '\t')
2586  *q = ' ';
2587  q++;
2588  }
2589  }
2590 
2591  fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2592  article->artnum,
2593  p,
2594 #ifdef CHARSET_CONVERSION
2595  print_from(group, article, c),
2596 #else
2597  print_from(group, article, -1),
2598 #endif /* CHARSET_CONVERSION */
2599  print_date(article->date),
2600  BlankIfNull(article->msgid),
2601  BlankIfNull(ref),
2602  0, /* bytes */
2603  article->line_count);
2604 
2605  if (article->xref)
2606  fprintf(fp, "\tXref: %s", article->xref);
2607 
2608  if (article->path)
2609  fprintf(fp, "\tPath: %s", article->path);
2610 
2611  fprintf(fp, "\n");
2612  free(p);
2613  if (article->refs) {
2614  FreeIfNeeded(ref);
2615  q = NULL;
2616  }
2617  }
2618  if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
2619  show_progress(_("Writing overview cache..."), i, top_art);
2620  }
2621 #ifdef HAVE_FCHMOD
2622  fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2623  /*
2624  * TODO:
2625  * add code for !HAVE_FCHMOD && HAVE_CHMOD
2626  */
2627 #endif /* HAVE_FCHMOD */
2628  fclose(fp);
2629 }
2630 
2631 
2632 /*
2633  * A complex little function to determine the correct overview index file
2634  * according to 'mode' (read or write)
2635  * NULL is returned if the current setup dictates otherwise
2636  *
2637  * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2638  * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2639  *
2640  * Both of these are hashed
2641  *
2642  * GROUP_TYPE_NEWS index files are a little bit more complex
2643  *
2644  * When hashing the index filename will be in format number.number.
2645  * Hashing the groupname gets a number. See if that #.1 file exists;
2646  * if so, read first line. Is this the group we want? If no, try #.2.
2647  * Repeat until no such file or we find an existing file that matches
2648  * our group. Return pointer to path or NULL if not found.
2649  */
2650 static char *
2652  struct t_group *group,
2653  int mode)
2654 {
2655  FILE *fp;
2656  const char *dir;
2657  char buf[PATH_LEN];
2658  int i;
2659  struct stat sb;
2660  unsigned long hash;
2661  static char nov_file[PATH_LEN];
2662  static t_bool once_only = FALSE; /* Trap things that are done only 1 time */
2663 
2664  if (group == NULL || (mode != R_OK && mode != W_OK))
2665  return NULL;
2666 
2667  switch (group->type) {
2668  case GROUP_TYPE_MAIL:
2669  dir = index_maildir;
2670  break;
2671 
2672  case GROUP_TYPE_SAVE:
2673  dir = index_savedir;
2674  break;
2675 
2676  case GROUP_TYPE_NEWS:
2677  /*
2678  * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2679  * about [X]OVER are handled in nntp_open()
2680  */
2681 
2682  /*
2683  * When reading via NNTP, system wide overviews are irrelevant, of
2684  * course, and the private overview filename will be the same for
2685  * both reading and writing.
2686  *
2687  * When working locally, we only use a private cache for reading
2688  * if requested and when system wide overviews don't already exist.
2689  * When writing then only private overviews can be used since
2690  * updating system wide overviews is not safe wrt locking etc.
2691  *
2692  * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2693  */
2694 #ifndef NNTP_ONLY
2695  if (!read_news_via_nntp) {
2696  make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2697  joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2698  if (access(nov_file, R_OK) == 0) {
2699  if (mode == R_OK)
2700  return nov_file; /* Use system wide overviews */
2701  else
2702  return NULL; /* Don't write cache in this case */
2703  }
2704  }
2705 #endif /* !NNTP_ONLY */
2706 
2707  /*
2708  * We only get here when private overviews are going to be used
2709  * Go no further if they are explicitly turned off
2710  */
2712  return NULL;
2713 
2714  /*
2715  * Append -<nntpserver> to private cache dir
2716  */
2717  if (!once_only && nntp_server) {
2718  size_t sp, ln = strlen(index_newsdir);
2719 
2720  if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
2721  char *srv = my_strdup(nntp_server);
2722 
2723  strcat(index_newsdir, "-");
2724  sp--;
2725  ln++;
2726  str_lwr(srv);
2727  my_strncpy(index_newsdir + ln, srv, sp);
2728  free(srv);
2729  }
2730  once_only = TRUE;
2731  }
2732 
2733  /*
2734  * Only try to set up the private cache when writing. If it
2735  * doesn't exist yet, then ergo we can't read from it.
2736  * The cache will be checked/created on every write; a previous
2737  * bug report complained that this was not the case
2738  */
2739  if (stat(index_newsdir, &sb) == -1) { /* Private cache doesn't exist */
2740  if (mode == R_OK)
2741  return NULL;
2742  if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2743  return NULL;
2744  } else {
2745  if (!S_ISDIR(sb.st_mode))
2746  return NULL;
2747  }
2748 
2749  /*
2750  * Update the newsgroups cache to point to the new location
2751  * now that we know it is valid
2752  */
2753  if (!once_only)
2755 
2756  dir = index_newsdir;
2757  break;
2758 
2759  default: /* not reached */
2760  return NULL;
2761  }
2762 
2763  /*
2764  * We only get here if writing to a private overview.
2765  * These always have hashed filenames.
2766  * Try <hash>.<seqno> and check the group name tagline until
2767  * matching index file is found. If not found return next unused
2768  * filename
2769  */
2770  hash = hash_groupname(group->name);
2771 
2772  for (i = 1; ; i++) {
2773  char *ptr;
2774 
2775  snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2776  joinpath(nov_file, sizeof(nov_file), dir, buf);
2777 
2778  if ((fp = fopen(nov_file, "r")) == NULL)
2779  break;
2780 
2781  /*
2782  * No group name header, so not a valid index file => overwrite it
2783  */
2784  if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2785  fclose(fp);
2786  break;
2787  }
2788  fclose(fp);
2789 
2790  if ((ptr = strrchr(buf, '\n')) != NULL)
2791  *ptr = '\0';
2792 
2793  if (strcmp(buf, group->name) == 0)
2794  break;
2795  }
2796 
2797  return nov_file;
2798 }
2799 
2800 
2801 /*
2802  * Run the index file updater only for the groups we've loaded.
2803  */
2804 void
2806  t_bool catchup)
2807 {
2808  int i, j, k = 0;
2809  time_t beg_epoch = 0;
2810  struct t_article *art;
2811  struct t_group *group;
2812 
2813  if (verbose)
2814  (void) time(&beg_epoch);
2815 
2816  /*
2817  * loop through groups and update any required index files
2818  */
2819  for (i = 0; i < selmenu.max; i++) {
2820  group = &active[my_group[i]];
2821  /*
2822  * FIXME: workaround to get a valid CURR_GROUP
2823  * it also points to the currently processed group so that
2824  * the correct attributes are used
2825  * The correct fix is to get rid of CURR_GROUP
2826  */
2827  selmenu.curr = i;
2828 
2829  if (group->bogus || !group->subscribed)
2830  continue;
2831 
2832  if (!index_group(group)) {
2833  for_each_art(j) {
2834  art = &arts[j];
2835  FreeAndNull(art->refs);
2836  FreeAndNull(art->msgid);
2837  }
2838  continue;
2839  }
2840 
2841  k++;
2842 
2843  if (verbose) {
2844  my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2845  my_flush();
2846  }
2847 
2848  if (catchup) {
2849  for_each_art(j)
2850  art_mark(group, &arts[j], ART_READ);
2851  }
2852  }
2853 
2854  if (verbose) {
2856  (catchup ? _(txt_caughtup) : _(txt_updated)), k,
2857  PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2858  }
2859 }
2860 
2861 
2862 static int
2864  t_comptype p1,
2865  t_comptype p2)
2866 {
2867  const struct t_article *s1 = (const struct t_article *) p1;
2868  const struct t_article *s2 = (const struct t_article *) p2;
2869 
2870  /*
2871  * s1->artnum less than s2->artnum
2872  */
2873  if (s1->artnum < s2->artnum)
2874  return -1;
2875 
2876  /*
2877  * s1->artnum greater than s2->artnum
2878  */
2879  if (s1->artnum > s2->artnum)
2880  return 1;
2881 
2882  return 0;
2883 }
2884 
2885 
2886 /*
2887  * return result of strcmp (reversed for descending)
2888  */
2889 static int
2891  t_comptype p1,
2892  t_comptype p2)
2893 {
2894  int retval;
2895  const struct t_article *s1 = (const struct t_article *) p1;
2896  const struct t_article *s2 = (const struct t_article *) p2;
2897 
2898  if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2899  return retval;
2900 
2901  return s1->date - s2->date > 0 ? 1 : -1;
2902 }
2903 
2904 
2905 static int
2907  t_comptype p1,
2908  t_comptype p2)
2909 {
2910  int retval;
2911  const struct t_article *s1 = (const struct t_article *) p1;
2912  const struct t_article *s2 = (const struct t_article *) p2;
2913 
2914  if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2915  return retval;
2916 
2917  return s1->date - s2->date > 0 ? 1 : -1;
2918 }
2919 
2920 
2921 /*
2922  * return result of strcmp (reversed for descending)
2923  */
2924 static int
2926  t_comptype p1,
2927  t_comptype p2)
2928 {
2929  int retval;
2930  const struct t_article *s1 = (const struct t_article *) p1;
2931  const struct t_article *s2 = (const struct t_article *) p2;
2932 
2933  if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2934  return retval;
2935 
2936  return s1->date - s2->date > 0 ? 1 : -1;
2937 }
2938 
2939 
2940 static int
2942  t_comptype p1,
2943  t_comptype p2)
2944 {
2945  int retval;
2946  const struct t_article *s1 = (const struct t_article *) p1;
2947  const struct t_article *s2 = (const struct t_article *) p2;
2948 
2949  if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2950  return retval;
2951 
2952  return s1->date - s2->date > 0 ? 1 : -1;
2953 }
2954 
2955 
2956 /*
2957  * Works like strcmp() for comparing time_t type values
2958  * Return codes:
2959  * -1: If p1 is before p2
2960  * 0: If they are the same time
2961  * 1: If p1 is after p2
2962  * If the sort order is _not_ DATE_ASCEND then the sense of the above
2963  * is reversed.
2964  */
2965 static int
2967  t_comptype p1,
2968  t_comptype p2)
2969 {
2970  const struct t_article *s1 = (const struct t_article *) p1;
2971  const struct t_article *s2 = (const struct t_article *) p2;
2972 
2973  /*
2974  * s1->date less than s2->date
2975  */
2976  if (s1->date < s2->date)
2977  return -1;
2978 
2979  /*
2980  * s1->date greater than s2->date
2981  */
2982  if (s1->date > s2->date)
2983  return 1;
2984 
2985  return 0;
2986 }
2987 
2988 
2989 static int
2991  t_comptype p1,
2992  t_comptype p2)
2993 {
2994  const struct t_article *s1 = (const struct t_article *) p1;
2995  const struct t_article *s2 = (const struct t_article *) p2;
2996 
2997  /*
2998  * s2->date less than s1->date
2999  */
3000  if (s2->date < s1->date)
3001  return -1;
3002 
3003  /*
3004  * s2->date greater than s1->date
3005  */
3006  if (s2->date > s1->date)
3007  return 1;
3008 
3009  return 0;
3010 }
3011 
3012 
3013 /*
3014  * Same again, but for art[].score
3015  */
3016 static int
3018  t_comptype p1,
3019  t_comptype p2)
3020 {
3021  const struct t_article *s1 = (const struct t_article *) p1;
3022  const struct t_article *s2 = (const struct t_article *) p2;
3023 
3024  if (s1->score < s2->score)
3025  return -1;
3026 
3027  if (s1->score > s2->score)
3028  return 1;
3029 
3030  return s1->date - s2->date > 0 ? 1 : -1;
3031 }
3032 
3033 
3034 static int
3036  t_comptype p1,
3037  t_comptype p2)
3038 {
3039  const struct t_article *s1 = (const struct t_article *) p1;
3040  const struct t_article *s2 = (const struct t_article *) p2;
3041 
3042  if (s2->score < s1->score)
3043  return -1;
3044 
3045  if (s2->score > s1->score)
3046  return 1;
3047 
3048  return s1->date - s2->date > 0 ? 1 : -1;
3049 }
3050 
3051 
3052 /*
3053  * Same again, but for art[].line_count
3054  */
3055 static int
3057  t_comptype p1,
3058  t_comptype p2)
3059 {
3060  const struct t_article *s1 = (const struct t_article *) p1;
3061  const struct t_article *s2 = (const struct t_article *) p2;
3062 
3063  if (s1->line_count < s2->line_count)
3064  return -1;
3065 
3066  if (s1->line_count > s2->line_count)
3067  return 1;
3068 
3069  return s1->date - s2->date > 0 ? 1 : -1;
3070 }
3071 
3072 
3073 static int
3075  t_comptype p1,
3076  t_comptype p2)
3077 {
3078  const struct t_article *s1 = (const struct t_article *) p1;
3079  const struct t_article *s2 = (const struct t_article *) p2;
3080 
3081  if (s2->line_count < s1->line_count)
3082  return -1;
3083 
3084  if (s2->line_count > s1->line_count)
3085  return 1;
3086 
3087  return s1->date - s2->date > 0 ? 1 : -1;
3088 }
3089 
3090 
3091 /*
3092  * Compares the total score of two threads. Used for sorting base[].
3093  */
3094 static int
3096  t_comptype p1,
3097  t_comptype p2)
3098 {
3099  int a = get_score_of_thread(*(const long *) p1);
3100  int b = get_score_of_thread(*(const long *) p2);
3101 
3102  /* If scores are equal, compare using the article sort order.
3103  * This determines the order in a group of equally scored threads.
3104  */
3105  if (a == b) {
3106  const struct t_article *s1 = &arts[*(const long *) p1];
3107  const struct t_article *s2 = &arts[*(const long *) p2];
3108  t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3109 
3110  if (comp_func)
3111  return (*comp_func)(s1, s2);
3112  return 0;
3113  }
3114 
3115  if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3116  return a > b ? 1 : -1;
3117  return a < b ? 1 : -1;
3118 }
3119 
3120 
3121 /*
3122  * Compare the date of the last posted article of two threads.
3123  * Used for sorting base[].
3124  */
3125 static int
3127  t_comptype p1,
3128  t_comptype p2)
3129 {
3130  time_t s1_last = get_last_posting_date(*(const long *) p1);
3131  time_t s2_last = get_last_posting_date(*(const long *) p2);
3132 
3133  if (s2_last < s1_last)
3134  return -1;
3135 
3136  if (s2_last > s1_last)
3137  return 1;
3138 
3139  return 0;
3140 }
3141 
3142 
3143 static int
3145  t_comptype p1,
3146  t_comptype p2)
3147 {
3148  time_t s1_last = get_last_posting_date(*(const long *) p1);
3149  time_t s2_last = get_last_posting_date(*(const long *) p2);
3150 
3151  if (s2_last > s1_last)
3152  return -1;
3153 
3154  if (s2_last < s1_last)
3155  return 1;
3156 
3157  return 0;
3158 }
3159 
3160 
3161 static time_t
3163  long n)
3164 {
3165  long i;
3166  time_t last = (time_t) 0;
3167 
3168  for (i = n; i >= 0; i = arts[i].thread) {
3169  if (arts[i].date > last)
3170  last = arts[i].date;
3171  }
3172 
3173  return last;
3174 }
3175 
3176 
3177 void
3179  struct t_article *art)
3180 {
3181  art->subject = NULL;
3182  art->from = NULL;
3183  art->name = NULL;
3184  art->date = (time_t) 0;
3185  art->xref = NULL;
3186  art->msgid = NULL;
3187  art->refs = NULL;
3188  art->refptr = NULL;
3189  art->line_count = -1;
3190  art->archive = NULL;
3191  art->tagged = 0;
3192  art->thread = ART_EXPIRED;
3193  art->prev = ART_NORMAL;
3194  art->score = 0;
3195  art->status = ART_UNREAD;
3196  art->killed = ART_NOTKILLED;
3197  art->zombie = FALSE;
3198  art->delete_it = FALSE;
3199  art->selected = FALSE;
3200  art->inrange = FALSE;
3201  art->matched = FALSE;
3202  art->keep_in_base = FALSE;
3203  art->multipart_subj = FALSE;
3204 }
3205 
3206 
3207 /*
3208  * Do a binary chop to see if 'art' (an article number) exists in arts[]
3209  * Naturally arts[] must be sorted on artnum
3210  * Return index into arts[] or -1
3211  */
3212 static int
3214  t_artnum art)
3215 {
3216  int prev, range;
3217  int dctop = top_art;
3218  int cur = 1;
3219 
3220  while ((dctop >>= 1))
3221  cur <<= 1;
3222 
3223  range = cur >> 1;
3224  cur--;
3225 
3226  forever {
3227  if (arts[cur].artnum == art)
3228  return cur;
3229 
3230  prev = cur;
3231  cur += ((arts[cur].artnum < art) ? range : -range);
3232  if (prev == cur)
3233  break;
3234 
3235  if (cur >= top_art)
3236  cur = top_art - 1;
3237 
3238  range >>= 1;
3239  }
3240  return -1;
3241 }
3242 
3243 
3244 /*
3245  * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3246  * Needed if arts[] is not sorted on artnum
3247  * Return index into arts[] or -1
3248  */
3249 int
3251  t_artnum art)
3252 {
3253  int i;
3254 
3255  for_each_art(i) {
3256  if (arts[i].artnum == art)
3257  return i;
3258  }
3259  return -1;
3260 }
3261 
3262 
3263 /*----------------------------- Overview handling -----------------------*/
3264 /* TODO: use
3265  * setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3266  * my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3267  * instead?
3268  */
3269 static char *
3271  time_t secs)
3272 {
3273  static char date[25];
3274  struct tm *tm;
3275  static const char *const months_a[] = {
3276  "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3277  "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3278  };
3279 
3280  if ((tm = gmtime(&secs)) != NULL)
3281  snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3282  tm->tm_mday,
3283  months_a[tm->tm_mon],
3284  tm->tm_year + 1900,
3285  tm->tm_hour, tm->tm_min, tm->tm_sec);
3286  else
3287  snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3288 
3289  return date;
3290 }
3291 
3292 
3293 static char *
3295  struct t_group *group,
3296  struct t_article *article,
3297  int charset)
3298 {
3299  char *p, *q;
3300  static char from[PATH_LEN];
3301 
3302  *from = '\0';
3303 
3304  if (article->name != NULL) {
3305  q = my_strdup(article->name);
3306 #ifdef CHARSET_CONVERSION
3307  if (charset != -1)
3308  buffer_to_network(q, charset);
3309 #endif /* CHARSET_CONVERSION */
3311  unfold_header(p);
3312  if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3313  snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3314  else
3315  snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3316 
3317  free(p);
3318  free(q);
3319  } else
3320  snprintf(from, sizeof(from), "<%s>", article->from);
3321 
3322  return from;
3323 }
3324 
3325 
3326 /*
3327  * Open a group news overview file
3328  * Use NNTP XOVER where possible unless 'local' is set
3329  */
3330 static FILE *
3332  struct t_group *group,
3333  const char *mode,
3334  t_artnum min,
3335  t_artnum max,
3336  t_bool local)
3337 {
3338 #ifdef NNTP_ABLE
3339  if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3340  char line[NNTP_STRLEN];
3341 
3342  if (!max)
3343  return NULL;
3344  if (min == max)
3345  snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3346  else
3347  snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3348  return (nntp_command(line, OK_XOVER, NULL, 0));
3349  }
3350 #endif /* NNTP_ABLE */
3351  {
3352  FILE *fp;
3353  char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3354 
3355  if (nov_file != NULL) {
3356  if ((fp = fopen(nov_file, mode)) != NULL)
3357  return fp;
3358 
3359  if (*mode != 'r')
3360  error_message(2, _(txt_cannot_open), nov_file);
3361  }
3362  }
3363  return NULL;
3364 }
3365 
3366 
3367 #ifdef USE_HEAPSORT
3368 int
3369 tin_sort(
3370  void *sbase,
3371  size_t nel,
3372  size_t width,
3373  t_compfunc compar)
3374 {
3375  int rc;
3376 
3377  switch (tinrc.sort_function) {
3378  case 0:
3379  qsort(sbase, nel, width, compar);
3380  rc = 0;
3381  break;
3382 
3383  case 1:
3384  rc = heapsort(sbase, nel, width, compar);
3385  break;
3386 
3387  default:
3388  rc = -1;
3389  break;
3390  }
3391  return rc;
3392 }
3393 #endif /* USE_HEAPSORT */
static char * find_nov_file(struct t_group *group, int mode)
Definition: art.c:2651
static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache)
Definition: art.c:1882
void make_threads(struct t_group *group, t_bool rethread)
Definition: art.c:1235
static void thread_by_multipart(void)
Definition: art.c:1171
void do_update(t_bool catchup)
Definition: art.c:2805
static int subj_comp_desc(t_comptype p1, t_comptype p2)
Definition: art.c:2906
void find_base(struct t_group *group)
Definition: art.c:123
int find_artnum(t_artnum art)
Definition: art.c:3250
void set_article(struct t_article *art)
Definition: art.c:3178
static void write_overview(struct t_group *group)
Definition: art.c:2499
t_bool index_group(struct t_group *group)
Definition: art.c:396
static void sort_base(unsigned int sort_threads_type)
Definition: art.c:1389
static void thread_by_percentage(unsigned int percentage)
Definition: art.c:910
int global_get_multipart_info(int aindex, MultiPartInfo *setme)
Definition: art.c:982
static FILE * open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local)
Definition: art.c:3331
static int score_comp_base(t_comptype p1, t_comptype p2)
Definition: art.c:3095
static int from_comp_asc(t_comptype p1, t_comptype p2)
Definition: art.c:2925
static int read_art_headers(struct t_group *group, int total, t_artnum top)
Definition: art.c:723
static time_t get_last_posting_date(long n)
Definition: art.c:3162
static int from_comp_desc(t_comptype p1, t_comptype p2)
Definition: art.c:2941
static FILE * open_art_header(char *groupname, t_artnum art, t_artnum *next)
Definition: art.c:624
static int date_comp_desc(t_comptype p1, t_comptype p2)
Definition: art.c:2990
static int score_comp_desc(t_comptype p1, t_comptype p2)
Definition: art.c:3035
static t_artnum setup_hard_base(struct t_group *group)
Definition: art.c:207
static t_compfunc eval_sort_arts_func(unsigned int sort_art_type)
Definition: art.c:1333
static int score_comp_asc(t_comptype p1, t_comptype p2)
Definition: art.c:3017
static t_bool parse_headers(FILE *fp, struct t_article *h)
Definition: art.c:1421
static char * print_date(time_t secs)
Definition: art.c:3270
static int subj_comp_asc(t_comptype p1, t_comptype p2)
Definition: art.c:2890
static void thread_by_subject(void)
Definition: art.c:851
#define SortBy(func)
Definition: art.c:58
void sort_arts(unsigned int sort_art_type)
Definition: art.c:1378
static t_artnum find_first_unread(struct t_group *group)
Definition: art.c:601
static int date_comp_asc(t_comptype p1, t_comptype p2)
Definition: art.c:2966
static char * print_from(struct t_group *group, struct t_article *article, int charset)
Definition: art.c:3294
int global_get_multiparts(int aindex, MultiPartInfo **malloc_and_setme_info, t_bool tagging)
Definition: art.c:1095
int top_art
Definition: art.c:60
void show_art_msg(const char *group)
Definition: art.c:110
static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset)
Definition: art.c:1005
static int last_date_comp_base_desc(t_comptype p1, t_comptype p2)
Definition: art.c:3126
t_bool global_look_for_multipart(int aindex, char start, char stop)
Definition: art.c:1055
static int base_comp(t_comptype p1, t_comptype p2)
Definition: art.c:172
static int artnum_comp(t_comptype p1, t_comptype p2)
Definition: art.c:2863
static int lines_comp_asc(t_comptype p1, t_comptype p2)
Definition: art.c:3056
static int last_date_comp_base_asc(t_comptype p1, t_comptype p2)
Definition: art.c:3144
static int valid_artnum(t_artnum art)
Definition: art.c:3213
static int lines_comp_desc(t_comptype p1, t_comptype p2)
Definition: art.c:3074
unsigned t_bool
Definition: bool.h:77
#define TRUE
Definition: bool.h:74
#define FALSE
Definition: bool.h:70
static t_openartinfo * art
Definition: cook.c:78
#define DEBUG_MISC
Definition: debug.h:54
#define DEBUG_NEWSRC
Definition: debug.h:50
#define DEBUG_REFS
Definition: debug.h:51
#define DEBUG_FILTER
Definition: debug.h:49
#define DEBUG_NNTP
Definition: debug.h:47
int verbose
Definition: init.c:153
constext txt_cannot_filter_on_path[]
Definition: lang.c:124
int * my_group
Definition: memory.c:64
constext txt_caughtup[]
Definition: lang.c:141
constext txt_not_exist[]
Definition: lang.c:700
t_bool xref_supported
Definition: init.c:155
char novrootdir[PATH_LEN]
Definition: init.c:114
int max_base
Definition: memory.c:55
struct t_capabilities nntp_caps
Definition: init.c:516
char * nntp_server
Definition: nntplib.c:28
t_menu grpmenu
Definition: group.c:83
int tin_errno
Definition: read.c:59
t_bool cmd_line
Definition: init.c:128
t_bool read_saved_news
Definition: init.c:151
t_menu selmenu
Definition: select.c:85
constext txt_catchup[]
Definition: lang.c:136
struct t_article * arts
Definition: memory.c:69
int need_resize
Definition: signal.c:107
char index_newsdir[PATH_LEN]
Definition: init.c:80
constext txt_group[]
Definition: lang.c:1217
int max_art
Definition: memory.c:54
constext txt_catchup_update_info[]
Definition: lang.c:140
struct t_overview_fmt * ofmt
Definition: xref.c:53
constext txt_cannot_open[]
Definition: lang.c:128
constext txt_unthreading_arts[]
Definition: lang.c:907
constext txt_updated[]
Definition: lang.c:908
int signal_context
Definition: signal.c:105
char novfilename[NAME_LEN+1]
Definition: init.c:113
char index_savedir[PATH_LEN]
Definition: init.c:81
t_artnum * base
Definition: memory.c:65
constext txt_threading_arts[]
Definition: lang.c:886
char index_maildir[PATH_LEN]
Definition: init.c:79
int cCOLS
Definition: curses.c:53
t_bool expensive_over_parse
Definition: xref.c:54
struct t_config tinrc
Definition: init.c:191
unsigned short debug
Definition: debug.c:51
char local_newsgroups_file[PATH_LEN]
Definition: init.c:86
constext txt_updating[]
Definition: lang.c:909
t_bool no_write
Definition: init.c:144
struct t_cmdlineopts cmdline
Definition: init.c:189
t_bool read_news_via_nntp
Definition: init.c:150
struct t_group * active
Definition: memory.c:66
t_bool batch_mode
Definition: init.c:126
static char buf[16]
Definition: langinfo.c:50
static t_bool catchup
Definition: main.c:57
#define NBITS
Definition: newsrc.h:112
#define NNTP_STRLEN
Definition: nntplib.h:155
#define OK_XPAT
Definition: nntplib.h:102
#define ERR_NCING
Definition: nntplib.h:127
#define OK_XHDR
Definition: nntplib.h:101
#define OK_GROUP
Definition: nntplib.h:95
#define OK_NOTEXT
Definition: nntplib.h:104
#define OK_HDR
Definition: nntplib.h:106
#define OK_XOVER
Definition: nntplib.h:105
@ CAPABILITIES
Definition: nntplib.h:171
#define OK_HEAD
Definition: nntplib.h:100
@ OVER_T_INT
Definition: nntplib.h:164
@ OVER_T_STRING
Definition: nntplib.h:164
@ OVER_T_ERROR
Definition: nntplib.h:164
@ OVER_T_FSTRING
Definition: nntplib.h:164
char * strerror(int n)
Definition: pcregrep.c:477
static char * end
Definition: plp_snprintf.c:205
int errno
long strtol(const char *str, char **ptr, int use_base)
Definition: string.c:416
void make_base_group_path(const char *base_dir, const char *group_name, char *group_path, size_t group_path_len)
Definition: misc.c:2110
t_bool filter_on_path(struct t_group *group)
Definition: filter.c:2229
char * fmt_string(const char *fmt,...)
Definition: string.c:1385
void parse_unread_arts(struct t_group *group, t_artnum min)
Definition: newsrc.c:1044
char * eat_tab(char *s)
Definition: string.c:573
t_bool filter_articles(struct t_group *group)
Definition: filter.c:1836
char * my_strdup(const char *str)
Definition: string.c:139
int atoi(const char *s)
void expand_art(void)
Definition: memory.c:135
void expand_base(void)
Definition: memory.c:167
void error_message(unsigned int sdelay, const char *fmt,...)
Definition: screen.c:184
void art_mark(struct t_group *group, struct t_article *art, int flag)
Definition: newsrc.c:1607
char * buffer_to_ascii(char *c)
Definition: misc.c:2638
void perror_message(const char *fmt,...)
Definition: screen.c:220
void expand_bitmap(struct t_group *group, t_artnum min)
Definition: newsrc.c:1464
char * strpbrk(const char *str1, const char *str2)
Definition: string.c:311
void info_message(const char *fmt,...)
Definition: screen.c:102
char * parse_header(char *buf, const char *pat, t_bool decode, t_bool structured, t_bool keep_tab)
Definition: rfc2046.c:908
void str_lwr(char *str)
Definition: string.c:290
void build_references(struct t_group *group)
Definition: refs.c:982
char * rfc1522_encode(char *s, const char *charset, t_bool ismail)
Definition: rfc2047.c:854
unsigned long hash_groupname(const char *group)
Definition: list.c:83
char * hash_str(const char *s)
Definition: hashstr.c:60
void joinpath(char *result, size_t result_size, const char *dir, const char *file)
Definition: joinpath.c:50
void hash_reclaim(void)
Definition: hashstr.c:134
int heapsort(void *, size_t, size_t, t_compfunc)
void collate_subjects(void)
Definition: refs.c:913
time_t parsedate(char *p, TIMEINFO *now)
int my_mkdir(char *path, mode_t mode)
Definition: misc.c:714
void free_art_array(void)
Definition: memory.c:329
void clear_message(void)
Definition: screen.c:243
void handle_resize(t_bool repaint)
Definition: signal.c:249
void get_cwd(char *buf)
Definition: misc.c:2070
int strwidth(const char *str)
Definition: string.c:1049
void unfold_header(char *line)
Definition: rfc2046.c:1143
char * tin_fgets(FILE *fp, t_bool header)
Definition: read.c:317
int my_toupper(int)
Definition: string.c:275
void wait_message(unsigned int sdelay, const char *fmt,...)
Definition: screen.c:133
void my_strncpy(char *p, const char *q, size_t n)
Definition: string.c:196
const char * eat_re(char *s, t_bool eat_was)
Definition: misc.c:974
int get_score_of_thread(int n)
Definition: thread.c:1102
void free_msgids(void)
Definition: refs.c:554
char * tin_strtok(char *str, const char *delim)
Definition: string.c:160
int strncasecmp(const char *p, const char *q, size_t n)
Definition: string.c:490
char * rfc1522_decode(const char *s)
Definition: rfc2047.c:232
char * convert_to_printable(char *buf, t_bool keep_tab)
Definition: charset.c:387
int strcasecmp(const char *p, const char *q)
Definition: string.c:474
char * strstr(const char *text, const char *pattern)
Definition: string.c:333
void thread_by_reference(void)
Definition: refs.c:855
void clear_art_ptrs(void)
Definition: refs.c:683
int parse_from(const char *from, char *address, char *realname)
Definition: misc.c:3630
void show_progress(const char *txt, t_artnum count, t_artnum total)
Definition: screen.c:489
static int offset
Definition: read.c:62
const char * name
Definition: signal.c:117
#define PrintStopWatch()
Definition: stpwatch.h:99
#define EndStopWatch()
Definition: stpwatch.h:98
#define BegStopWatch(msg)
Definition: stpwatch.h:97
int subject_compare_len
Definition: tin.h:2006
int part_number
Definition: tin.h:2007
int arts_index
Definition: tin.h:2009
char * subject
Definition: tin.h:2005
int total
Definition: tin.h:2008
char * partnum
Definition: tin.h:1489
char * name
Definition: tin.h:1488
t_bool ispart
Definition: tin.h:1490
Definition: tin.h:1520
time_t date
Definition: tin.h:1532
struct t_archive * archive
Definition: tin.h:1531
char * xref
Definition: tin.h:1525
char * subject
Definition: tin.h:1522
int prev
Definition: tin.h:1537
char * refs
Definition: tin.h:1529
int thread
Definition: tin.h:1536
int score
Definition: tin.h:1538
t_bool multipart_subj
Definition: tin.h:1547
int line_count
Definition: tin.h:1533
t_bool keep_in_base
Definition: tin.h:1546
struct t_msgid * refptr
Definition: tin.h:1530
t_artnum artnum
Definition: tin.h:1521
int gnksa_code
Definition: tin.h:1534
char * path
Definition: tin.h:1526
unsigned int killed
Definition: tin.h:1540
char * msgid
Definition: tin.h:1528
t_bool selected
Definition: tin.h:1543
int tagged
Definition: tin.h:1535
char * from
Definition: tin.h:1523
char * name
Definition: tin.h:1524
unsigned int status
Definition: tin.h:1539
unsigned thread_perc
Definition: tin.h:1644
unsigned thread_articles
Definition: tin.h:1642
unsigned sort_threads_type
Definition: tin.h:1663
unsigned post_8bit_header
Definition: tin.h:1630
unsigned sort_article_type
Definition: tin.h:1659
unsigned show_only_unread_arts
Definition: tin.h:1638
const char * hdr_cmd
Definition: nntplib.h:207
t_bool hdr
Definition: nntplib.h:206
char * headers_range
Definition: nntplib.h:196
const char * over_cmd
Definition: nntplib.h:210
t_bool xpat
Definition: nntplib.h:205
enum extension_type type
Definition: nntplib.h:187
t_bool list_headers
Definition: nntplib.h:195
t_bool reader
Definition: nntplib.h:190
t_bool broken_listgroup
Definition: nntplib.h:224
unsigned int args
Definition: tin.h:1481
int getart_limit
Definition: tin.h:1477
int kill_level
Definition: tinrc.h:139
t_bool cache_overview_files
Definition: tinrc.h:209
int thread_articles
Definition: tinrc.h:152
char mm_local_charset[LEN]
Definition: tinrc.h:103
int getart_limit
Definition: tinrc.h:135
Definition: tin.h:1783
t_bool bogus
Definition: tin.h:1798
t_artnum xmin
Definition: tin.h:1791
t_artnum xmax
Definition: tin.h:1790
struct t_attribute * attribute
Definition: tin.h:1801
unsigned int type
Definition: tin.h:1792
t_bool subscribed
Definition: tin.h:1796
char * name
Definition: tin.h:1784
char * spooldir
Definition: tin.h:1787
struct t_newsrc newsrc
Definition: tin.h:1800
int aptr
Definition: tin.h:1810
int curr
Definition: tin.h:2017
int max
Definition: tin.h:2018
struct t_msgid * sibling
Definition: tin.h:1499
struct t_msgid * child
Definition: tin.h:1500
struct t_msgid * parent
Definition: tin.h:1498
int article
Definition: tin.h:1501
char txt[1]
Definition: tin.h:1502
t_artnum xbitlen
Definition: tin.h:1776
t_bitmap * xbitmap
Definition: tin.h:1777
t_artnum xmin
Definition: tin.h:1775
enum f_type type
Definition: tin.h:2482
char * name
Definition: tin.h:2481
#define my_flush()
Definition: tcurses.h:171
#define my_fprintf
Definition: tcurses.h:170
#define my_printf
Definition: tcurses.h:169
#define PLURAL(x, y)
Definition: tin.h:1056
#define LEN
Definition: tin.h:857
#define ART_NOTKILLED
Definition: tin.h:1331
#define S_IROTH
Definition: tin.h:2157
#define SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND
Definition: tin.h:1196
long t_artnum
Definition: tin.h:229
#define TIN_FCLOSE(x)
Definition: tin.h:1040
#define SORT_ARTICLES_BY_DATE_ASCEND
Definition: tin.h:1182
#define THREAD_NONE
Definition: tin.h:1130
#define SORT_ARTICLES_BY_FROM_ASCEND
Definition: tin.h:1180
#define R_OK
Definition: tin.h:2189
@ cArt
Definition: tin.h:107
#define IS_LOCAL_CHARSET(c)
Definition: tin.h:779
#define T_ARTNUM_SFMT
Definition: tin.h:231
#define THREAD_BOTH
Definition: tin.h:1133
#define MIN(a, b)
Definition: tin.h:808
#define MODULO_COUNT_NUM
Definition: tin.h:865
#define for_each_art(x)
Definition: tin.h:2222
#define IGNORE_ART_THREAD(i)
Definition: tin.h:1023
#define S_IRGRP
Definition: tin.h:2152
int(* t_compfunc)(t_comptype, t_comptype)
Definition: tin.h:2102
#define SORT_ARTICLES_BY_LINES_ASCEND
Definition: tin.h:1186
#define my_malloc(size)
Definition: tin.h:2207
#define atoartnum
Definition: tin.h:233
#define SORT_THREADS_BY_SCORE_ASCEND
Definition: tin.h:1194
#define S_ISDIR(m)
Definition: tin.h:2138
#define FreeIfNeeded(p)
Definition: tin.h:2214
#define NEWSGROUPS_FILE
Definition: tin.h:743
#define ART_NORMAL
Definition: tin.h:1318
#define CURR_GROUP
Definition: tin.h:1046
#define SORT_ARTICLES_BY_FROM_DESCEND
Definition: tin.h:1179
#define SORT_ARTICLES_BY_SUBJ_ASCEND
Definition: tin.h:1178
#define SORT_ARTICLES_BY_SUBJ_DESCEND
Definition: tin.h:1177
#define SORT_ARTICLES_BY_SCORE_DESCEND
Definition: tin.h:1183
#define S_IRUSR
Definition: tin.h:2147
#define _(Text)
Definition: tin.h:94
#define KILL_NOTHREAD
Definition: tin.h:1210
#define forever
Definition: tin.h:813
#define PATH_LEN
Definition: tin.h:840
#define STRNCASECMPEQ(s1, s2, n)
Definition: tin.h:821
#define CMDLINE_GETART_LIMIT
Definition: tin.h:1094
#define SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND
Definition: tin.h:1195
#define SORT_THREADS_BY_NOTHING
Definition: tin.h:1192
@ cNo
Definition: tin.h:109
@ cRedraw
Definition: tin.h:109
#define SORT_ARTICLES_BY_DATE_DESCEND
Definition: tin.h:1181
#define SORT_ARTICLES_BY_LINES_DESCEND
Definition: tin.h:1185
#define SORT_ARTICLES_BY_SCORE_ASCEND
Definition: tin.h:1184
#define snprintf
Definition: tin.h:2432
#define SORT_ARTICLES_BY_NOTHING
Definition: tin.h:1176
#define FreeAndNull(p)
Definition: tin.h:2215
#define T_ARTNUM_CONST(v)
Definition: tin.h:232
#define THREAD_REFS
Definition: tin.h:1132
#define GROUP_TYPE_SAVE
Definition: tin.h:1063
#define tin_sort
Definition: tin.h:2109
#define ART_EXPIRED
Definition: tin.h:1313
#define GROUP_TYPE_NEWS
Definition: tin.h:1062
#define GROUP_TYPE_MAIL
Definition: tin.h:1061
#define HEADER_LEN
Definition: tin.h:860
#define T_ARTNUM_PFMT
Definition: tin.h:230
#define ART_READ
Definition: tin.h:1323
#define my_realloc(ptr, size)
Definition: tin.h:2209
#define THREAD_SUBJ
Definition: tin.h:1131
#define S_IWUSR
Definition: tin.h:2148
#define SORT_THREADS_BY_SCORE_DESCEND
Definition: tin.h:1193
#define THREAD_MULTI
Definition: tin.h:1134
#define W_OK
Definition: tin.h:2192
#define ART_UNTHREADED
Definition: tin.h:1312
#define assert(p)
Definition: tin.h:1298
#define ART_UNREAD
Definition: tin.h:1324
#define DIR_BUF
Definition: tin.h:383
#define CLOSEDIR(DIR)
Definition: tin.h:2366
#define THREAD_PERC
Definition: tin.h:1135
#define MAX(a, b)
Definition: tin.h:805
#define BlankIfNull(p)
Definition: tin.h:2217
#define FAKE_NNTP_FP
Definition: tin.h:458
#define S_IRWXU
Definition: tin.h:2146