"Fossies" - the Fresh Open Source Software Archive 
Member "tin-2.4.5/src/art.c" (1 Dec 2020, 90939 Bytes) of package /linux/misc/tin-2.4.5.tar.xz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "art.c" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
2.4.4_vs_2.4.5.
1 /*
2 * Project : tin - a Usenet reader
3 * Module : art.c
4 * Author : I.Lea & R.Skrenta
5 * Created : 1991-04-01
6 * Updated : 2020-07-08
7 * Notes :
8 *
9 * Copyright (c) 1991-2021 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10 * All rights reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * 1. Redistributions of source code must retain the above copyright notice,
17 * this list of conditions and the following disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 *
23 * 3. Neither the name of the copyright holder nor the names of its
24 * contributors may be used to endorse or promote products derived from
25 * this software without specific prior written permission.
26 *
27 * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
31 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40
41 #ifndef TIN_H
42 # include "tin.h"
43 #endif /* !TIN_H */
44 #ifndef TCURSES_H
45 # include "tcurses.h"
46 #endif /* !TCURSES_H */
47 #ifndef NEWSRC_H
48 # include "newsrc.h"
49 #endif /* !NEWSRC_H */
50
51 #ifndef STPWATCH_H
52 # include "stpwatch.h"
53 #endif /* !STPWATCH_H */
54
55 /*
56 * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
57 */
58 #define SortBy(func) tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
59
60 int top_art = 0; /* # of articles in arts[] */
61
62 /*
63 * Local prototypes
64 */
65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
67 static char *find_nov_file(struct t_group *group, int mode);
68 static char *print_date(time_t secs);
69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
70 static int artnum_comp(t_comptype p1, t_comptype p2);
71 static int base_comp(t_comptype p1, t_comptype p2);
72 static int date_comp_asc(t_comptype p1, t_comptype p2);
73 static int date_comp_desc(t_comptype p1, t_comptype p2);
74 static int from_comp_asc(t_comptype p1, t_comptype p2);
75 static int from_comp_desc(t_comptype p1, t_comptype p2);
76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
83 static int score_comp_asc(t_comptype p1, t_comptype p2);
84 static int score_comp_desc(t_comptype p1, t_comptype p2);
85 static int score_comp_base(t_comptype p1, t_comptype p2);
86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
88 static int valid_artnum(t_artnum art);
89 static t_artnum find_first_unread(struct t_group *group);
90 static t_artnum setup_hard_base(struct t_group *group);
91 static t_bool parse_headers(FILE *fp, struct t_article *h);
92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
93 static time_t get_last_posting_date(long n);
94 static void sort_base(unsigned int sort_threads_type);
95 static void thread_by_multipart(void);
96 static void thread_by_percentage(unsigned int percentage);
97 static void thread_by_subject(void);
98 static void write_overview(struct t_group *group);
99 #ifdef NNTP_ABLE
100 static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
101 static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
102 #endif /* NNTP_ABLE */
103
104
105 /*
106 * Display a suitable 'entering group' message if screen needs redrawing
107 * Allow for the non-printing %s, and the %-age counter
108 */
109 void
110 show_art_msg(
111 const char *group)
112 {
113 wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
114 }
115
116
117 /*
118 * Construct the pointers to the first (base) article in each thread.
119 * If we are showing only unread, then point to the first unread. I have
120 * no idea why this should be so, it causes problems elsewhere [which_response]
121 */
122 void
123 find_base(
124 struct t_group *group)
125 {
126 int i, j;
127
128 grpmenu.max = 0;
129
130 #ifdef DEBUG
131 if (debug & DEBUG_FILTER)
132 debug_print_arts();
133 #endif /* DEBUG */
134
135 for_each_art(i) {
136 /*
137 * .prev will be set on each article that is after the first article in
138 * the thread. Invalid articles which have been expired will have
139 * .thread set to ART_EXPIRED
140 */
141 if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
142 continue;
143
144 if (grpmenu.max >= max_base)
145 expand_base();
146
147 if (group->attribute->show_only_unread_arts) {
148 if (arts[i].status != ART_READ || arts[i].keep_in_base)
149 base[grpmenu.max++] = i;
150 else {
151 /* Find 1st unread art in thread */
152 for (j = i; j >= 0; j = arts[j].thread) {
153 if (arts[j].status != ART_READ || arts[j].keep_in_base) {
154 base[grpmenu.max++] = i;
155 break;
156 }
157 }
158 }
159 } else
160 base[grpmenu.max++] = i;
161 }
162 /* sort base[] */
163 if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
164 sort_base(group->attribute->sort_threads_type);
165 }
166
167
168 /*
169 * Longword comparison routine for the tin_sort()
170 */
171 static int
172 base_comp(
173 t_comptype p1,
174 t_comptype p2)
175 {
176 const t_artnum *a = (const t_artnum *) p1;
177 const t_artnum *b = (const t_artnum *) p2;
178
179 if (*a < *b)
180 return -1;
181
182 if (*a > *b)
183 return 1;
184
185 return 0;
186 }
187
188
189 /*
190 * via NNTP:
191 * Issue a LISTGROUP command
192 * Read the article numbers existing in the group into base[]
193 * If the LISTGROUP failed, issue a GROUP command. Use the results to
194 * create a less accurate version of base[]
195 * This data will already be sorted
196 *
197 * on local spool:
198 * Read the spool dir to populate base[] as above. Sort it.
199 *
200 * Grow the arts[] and bitmaps as needed.
201 * NB: the output will be sorted on artnum
202 *
203 * grpmenu.max is one past top.
204 * Returns total number of articles in group, or -1 on error
205 */
206 static t_artnum
207 setup_hard_base(
208 struct t_group *group)
209 {
210 t_artnum total = 0;
211
212 grpmenu.max = 0;
213
214 /*
215 * If reading with NNTP, issue a LISTGROUP
216 */
217 if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
218 #ifdef NNTP_ABLE
219 char buf[NNTP_STRLEN];
220 char line[NNTP_STRLEN];
221 int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
222 FILE *fp;
223 t_artnum last, start, count = 0, j = 0;
224 static t_bool skip_listgroup = FALSE;
225
226 /*
227 * Some nntp servers are broken and need an extra GROUP command
228 * (reported by reorx@irc.pl). This affects (old?) versions of
229 * nntpcache, leafnode and SurgeNews. Usually this should not be
230 * needed.
231 *
232 * For getart_limit recheck lowwatermark as at least giganews gives
233 * very different results for LIST ACTIVE (3 year retention for all)
234 * and GROUP (based on the clients contract).
235 * Calculate range and prepare base[] not to lose unread arts.
236 */
237 if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
238 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
239 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
240 return -1;
241
242 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
243 return -1;
244
245 if (getart_limit > 0) {
246 j = group->xmax - getart_limit;
247 count = MAX(find_first_unread(group), start);
248 }
249 if (getart_limit < 0) {
250 j = getart_limit + find_first_unread(group);
251 count = group->xmin;
252 }
253 if (j < group->xmin)
254 j = group->xmin;
255
256 for (; count < j; count++) {
257 if (grpmenu.max >= max_base)
258 expand_base();
259 base[grpmenu.max++] = count;
260 }
261 }
262
263 /*
264 * See if LISTGROUP works
265 */
266 if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
267 if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
268 /* RFC 3977 allows ranges in LISTGROUP */
269 if (getart_limit > 0)
270 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
271 else /* getart_limit < 0; fetch till newest art */
272 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
273
274 } else /* for RFC 977 just use GROUP */
275 skip_listgroup = TRUE;
276
277 } else /* get all article numbers */
278 snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
279
280 if (!skip_listgroup) {
281 if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
282 char *ptr;
283
284 # ifdef DEBUG
285 if ((debug & DEBUG_NNTP) && verbose > 1)
286 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
287 # endif /* DEBUG */
288
289 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
290 if (grpmenu.max >= max_base)
291 expand_base();
292 base[grpmenu.max++] = atoartnum(ptr);
293 total++;
294 }
295
296 if (tin_errno)
297 return -1;
298 } else
299 skip_listgroup = TRUE;
300 }
301
302 if (skip_listgroup) { /* LISTGROUP was skipped or failed */
303 /*
304 * Handle the obscure case that the user aborted before the LISTGROUP
305 * had a chance to respond
306 */
307 if (tin_errno)
308 return -1;
309
310 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
311 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
312 return -1;
313
314 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
315 return -1;
316
317 # ifdef DEBUG
318 if ((debug & DEBUG_NNTP) && verbose > 1)
319 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
320 # endif /* DEBUG */
321 total = count;
322 grpmenu.max = 0;
323 if (getart_limit > 0) {
324 if ((j = find_first_unread(group)) > start) {
325 if (group->xmax > getart_limit) {
326 start = MIN(j, group->xmax - getart_limit);
327 total = getart_limit;
328 } else
329 start = j;
330 }
331 }
332 if (getart_limit < 0) {
333 if ((j = (getart_limit + find_first_unread(group))) > start)
334 start = j;
335 }
336 while (start <= last) {
337 if (grpmenu.max >= max_base)
338 expand_base();
339 base[grpmenu.max++] = start++;
340 }
341 }
342 #endif /* NNTP_ABLE */
343 /*
344 * Reading off local spool, read the directory files
345 */
346 } else {
347 DIR *d;
348 DIR_BUF *e;
349 char group_path[PATH_LEN];
350 t_artnum art;
351
352 make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
353
354 if ((d = opendir(group_path)) != NULL) {
355 while ((e = readdir(d)) != NULL) {
356 art = atoartnum(e->d_name);
357 if (art >= 1) {
358 total++;
359 if (grpmenu.max >= max_base)
360 expand_base();
361 base[grpmenu.max++] = art;
362 }
363 }
364 CLOSEDIR(d);
365 tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
366 } else {
367 perror_message(_(txt_cannot_open), group_path);
368 #if 0
369 if (access(group_path, R_OK) != 0)
370 error_message(2, _(txt_not_exist));
371 #endif /* 0 */
372 return -1;
373 }
374 }
375
376 if (grpmenu.max) {
377 if (base[grpmenu.max - 1] > group->xmax)
378 group->xmax = base[grpmenu.max - 1];
379 expand_bitmap(group, base[0]);
380 }
381
382 return total;
383 }
384
385
386 /*
387 * Main group indexing routine.
388 *
389 * Will read any existing index, create or incrementally update
390 * the index by looking at the articles in the spool directory,
391 * and attempt to write a new index if necessary.
392 *
393 * Returns FALSE if the user aborted the indexing, otherwise TRUE
394 */
395 t_bool
396 index_group(
397 struct t_group *group)
398 {
399 int i;
400 int changed; /* Count of articles whose overview has changed */
401 int getart_limit;
402 int respnum;
403 int total;
404 t_artnum last_read_article;
405 t_artnum min, new_min, max;
406 t_bool caching_xover;
407 t_bool filtered;
408 t_bool path_in_nov = FALSE;
409 t_bool rebuild_cache = FALSE;
410
411 if (group == NULL)
412 return TRUE;
413
414 if (!batch_mode)
415 show_art_msg(group->name);
416 else {
417 if (verbose) /* -> lang.c */
418 wait_message(0, _("Reading %s\n"), group->name);
419 }
420
421 signal_context = cArt; /* Set this only once curr_group is valid */
422
423 hash_reclaim();
424 free_art_array();
425 free_msgids();
426
427 BegStopWatch("setup_hard_base()");
428
429 /*
430 * Get list of valid article numbers
431 */
432 if (setup_hard_base(group) < 0)
433 return FALSE;
434
435 EndStopWatch();
436 PrintStopWatch();
437
438 #ifdef DEBUG
439 if (debug & DEBUG_NEWSRC) {
440 debug_print_comment("Before read_overview");
441 debug_print_bitmap(group, NULL);
442 }
443 #endif /* DEBUG */
444
445 min = grpmenu.max ? base[0] : group->xmin;
446 max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
447
448 getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
449
450 if (getart_limit > 0) {
451 if (grpmenu.max && (grpmenu.max > getart_limit))
452 min = base[grpmenu.max - getart_limit];
453 else
454 getart_limit = 0;
455 } else if (getart_limit < 0) {
456 t_artnum first_unread = find_first_unread(group);
457
458 if (min - first_unread < getart_limit)
459 min = first_unread + getart_limit;
460 else
461 getart_limit = 0;
462 }
463
464 /*
465 * Quit now if no articles
466 */
467 if (max < 0)
468 return FALSE;
469
470 top_art = 0;
471 last_read_article = T_ARTNUM_CONST(0);
472
473 /*
474 * Read in the existing overview data for min..max
475 * This read has local=TRUE set if locally caching XOVER records to ensure
476 * we pull in any private overview caches in preference to using OVER
477 *
478 * When reading local spool, this will pull in the system wide overview
479 * cache (if found) otherwise the private overview cache will be read
480 */
481 caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
482 if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
483 return FALSE; /* user aborted indexing */
484
485 /*
486 * Fill in the range last_read_article...max using XOVER
487 * Only do this if the previous read_overview() was against private cache
488 */
489 if ((last_read_article < max) && caching_xover) {
490 new_min = (last_read_article >= min) ? last_read_article + 1 : min;
491
492 if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
493 return FALSE; /* user aborted indexing */
494 else
495 changed += i;
496 } else
497 caching_xover = FALSE;
498
499 /*
500 * Mark as UNTHREADED all articles that have been verified as valid
501 * Get num of new arts to index so the user will have an idea of index time
502 */
503 for (i = 0, total = 0; i < grpmenu.max; i++) {
504 if ((respnum = valid_artnum(base[i])) >= 0) {
505 arts[respnum].thread = ART_UNTHREADED;
506 continue;
507 }
508 if (base[i] <= last_read_article) /* It is vital this test be done last */
509 continue;
510 total++;
511 }
512
513 /*
514 * Add any articles to arts[] that are new or were killed
515 */
516 if (total > 0) {
517 new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
518
519 if ((i = read_art_headers(group, total, new_min)) == -1)
520 return FALSE; /* user aborted indexing */
521 else
522 changed += i;
523 }
524
525 #ifdef DEBUG
526 if (debug & DEBUG_NEWSRC) {
527 debug_print_comment("Before parse_unread_arts()");
528 debug_print_bitmap(group, NULL);
529 }
530 #endif /* DEBUG */
531 /*
532 * Do this before calling art_mark(,, ART_READ) if you want
533 * the unread count to be correct.
534 */
535 min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
536 parse_unread_arts(group, min);
537 #ifdef DEBUG
538 if (debug & DEBUG_NEWSRC) {
539 debug_print_comment("After parse_unread_arts()");
540 debug_print_bitmap(group, NULL);
541 }
542 #endif /* DEBUG */
543
544 /*
545 * Stat all articles to see if any have expired
546 */
547 for_each_art(i) {
548 if (arts[i].thread == ART_EXPIRED) {
549 changed++;
550 #ifdef DEBUG
551 if (debug & DEBUG_NEWSRC)
552 debug_print_comment("art.c: index_group() purging...");
553 #endif /* DEBUG */
554 art_mark(group, &arts[i], ART_READ);
555 if (group->attribute->show_only_unread_arts)
556 arts[i].keep_in_base = FALSE;
557 }
558 if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
559 path_in_nov = TRUE;
560 }
561
562 /*
563 * Only rewrite the index if it has changed
564 * TODO review the exact logic behind "|| caching_xover"
565 */
566 if (changed || caching_xover || rebuild_cache)
567 write_overview(group);
568
569 /*
570 * Create the reference tree. The msgid and ref ptrs will
571 * be free()d now that the NovFile has been written.
572 */
573 build_references(group);
574
575 /*
576 * Needs access to the reference tree
577 */
578 filtered = filter_articles(group);
579
580 BegStopWatch("make_threads()");
581
582 /*
583 * Thread the group
584 */
585 make_threads(group, FALSE);
586
587 EndStopWatch();
588 PrintStopWatch();
589
590 if ((changed > 0 || filtered) && !batch_mode)
591 clear_message();
592
593 return TRUE;
594 }
595
596
597 /*
598 * Returns number of first unread article
599 */
600 static t_artnum
601 find_first_unread(
602 struct t_group *group)
603 {
604 unsigned char *p;
605 unsigned char *end = group->newsrc.xbitmap;
606 t_artnum first = group->newsrc.xmin; /* initial value */
607
608 if ((p = group->newsrc.xbitmap)) {
609 end += group->newsrc.xbitlen / NBITS;
610 for (; *p == '\0' && p < end; p++, first += NBITS)
611 ;
612 }
613 return first;
614 }
615
616
617 /*
618 * Open an article for reading just the header
619 * 'next' is used/updated with the next article number
620 * to optimise the number of 'HEAD' commands issued on
621 * groups with holes.
622 */
623 static FILE *
624 open_art_header(
625 char *groupname,
626 t_artnum art,
627 t_artnum *next)
628 {
629 char buf[NNTP_STRLEN];
630 #ifdef NNTP_ABLE
631 FILE *fp;
632 int i;
633
634 if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
635 /*
636 * Don't bother requesting if we have not got there yet.
637 * This is a big win if the group has got holes in it (ie. if 000's
638 * of articles have expired between active files min & max values).
639 */
640 if (art < *next)
641 return NULL;
642
643 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
644 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
645 return fp;
646
647 /*
648 * TODO:
649 * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
650 * "503 Access denied" instead of 480 but NEXT still works,
651 * so tin loops over all articles without getting useful data
652 */
653
654 /*
655 * HEAD failed, try to find NEXT
656 * Should return "223 artno message-id more text...."
657 */
658 i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
659 switch (i) {
660 case OK_NOTEXT:
661 *next = atoartnum(buf); /* Set next art number */
662 break;
663
664 # ifndef BROKEN_LISTGROUP
665 /*
666 * might happen if LISTGROUP doesn't select group, but
667 * we are not -DBROKEN_LISTGROUP
668 */
669 case ERR_NCING:
670 nntp_caps.broken_listgroup = TRUE;
671 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
672 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
673 return NULL;
674 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
675 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
676 return fp;
677 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
678 *next = atoartnum(buf);
679 break;
680 # endif /* !BROKEN_LISTGROUP */
681
682 default:
683 /*
684 * TODO: abort loop over all arts on ERR_NONEXT
685 */
686 # ifndef BROKEN_LISTGROUP
687 /*
688 * to avoid out of sync responses
689 * (listgroup seems to work, but didn't select new group,
690 * so xover seems to work but returns old data)
691 * we set listgroup_broken = TRUE; once we saw a
692 * ERR_NOARTIG / ERR_NONEXT or the like - even if
693 * ERR_NOARTIG may occur on servers where listgroup
694 * isn't broken...
695 */
696 nntp_caps.broken_listgroup = TRUE;
697 # endif /* !BROKEN_LISTGROUP */
698 break;
699 }
700
701 return NULL;
702 }
703 #endif /* NNTP_ABLE */
704
705 snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
706 return (fopen(buf, "r"));
707 }
708
709
710 /*
711 * Called after XOVER/local/private overview databases have been loaded
712 * Read and parse in headers for any arts not already found (usually
713 * new articles that have not been indexed yet)
714 * Any new articles that are added have ->thread set to ART_UNTHREADED
715 * 'top' is the current highest artnum read
716 *
717 * Return values are:
718 * >0 Number of additional articles read in
719 * 0 No additional (new) articles were found
720 * -1 user aborted during read
721 */
722 static int
723 read_art_headers(
724 struct t_group *group,
725 int total,
726 t_artnum top)
727 {
728 FILE *fp;
729 char dir[PATH_LEN];
730 char group_msg[LEN];
731 int i;
732 int modified = 0;
733 t_artnum art;
734 t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
735 t_bool res;
736
737 /*
738 * Change to groups spooldir to optimize fopen()'s on local articles
739 * NB open_art_header() requires this
740 */
741 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
742 char buf[PATH_LEN];
743
744 get_cwd(dir);
745 make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
746 if (chdir(buf) != 0) {
747 #ifdef DEBUG
748 if (debug & DEBUG_MISC)
749 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
750 #endif /* DEBUG */
751 return -1;
752 }
753 }
754
755 snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
756
757 for (i = 0; i < grpmenu.max; i++) { /* for each article number */
758 art = base[i];
759
760 /*
761 * Skip articles that are below the low water mark or are
762 * already present
763 */
764 if (valid_artnum(art) >= 0)
765 continue;
766 if (art <= top)
767 continue;
768
769 /*
770 * Try and open the article
771 */
772 if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
773 continue;
774
775 /*
776 * Add article to arts[]
777 */
778 if (top_art >= max_art)
779 expand_art();
780
781 set_article(&arts[top_art]);
782 arts[top_art].artnum = art;
783 arts[top_art].thread = ART_UNTHREADED;
784
785 res = parse_headers(fp, &arts[top_art]);
786
787 TIN_FCLOSE(fp);
788 if (tin_errno) {
789 modified = -1;
790 break;
791 }
792
793 if (!res) {
794 #ifdef DEBUG
795 if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
796 char buf[PATH_LEN];
797
798 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
799 debug_print_file("ARTS", "read_art_headers() %s", buf);
800 }
801 #endif /* DEBUG */
802 arts[top_art].artnum = T_ARTNUM_CONST(0);
803 arts[top_art].date = (time_t) 0;
804 FreeAndNull(arts[top_art].xref);
805 FreeAndNull(arts[top_art].path);
806 FreeAndNull(arts[top_art].refs);
807 FreeAndNull(arts[top_art].msgid);
808 if (arts[top_art].archive) {
809 FreeAndNull(arts[top_art].archive->partnum);
810 FreeAndNull(arts[top_art].archive);
811 }
812 arts[top_art].tagged = 0;
813 arts[top_art].thread = ART_EXPIRED;
814 arts[top_art].prev = ART_NORMAL;
815 arts[top_art].status = ART_UNREAD;
816 arts[top_art].killed = ART_NOTKILLED;
817 arts[top_art].selected = FALSE;
818 continue;
819 }
820
821 top = arts[top_art].artnum; /* used if arts are killed */
822 top_art++;
823
824 if (++modified % (MODULO_COUNT_NUM * 20) == 0)
825 show_progress(group_msg, modified, total);
826 }
827
828 /*
829 * Change back to previous dir before indexing started
830 */
831 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
832 chdir(dir);
833
834 return modified;
835 }
836
837
838 /*
839 * The algorithm is elegant, using the fact that identical Subject lines
840 * are hashed to the same node in table[] (see hashstr.c)
841 *
842 * Mark i as being in j's thread list if
843 * . The article is _not_ being ignored
844 * . The article is not already threaded
845 * . One of the following is true:
846 * 1) The subject lines are the same
847 * 2) Both are part of the same archive (name's match and arch bit set)
848 * IMHO the tests for archive name are redundant and have been for years
849 */
850 static void
851 thread_by_subject(
852 void)
853 {
854 int i, j;
855 struct t_hashnode *h;
856
857 for_each_art(i) {
858 if (IGNORE_ART_THREAD(i))
859 continue;
860
861 /*
862 * Get the contents of the magic marker in the hashnode
863 */
864 h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
865
866 j = h->aptr;
867
868 if (j != -1 && j < i) {
869 #if 1
870 if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
871 #else
872 /* see also refs.c:collate_subjects() */
873 if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
874 #endif /* 1 */
875 {
876 arts[j].thread = i;
877 arts[i].prev = j;
878 }
879 }
880
881 /*
882 * Update the magic marker with the highest numbered mesg in
883 * arts[] that has been used in this thread so far
884 */
885 h->aptr = i;
886 }
887
888 #if 0
889 fprintf(stderr, "Subj dump\n");
890 fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
891 for_each_art(i) {
892 fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
893 (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
894 (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
895 (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
896 arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
897 }
898 #endif /* 0 */
899 }
900
901
902 /*
903 * This Threading algorithm threads articles into 'buckets' where each bucket
904 * contains all the articles which match the root article's subject line to
905 * the configured percentage. Eg, if the root article had the subject "asdf"
906 * and the match percentage was configured to be 75% then any article would
907 * match if its subject was no different in more than a single character.
908 */
909 static void
910 thread_by_percentage(
911 unsigned int percentage)
912 {
913 int i, j, k;
914 int root_num = 0; /* The index number of the root we are currently working on. */
915 unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
916 size_t slen;
917
918 /* First we need to sort art[] to simplify and speed up the matching. */
919 SortBy(subj_comp_asc);
920
921 /*
922 * Now we put all the articles which match enough into the thread. If
923 * an article doesn't match enough we create a new thread and then add
924 * to that and so on.
925 */
926 base[0] = 0;
927 arts[0].prev = ART_NORMAL;
928 for_each_art(i) {
929 if (i == 0)
930 continue;
931
932 /* Check each character to see if it matched enough */
933 k = 0;
934 unmatched = 0;
935 for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
936 if (arts[base[root_num]].subject[j] != arts[i].subject[k])
937 unmatched++;
938 }
939
940 /*
941 * By getting here we have a number of unmatched characters
942 * between the two strings. We also have the length of the
943 * strings available to us easily.
944 * All we need to do is see if the match is good enough, but
945 * we count differences in the length of the strings against
946 * them matching.
947 */
948 if (!(slen = strlen(arts[base[root_num]].subject)))
949 slen++;
950 unmatched += slen - strlen(arts[i].subject);
951 if (unmatched * 100 / slen > percentage) {
952 /*
953 * If there is less greater than percentage% different start a
954 * new thread.
955 */
956 base[++root_num] = i;
957 arts[i].prev = ART_NORMAL;
958 continue;
959 } else {
960 /*
961 * The subject lines match enough to consider them part of a single
962 * thread, so add the current article to the thread.
963 */
964 if (arts[base[root_num]].thread < 0)
965 arts[base[root_num]].thread = i;
966 arts[i].prev = i - 1;
967 arts[i - 1].thread = i;
968 continue;
969 }
970 }
971 }
972
973
974 /*
975 * Parses a subject header of the type "multipart message subject (01/42)"
976 * into a MultiPartInfo struct, or fails if the message subject isn't in the
977 * right form.
978 *
979 * @return nonzero on success
980 */
981 int
982 global_get_multipart_info(
983 int aindex,
984 MultiPartInfo *setme)
985 {
986 int i, j, offi, offj;
987 MultiPartInfo setmei, setmej;
988
989 i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
990 j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
991
992 /* Ok i hits first */
993 if (offi > offj) {
994 *setme = setmei;
995 return i;
996 }
997
998 /* Its j or they are both the same (which must be zero!) so we don't care */
999 *setme = setmej;
1000 return j;
1001 }
1002
1003
1004 static int
1005 global_look_for_multipart_info(
1006 int aindex,
1007 MultiPartInfo* setme,
1008 char start,
1009 char stop,
1010 int *offset)
1011 {
1012 char *subj;
1013 char *pch;
1014 MultiPartInfo tmp;
1015
1016 *offset = 0;
1017
1018 /* entry assertions */
1019 assert(0 <= aindex && aindex < top_art && "invalid index");
1020 assert(setme != NULL && "setme must not be NULL");
1021
1022 /* parse the message */
1023 subj = arts[aindex].subject;
1024 pch = strrchr(subj, start);
1025 if (!pch || !isdigit((int) pch[1]))
1026 return 0;
1027
1028 tmp.arts_index = aindex;
1029 tmp.subject_compare_len = pch - subj;
1030 tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1031 if (*pch != '/' && *pch != '|')
1032 return 0;
1033
1034 if (!isdigit((int) pch[1]))
1035 return 0;
1036
1037 tmp.total = (int) strtol(pch + 1, &pch, 10);
1038 if (*pch != stop)
1039 return 0;
1040
1041 /*
1042 * skip "blah (00/102)" or "blah (103/102)" subjects
1043 */
1044 if (tmp.part_number < 1 || tmp.part_number > tmp.total)
1045 return 0;
1046
1047 tmp.subject = subj;
1048 *setme = tmp;
1049 *offset = pch - subj;
1050 return 1;
1051 }
1052
1053
1054 t_bool
1055 global_look_for_multipart(
1056 int aindex,
1057 char start,
1058 char stop)
1059 {
1060 char *pch;
1061
1062 pch = strrchr(arts[aindex].subject, start);
1063 if (!pch || !isdigit((int) pch[1]))
1064 return FALSE;
1065
1066 strtol(pch + 1, &pch, 10);
1067 if (*pch != '/' && *pch != '|')
1068 return FALSE;
1069
1070 if (!isdigit((int) pch[1]))
1071 return FALSE;
1072
1073 strtol(pch + 1, &pch, 10);
1074 if (*pch != stop)
1075 return FALSE;
1076
1077 arts[aindex].multipart_subj = TRUE;
1078 return TRUE;
1079 }
1080
1081
1082 /*
1083 * Tries to find all the parts to the multipart message pointed to by
1084 * aindex.
1085 *
1086 * @return on success, the number of parts found. On failure, zero if not
1087 * a multipart or the negative value of the first missing part in case of
1088 * tagging.
1089 * @param aindex index pointing to one of the messages in a multipart
1090 * message.
1091 * @param malloc_and_setme_info on success, set to a malloced array the
1092 * parts found.
1093 */
1094 int
1095 global_get_multiparts(
1096 int aindex,
1097 MultiPartInfo **malloc_and_setme_info,
1098 t_bool tagging)
1099 {
1100 int i, part_index, part_cnt = 0;
1101 MultiPartInfo tmp, tmp2;
1102 MultiPartInfo *info = NULL;
1103
1104 /* entry assertions */
1105 assert(0 <= aindex && aindex < top_art && "Invalid index");
1106 assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1107
1108 /* make sure this is a multipart message... */
1109 if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1110 return 0;
1111
1112 /* make a temporary buffer to hold the multipart info... */
1113 info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
1114
1115 /* zero out part-number for the repost check below */
1116 for (i = 0; i < tmp.total; ++i) {
1117 info[i].total = tmp.total; /* Added this for thread_by_multipart */
1118 info[i].part_number = -1;
1119 }
1120
1121 /* try to find all the multiparts... */
1122 for (i = (tagging ? 0 : aindex); i < top_art; i++) {
1123 if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
1124 continue;
1125
1126 if (!global_get_multipart_info(i, &tmp2))
1127 continue;
1128
1129 /* 'test (1/5)' is not the same as 'test (1/15)' */
1130 if (tmp.total != tmp2.total)
1131 continue;
1132
1133 part_index = tmp2.part_number - 1;
1134
1135 /* repost check: do we already have this part? */
1136 if (info[part_index].part_number != -1) {
1137 assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1138 continue;
1139 }
1140
1141 /* we have a match, hooray! */
1142 info[part_index] = tmp2;
1143
1144 arts[i].multipart_subj = FALSE;
1145
1146 /* all parts found? */
1147 if (++part_cnt == tmp.total)
1148 break;
1149 }
1150
1151 /* see if we got them all. */
1152 if (tagging) {
1153 for (i = 0; i < tmp.total; ++i) {
1154 if (info[i].part_number != i + 1) {
1155 free(info);
1156 return -(i + 1); /* missing part #(i+1) */
1157 }
1158 }
1159 }
1160
1161 /* looks like a success .. */
1162 *malloc_and_setme_info = info;
1163 return tmp.total;
1164 }
1165
1166
1167 /*
1168 * The algorithm uses the tag multipart searches to thread articles together.
1169 */
1170 static void
1171 thread_by_multipart(
1172 void)
1173 {
1174 int i, j, threadNum;
1175 MultiPartInfo *minfo = NULL;
1176
1177 for_each_art(i) {
1178 if (!global_look_for_multipart(i, '[', ']'))
1179 global_look_for_multipart(i, '(', ')');
1180 }
1181
1182 for_each_art(i) {
1183 if (!arts[i].multipart_subj)
1184 continue;
1185
1186 if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
1187 FreeAndNull(minfo);
1188 arts[i].multipart_subj = FALSE;
1189 continue;
1190 }
1191
1192 threadNum = -1;
1193 for (j = minfo[0].total - 1; j >= 0; j--) {
1194 if (minfo[j].part_number != -1) {
1195 if (threadNum != -1) {
1196 arts[minfo[j].arts_index].thread = threadNum;
1197 arts[threadNum].prev = minfo[j].arts_index;
1198 }
1199 threadNum = minfo[j].arts_index;
1200 }
1201 }
1202 FreeAndNull(minfo);
1203 arts[i].multipart_subj = FALSE;
1204 if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
1205 show_progress(_("Threading by multipart"), i, top_art);
1206 }
1207 }
1208
1209
1210 /*
1211 * Go through the articles in arts[] and create threads. There are
1212 * 5 strategies currently defined :
1213 *
1214 * THREAD_NONE No threading
1215 * THREAD_SUBJ Threads are created using like Subject lines
1216 * THREAD_REFS Threads are created using the References headers
1217 * THREAD_BOTH Threads created using References and then Subject
1218 * THREAD_MULTI Threads created using Subject to search for Multiparts
1219 * THREAD_PERC Threads based upon a char for char match of greater than x%
1220 *
1221 * .thread and .prev are used to hold the threading information, see tin.h for
1222 * more information
1223 * Only process valid (unexpired) articles we haven't visited yet
1224 * (ie arts[].thread == ART_UNTHREADED)
1225 *
1226 * The rethread parameter is used to force the deletion of existing threading
1227 * information before threading which happens anyway expect when using
1228 * THREAD_NONE (I don't immediately see how this is useful)
1229 */
1230 /* TODO: rewrite that user can easily combine different 'threading'
1231 * methods, i.e:
1232 * - thread_by_multipart() + collate_subjects()
1233 */
1234 void
1235 make_threads(
1236 struct t_group *group,
1237 t_bool rethread)
1238 {
1239 if (!cmd_line && !batch_mode) {
1240 info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
1241 my_flush();
1242 }
1243
1244 #ifdef DEBUG
1245 if (debug & DEBUG_MISC)
1246 error_message(2, "rethread=[%d] thread_articles=[%d] attr_thread_articles=[%d]",
1247 rethread, tinrc.thread_articles, group->attribute->thread_articles);
1248 #endif /* DEBUG */
1249
1250 /*
1251 * Sort all the articles using the preferred method
1252 * When find_base() is called, the bases are created ordered
1253 * on arts[] and so the base messages under all threading systems
1254 * will be sorted in this way.
1255 */
1256 sort_arts(group->attribute->sort_article_type);
1257
1258 /*
1259 * Reset all the ptrs to articles following the above sort
1260 */
1261 clear_art_ptrs();
1262
1263 /*
1264 * The threading pointers need to be reset if re-threading
1265 * If using ref threading, revector the links back to the articles
1266 */
1267 if (rethread || group->attribute->thread_articles) {
1268 int i;
1269
1270 for_each_art(i) {
1271 if (arts[i].thread >= 0)
1272 arts[i].thread = ART_UNTHREADED;
1273
1274 arts[i].prev = ART_NORMAL;
1275
1276 /* Should never happen if tree is built properly */
1277 if (arts[i].refptr == NULL) {
1278 #ifdef DEBUG
1279 if (debug & DEBUG_REFS) {
1280 my_fprintf(stderr, "\nError : art->refptr is NULL\n");
1281 my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1282 my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1283 my_fprintf(stderr, "From : %s\n", arts[i].from);
1284 assert(arts[i].refptr != NULL);
1285 } else
1286 #endif /* DEBUG */
1287 continue;
1288 }
1289 arts[i].refptr->article = i;
1290 }
1291 }
1292
1293 /*
1294 * Do the right thing according to the threading strategy
1295 */
1296 switch (group->attribute->thread_articles) {
1297 case THREAD_NONE:
1298 break;
1299
1300 case THREAD_SUBJ:
1301 thread_by_subject();
1302 break;
1303
1304 case THREAD_REFS:
1305 thread_by_reference();
1306 break;
1307
1308 case THREAD_BOTH:
1309 thread_by_reference();
1310 collate_subjects();
1311 break;
1312
1313 case THREAD_MULTI:
1314 thread_by_multipart();
1315 break;
1316
1317 case THREAD_PERC:
1318 thread_by_percentage(100 - group->attribute->thread_perc);
1319 break;
1320
1321 default: /* not reached */
1322 break;
1323 }
1324
1325 /*
1326 * Rebuild base[]
1327 */
1328 find_base(group);
1329 }
1330
1331
1332 static t_compfunc
1333 eval_sort_arts_func(
1334 unsigned int sort_art_type)
1335 {
1336 switch (sort_art_type) {
1337 case SORT_ARTICLES_BY_NOTHING: /* don't sort at all */
1338 return artnum_comp;
1339
1340 case SORT_ARTICLES_BY_SUBJ_DESCEND:
1341 return subj_comp_desc;
1342
1343 case SORT_ARTICLES_BY_SUBJ_ASCEND:
1344 return subj_comp_asc;
1345
1346 case SORT_ARTICLES_BY_FROM_DESCEND:
1347 return from_comp_desc;
1348
1349 case SORT_ARTICLES_BY_FROM_ASCEND:
1350 return from_comp_asc;
1351
1352 case SORT_ARTICLES_BY_DATE_DESCEND:
1353 return date_comp_desc;
1354
1355 case SORT_ARTICLES_BY_DATE_ASCEND:
1356 return date_comp_asc;
1357
1358 case SORT_ARTICLES_BY_SCORE_DESCEND:
1359 return score_comp_desc;
1360
1361 case SORT_ARTICLES_BY_SCORE_ASCEND:
1362 return score_comp_asc;
1363
1364 case SORT_ARTICLES_BY_LINES_DESCEND:
1365 return lines_comp_desc;
1366
1367 case SORT_ARTICLES_BY_LINES_ASCEND:
1368 return lines_comp_asc;
1369
1370 default:
1371 break;
1372 }
1373 return NULL;
1374 }
1375
1376
1377 void
1378 sort_arts(
1379 unsigned int sort_art_type)
1380 {
1381 t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1382
1383 if (comp_func)
1384 SortBy(comp_func);
1385 }
1386
1387
1388 static void
1389 sort_base(
1390 unsigned int sort_threads_type)
1391 {
1392 switch (sort_threads_type) {
1393 case SORT_THREADS_BY_SCORE_DESCEND:
1394 case SORT_THREADS_BY_SCORE_ASCEND:
1395 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1396 break;
1397
1398 case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
1399 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
1400 break;
1401
1402 case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
1403 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
1404 break;
1405 }
1406 }
1407
1408
1409 /*
1410 * This is called to get header info for articles not already found in the
1411 * overview files.
1412 * Code reads (max_lineno) lines of article to catch headers like Archive-name:
1413 * which are not normally included in XOVER or even the normal block of headers.
1414 * How this is supposed to be useful when 99% of the time we'll have overview
1415 * data I don't know...
1416 * TODO: move Archive-name: parsing to article body parsing, remove the
1417 * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
1418 * TODO: currgrp->auto_save is set, otherwise it is redundant info
1419 */
1420 static t_bool
1421 parse_headers(
1422 FILE *fp,
1423 struct t_article *h)
1424 {
1425 char art_from_addr[HEADER_LEN];
1426 char art_full_name[HEADER_LEN];
1427 char *s, *hdr, *ptr;
1428 unsigned int lineno = 0;
1429 unsigned int max_lineno = 25;
1430 t_bool got_from, got_lines, got_received;
1431
1432 got_from = got_lines = got_received = FALSE;
1433
1434 while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1435 /*
1436 * Look for the end of information which tin wants to get.
1437 * Applies when reading local spool and via NNTP.
1438 */
1439
1440 /*
1441 * as Archive-name: is placed in the body it's safe to exit
1442 * the loop if it was found. Don't mix up with Archive: from
1443 * RFC 5536.
1444 */
1445 if (lineno++ > max_lineno || h->archive)
1446 break;
1447
1448 unfold_header(ptr);
1449 switch (my_toupper((unsigned char) *ptr)) {
1450 case 'A': /* Archive-name: optional */
1451 /*
1452 * Archive-name: {name}/{part|patch}{number}
1453 * eg, acorn/faq/part01
1454 */
1455 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
1456 if ((s = strrchr(hdr, '/')) != NULL) {
1457 struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
1458
1459 if (STRNCASECMPEQ(s + 1, "part", 4)) {
1460 archptr->partnum = my_strdup(s + 5);
1461 archptr->ispart = TRUE;
1462 } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
1463 archptr->partnum = my_strdup(s + 6);
1464 archptr->ispart = FALSE;
1465 } else { /* part or patch must be present */
1466 free(archptr);
1467 continue;
1468 }
1469 strtok(archptr->partnum, "\n");
1470 *s = '\0';
1471 archptr->name = hash_str(hdr);
1472 h->archive = archptr;
1473 }
1474 }
1475 break;
1476
1477 case 'D': /* Date: mandatory */
1478 if (!h->date) {
1479 if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1480 h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1481 }
1482 break;
1483
1484 case 'F': /* From: mandatory */
1485 if (!got_from) {
1486 if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1487 h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1488 h->from = hash_str(buffer_to_ascii(art_from_addr));
1489 if (*art_full_name)
1490 h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
1491 got_from = TRUE;
1492 }
1493 }
1494 break;
1495
1496 case 'L': /* Lines: optional */
1497 if (!got_lines) {
1498 if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1499 h->line_count = atoi(hdr);
1500 got_lines = TRUE;
1501 }
1502 }
1503 break;
1504
1505 case 'M': /* Message-ID: mandatory */
1506 if (!h->msgid) {
1507 if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1508 h->msgid = my_strdup(hdr);
1509 }
1510 break;
1511
1512 /* for Path:-filter when reading from local spool */
1513 case 'P': /* Path: */
1514 if (!h->path) {
1515 if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1516 h->path = my_strdup(hdr);
1517 }
1518 break;
1519
1520 case 'R': /* References: optional */
1521 if (!h->refs) {
1522 if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1523 h->refs = my_strdup(hdr);
1524 }
1525
1526 /* Received: If found it's probably a mail article */
1527 if (!got_received) {
1528 if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
1529 max_lineno <<= 1; /* double the max number of line to read for mails */
1530 got_received = TRUE;
1531 }
1532 }
1533 break;
1534
1535 case 'S': /* Subject: mandatory */
1536 if (!h->subject) {
1537 if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
1538 #ifdef HAVE_UNICODE_NORMALIZATION
1539 if (IS_LOCAL_CHARSET("UTF-8"))
1540 s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1541 else
1542 #endif /* HAVE_UNICODE_NORMALIZATION */
1543 s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1544
1545 h->subject = hash_str(s);
1546 free(s);
1547 }
1548 }
1549 break;
1550
1551 case 'X': /* Xref: optional */
1552 if (!h->xref) {
1553 if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1554 h->xref = my_strdup(hdr);
1555 }
1556 break;
1557
1558 default:
1559 break;
1560 } /* switch */
1561
1562 } /* while */
1563
1564 #ifdef NNTP_ABLE
1565 if (ptr)
1566 drain_buffer(fp);
1567 #endif /* NNTP_ABLE */
1568
1569 if (tin_errno)
1570 return FALSE;
1571
1572 /*
1573 * The son of RFC 1036 states that the following hdrs are mandatory. It
1574 * also states that Subject, Newsgroups and Path are too. Ho hum.
1575 *
1576 * What about reading mail from local spool via ~/.tin/active.mail,
1577 * they might not have a Message-ID but got_received is very likely to
1578 * be true.
1579 */
1580 if (got_from && h->date && h->msgid) {
1581 if (!h->subject)
1582 h->subject = hash_str("<No subject>");
1583
1584 #ifdef DEBUG
1585 if (debug & DEBUG_FILTER)
1586 debug_print_header(h);
1587 #endif /* DEBUG */
1588 return TRUE;
1589 }
1590
1591 return FALSE;
1592 }
1593
1594 #ifdef NNTP_ABLE
1595 /*
1596 * Loop over arts[] and find ranges without Path: header
1597 * If there are any try to optimize the ranges regarding traffic consumption
1598 * Start optimization if at least MIN_CNT ranges exist
1599 * If there are more than MAX_CNT ranges after optimization, fetch all in one
1600 * big range
1601 */
1602 #define MIN_CNT 10
1603 #define MAX_CNT 50
1604 static struct t_article_range *
1605 build_range_list(
1606 t_artnum min,
1607 t_artnum max,
1608 int *range_cnt)
1609 {
1610 int i, gap_cnt = 0;
1611 struct t_article_range *res = NULL, *gap_list, *curr, *from;
1612 t_artnum new_end;
1613
1614 new_end = T_ARTNUM_CONST(0);
1615 gap_list = my_malloc(sizeof(struct t_article_range));
1616 curr = gap_list;
1617 curr->start = min;
1618 curr->end = max;
1619 curr->cnt = T_ARTNUM_CONST(0);
1620 curr->next = NULL;
1621
1622 for_each_art(i) {
1623 if (arts[i].artnum < min)
1624 continue;
1625 if (arts[i].artnum > max)
1626 break;
1627 if (arts[i].path) {
1628 for (; i < top_art && arts[i].path; i++)
1629 ;
1630 /*
1631 * the current art has no path -> we use this one
1632 * if we reached top_art all arts have path
1633 * so we use max
1634 */
1635 curr->start = i == top_art ? max : arts[i--].artnum;
1636 } else {
1637 for (; i < top_art && !arts[i].path; i++)
1638 ;
1639 /* the current art has path -> we use the last one */
1640 new_end = curr->end = arts[--i].artnum;
1641 }
1642 if (new_end) {
1643 curr->cnt = curr->end - curr->start + 1;
1644 curr->next = my_malloc(sizeof(struct t_article_range));
1645 curr = curr->next;
1646 curr->start = new_end;
1647 curr->end = max;
1648 curr->cnt = T_ARTNUM_CONST(0);
1649 curr->next = NULL;
1650 new_end = T_ARTNUM_CONST(0);
1651 }
1652 }
1653
1654 curr = gap_list;
1655 while (curr && curr->cnt) {
1656 ++gap_cnt;
1657 # ifdef DEBUG
1658 if ((debug & DEBUG_NNTP) && verbose > 1)
1659 debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1660 # endif /* DEBUG */
1661 curr = curr->next;
1662 }
1663
1664 /*
1665 * Optimize only if there are at least MIN_CNT ranges
1666 */
1667 if (gap_cnt >= MIN_CNT) {
1668 res = my_malloc(sizeof(struct t_article_range));
1669 res->start = T_ARTNUM_CONST(0);
1670 res->end = T_ARTNUM_CONST(0);
1671 res->cnt = T_ARTNUM_CONST(0);
1672 res->next = NULL;
1673
1674 from = gap_list;
1675 curr = res;
1676 while (from) {
1677 curr->start = from->start;
1678 curr->end = from->end;
1679 curr->cnt = from->cnt;
1680 if ((from = from->next)) {
1681 /*
1682 * If the next range is grater then the gap between the current
1683 * one and the next one we build a new range including the
1684 * current one, the next one and the gap between
1685 */
1686 while (from && from->cnt >= from->start - curr->end - 1) {
1687 curr->end = from->end;
1688 from = from->next;
1689 }
1690 curr->cnt = curr->end - curr->start + 1;
1691 curr->next = my_malloc(sizeof(struct t_article_range));
1692 curr = curr->next;
1693 curr->start = T_ARTNUM_CONST(0);
1694 curr->end = T_ARTNUM_CONST(0);
1695 curr->cnt = T_ARTNUM_CONST(0);
1696 curr->next = NULL;
1697 }
1698 }
1699 }
1700
1701 /*
1702 * If there are less then MIN_CNT ranges
1703 * no res is build -> return the original list
1704 */
1705 if (res) {
1706 while (gap_list) {
1707 curr = gap_list;
1708 gap_list = curr->next;
1709 free(curr);
1710 }
1711 } else
1712 res = gap_list;
1713
1714 curr = res;
1715 gap_cnt = 0;
1716 while (curr && curr->cnt) {
1717 ++gap_cnt;
1718 # ifdef DEBUG
1719 if ((debug & DEBUG_NNTP) && verbose > 1)
1720 debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1721 # endif /* DEBUG */
1722 curr = curr->next;
1723 }
1724
1725 if (gap_cnt >= MAX_CNT) {
1726 curr = res;
1727 while (curr->next && curr->next->cnt) {
1728 res->end = curr->next->end;
1729 curr->next->cnt = 0;
1730 curr = curr->next;
1731 }
1732 res->cnt = res->end - res->start + 1;
1733 gap_cnt = 1;
1734 # ifdef DEBUG
1735 if ((debug & DEBUG_NNTP) && verbose > 1)
1736 debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1737 # endif /* DEBUG */
1738 }
1739 *range_cnt = gap_cnt;
1740
1741 return res;
1742 }
1743
1744
1745 /*
1746 * Fetch the Path header in case we want to filter on that in the given group
1747 *
1748 * Try [X]HDR first, then XPAT
1749 */
1750 static t_bool
1751 get_path_header(
1752 int cur,
1753 int cnt,
1754 struct t_group *group,
1755 t_artnum min,
1756 t_artnum max)
1757 {
1758 FILE *fp = NULL;
1759 char *prep_msg;
1760 char *buf, *ptr;
1761 char cmd[NNTP_STRLEN];
1762 t_artnum artnum, i;
1763 t_bool found = FALSE;
1764 static t_bool supported = TRUE; /* assume HDR || XPAT works */
1765
1766 if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1767 return FALSE;
1768
1769 # ifdef DEBUG
1770 if ((debug & DEBUG_NNTP) && verbose > 1)
1771 debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1772 # endif /* DEBUG */
1773
1774 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
1775 int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1776 switch (j) {
1777 case 215:
1778 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1779 # ifdef DEBUG
1780 if (debug & DEBUG_NNTP)
1781 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1782 # endif /* DEBUG */
1783 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
1784 strcat(nntp_caps.headers_range, ptr);
1785 strcat(nntp_caps.headers_range, "\n");
1786 }
1787 break;
1788
1789 default:
1790 break;
1791 }
1792 }
1793
1794 /* does HDR return Path? */
1795 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1796 do {
1797 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1798 found = TRUE;
1799 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1800 }
1801
1802 if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1803 if (min == max)
1804 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1805 else
1806 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1807 fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1808 if (!nntp_caps.hdr && fp)
1809 nntp_caps.hdr = TRUE;
1810 } else if (nntp_caps.xpat) {
1811 if (min == max)
1812 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1813 else
1814 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1815 fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1816 }
1817
1818 if (fp) {
1819 int j = 0;
1820
1821 prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1822 while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1823 # ifdef DEBUG
1824 if ((debug & DEBUG_NNTP) && verbose)
1825 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1826 # endif /* DEBUG */
1827 if ((ptr = tin_strtok(buf, " ")) == NULL)
1828 continue;
1829 artnum = atoartnum(ptr);
1830 if ((ptr = tin_strtok(NULL, " ")) == NULL)
1831 continue;
1832 for (i = j; i < top_art; i++) {
1833 if (arts[i].artnum == artnum) {
1834 FreeIfNeeded(arts[i].path);
1835 arts[i].path = my_strdup(ptr);
1836 j = i;
1837 break;
1838 }
1839 }
1840 if (++artnum % MODULO_COUNT_NUM == 0)
1841 show_progress(prep_msg, artnum - min, max - min);
1842 }
1843 free(prep_msg);
1844 return supported;
1845 }
1846
1847 /* !fp */
1848 supported = FALSE;
1849 if (nntp_caps.xpat)
1850 nntp_caps.xpat = FALSE;
1851 /* as nntp_caps.hdr may work with other headers we don't disable it */
1852
1853 # ifdef DEBUG
1854 if ((debug & DEBUG_NNTP) && verbose > 1)
1855 debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
1856 # endif /* DEBUG */
1857 return supported;
1858 }
1859 #endif /* NNTP_ABLE */
1860
1861
1862 /*
1863 * Read in an overview index file. Fields are separated by TAB.
1864 * return the number of expired articles encountered or -1 if the user aborted
1865 * the read
1866 * 'top' is set to the highest artnum read
1867 * If 'local' is set then always open local overview cache in preference to
1868 * using NNTP XOVER
1869 *
1870 * Format (mandatory as far as line count [RFC2980]):
1871 * 1. article number (ie. 183) [mandatory]
1872 * 2. Subject: line (ie. Which newsreader?) [mandatory]
1873 * 3. From: line (ie. iain@ecrc.de) [mandatory]
1874 * 4. Date: line (rfc822 format) [mandatory]
1875 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
1876 * 6. References: (ie. <message-id> ....) [optional]
1877 * 7. Byte count (Skipped - not used) [mandatory]
1878 * 8. Line count (ie. 23) [mandatory]
1879 * 9. Xref: line (ie. alt.test:389) [optional]
1880 */
1881 static int
1882 read_overview(
1883 struct t_group *group,
1884 t_artnum min,
1885 t_artnum max,
1886 t_artnum *top,
1887 t_bool local,
1888 t_bool *rebuild_cache)
1889 {
1890 FILE *fp;
1891 char *ptr;
1892 char *q;
1893 char *buf;
1894 char *group_msg;
1895 char art_full_name[HEADER_LEN];
1896 char art_from_addr[HEADER_LEN];
1897 unsigned int count;
1898 int expired = 0;
1899 t_artnum artnum;
1900 t_bool path_found = FALSE, path_in_ofmt = FALSE;
1901 struct t_article *art;
1902 size_t over_fields = 1;
1903
1904 /*
1905 * open the overview file (whether it be local or via nntp)
1906 */
1907 if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1908 return expired;
1909
1910 if (group->xmax > max)
1911 group->xmax = max;
1912
1913 group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1914
1915 /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1916 if (ofmt) {
1917 for (; ofmt[over_fields].name; over_fields++) {
1918 if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1919 path_in_ofmt = TRUE;
1920 }
1921 }
1922
1923 if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1924 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1925 ofmt[0].type = OVER_T_INT;
1926 ofmt[0].name = my_strdup("Artnum:");
1927 ofmt[1].type = OVER_T_STRING;
1928 ofmt[1].name = my_strdup("Subject:");
1929 ofmt[2].type = OVER_T_STRING;
1930 ofmt[2].name = my_strdup("From:");
1931 ofmt[3].type = OVER_T_STRING;
1932 ofmt[3].name = my_strdup("Date:");
1933 ofmt[4].type = OVER_T_STRING;
1934 ofmt[4].name = my_strdup("Message-ID:");
1935 ofmt[5].type = OVER_T_STRING;
1936 ofmt[5].name = my_strdup("References:");
1937 ofmt[6].type = OVER_T_INT;
1938 ofmt[6].name = my_strdup("Bytes:");
1939 ofmt[7].type = OVER_T_INT;
1940 ofmt[7].name = my_strdup("Lines:");
1941 ofmt[8].type = OVER_T_ERROR;
1942 ofmt[8].name = NULL;
1943 over_fields = 7;
1944 }
1945
1946 while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1947 #ifdef DEBUG
1948 if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1949 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1950 #endif /* DEBUG */
1951
1952 if (need_resize) {
1953 handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
1954 need_resize = cNo;
1955 }
1956
1957 /*
1958 * Read artnum
1959 */
1960 if ((ptr = tin_strtok(buf, "\t")) == NULL)
1961 continue;
1962
1963 /*
1964 * read the article number, guaranteed to be the first field
1965 */
1966 artnum = atoartnum(ptr);
1967
1968 /*
1969 * artnum field invalid/corrupt or is 1st line of local cached overview
1970 * (group name)
1971 */
1972 if (artnum <= 0)
1973 continue;
1974
1975 /*
1976 * skip artnums below the given minimum (getart_limit)
1977 */
1978 if (artnum < min)
1979 continue;
1980
1981 /*
1982 * Check to make sure article in nov file has not expired in group
1983 */
1984 if (artnum < group->xmin) {
1985 expired++;
1986 continue;
1987 }
1988
1989 /*
1990 * artnum in overview data higher than groups high mark
1991 *
1992 * TODO: - warn user about broken overviews?
1993 * - try to parse the Xref:-line to get the correct artnum
1994 * - see also parse_unread_arts()
1995 */
1996 if (artnum > group->xmax)
1997 continue;
1998
1999 if (top_art >= max_art)
2000 expand_art();
2001
2002 art = &arts[top_art];
2003 set_article(art);
2004 art->artnum = *top = artnum;
2005
2006 /*
2007 * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
2008 * to check for additions like we do with xref_supported
2009 */
2010 for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
2011 /* skip unexpected tailing fields */
2012 if (count > over_fields) {
2013 #ifdef DEBUG
2014 if ((debug & DEBUG_NNTP) && verbose > 1)
2015 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
2016 #endif /* DEBUG */
2017
2018 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
2019 if (count == over_fields + 1) {
2020 if (!strncasecmp(ptr, "Xref: ", 6)) {
2021 #ifdef DEBUG
2022 if ((debug & DEBUG_NNTP) && verbose > 1)
2023 debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
2024 #endif /* DEBUG */
2025 over_fields++;
2026 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2027 ofmt[over_fields].type = OVER_T_FSTRING;
2028 ofmt[over_fields].name = my_strdup("Xref:");
2029 ofmt[over_fields + 1].type = OVER_T_ERROR;
2030 ofmt[over_fields + 1].name = NULL;
2031 xref_supported = TRUE;
2032 } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
2033 #ifdef DEBUG
2034 if ((debug & DEBUG_NNTP) && verbose > 1)
2035 debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
2036 #endif /* DEBUG */
2037 over_fields++;
2038 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
2039 ofmt[over_fields].type = OVER_T_FSTRING;
2040 ofmt[over_fields].name = my_strdup("Path:");
2041 ofmt[over_fields + 1].type = OVER_T_ERROR;
2042 ofmt[over_fields + 1].name = NULL;
2043 xref_supported = TRUE;
2044 } else
2045 continue;
2046 } else
2047 continue;
2048 }
2049
2050 /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
2051 if (expensive_over_parse) { /* strange order */
2052 /* mandatory fields */
2053 if (ofmt[count].type == OVER_T_STRING) {
2054 if (!strcasecmp(ofmt[count].name, "Subject:")) {
2055 if (*ptr) {
2056 #ifdef HAVE_UNICODE_NORMALIZATION
2057 if (IS_LOCAL_CHARSET("UTF-8"))
2058 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2059 else
2060 #endif /* HAVE_UNICODE_NORMALIZATION */
2061 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2062
2063 art->subject = hash_str(q);
2064 free(q);
2065 } else {
2066 art->subject = hash_str("");
2067 #ifdef DEBUG
2068 if ((debug & DEBUG_NNTP) && verbose > 1)
2069 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2070 #endif /* DEBUG */
2071 }
2072 continue;
2073 }
2074
2075 if (!strcasecmp(ofmt[count].name, "From:")) {
2076 if (*ptr) {
2077 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2078 art->from = hash_str(buffer_to_ascii(art_from_addr));
2079 if (*art_full_name)
2080 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2081 } else {
2082 art->from = hash_str("");
2083 #ifdef DEBUG
2084 if ((debug & DEBUG_NNTP) && verbose > 1)
2085 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2086 #endif /* DEBUG */
2087 }
2088 continue;
2089 }
2090
2091 if (!strcasecmp(ofmt[count].name, "Date:")) {
2092 art->date = parsedate(ptr, (TIMEINFO *) 0);
2093 #ifdef DEBUG
2094 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2095 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2096 #endif /* DEBUG */
2097 continue;
2098 }
2099
2100 if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2101 if (*ptr) {
2102 FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2103 art->msgid = my_strdup(ptr);
2104 } else {
2105 art->msgid = NULL;
2106 #ifdef DEBUG
2107 if ((debug & DEBUG_NNTP) && verbose > 1)
2108 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2109 #endif /* DEBUG */
2110 }
2111 continue;
2112 }
2113
2114 if (!strcasecmp(ofmt[count].name, "References:")) {
2115 if (*ptr) {
2116 FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2117 art->refs = my_strdup(ptr);
2118 } else
2119 art->refs = NULL;
2120 continue;
2121 }
2122
2123 /*
2124 * non std. fields when doing
2125 * expensive overview parsing (very
2126 * rare, just happens if RFC 3977
2127 * 8.4.2 is violated) go here
2128 */
2129 /* for Path:-filter */
2130 if (!strcasecmp(ofmt[count].name, "Path:")) {
2131 if (!path_found)
2132 path_found = TRUE;
2133 if (*ptr) {
2134 FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2135 art->path = my_strdup(ptr);
2136 } else
2137 art->path = NULL;
2138 continue;
2139 }
2140 }
2141 /* metadata fields */
2142 if (ofmt[count].type == OVER_T_INT) {
2143 if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2144 if (*ptr) {
2145 #ifdef DEBUG
2146 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2147 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2148 #endif /* DEBUG */
2149 }
2150 continue;
2151 }
2152
2153 if (!strcasecmp(ofmt[count].name, "Lines:")) {
2154 if (*ptr) {
2155 if (isdigit((unsigned char) *ptr))
2156 art->line_count = atoi(ptr);
2157 else {
2158 art->line_count = 0;
2159 #ifdef DEBUG
2160 if ((debug & DEBUG_NNTP) && verbose > 1)
2161 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2162 #endif /* DEBUG */
2163 }
2164 } else
2165 art->line_count = 0;
2166 continue;
2167 }
2168 }
2169 } else { /* first 7 fields are in RFC 3977 order */
2170 switch (count) {
2171 case 1: /* Subject: */
2172 if (*ptr) {
2173 #ifdef HAVE_UNICODE_NORMALIZATION
2174 if (IS_LOCAL_CHARSET("UTF-8"))
2175 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2176 else
2177 #endif /* HAVE_UNICODE_NORMALIZATION */
2178 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2179
2180 art->subject = hash_str(q);
2181 free(q);
2182 } else {
2183 art->subject = hash_str("");
2184 #ifdef DEBUG
2185 if ((debug & DEBUG_NNTP) && verbose > 1)
2186 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2187 #endif /* DEBUG */
2188 }
2189 break;
2190
2191 case 2: /* From: */
2192 if (*ptr) {
2193 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2194 art->from = hash_str(buffer_to_ascii(art_from_addr));
2195 if (*art_full_name)
2196 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2197 } else {
2198 art->from = hash_str("");
2199 #ifdef DEBUG
2200 if ((debug & DEBUG_NNTP) && verbose > 1)
2201 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2202 #endif /* DEBUG */
2203 }
2204 break;
2205
2206 case 3: /* Date: */
2207 art->date = parsedate(ptr, (TIMEINFO *) 0);
2208 #ifdef DEBUG
2209 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2210 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2211 #endif /* DEBUG */
2212 break;
2213
2214 case 4: /* Message-ID: */
2215 if (*ptr)
2216 art->msgid = my_strdup(ptr);
2217 else {
2218 art->msgid = NULL;
2219 #ifdef DEBUG
2220 if ((debug & DEBUG_NNTP) && verbose > 1)
2221 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2222 #endif /* DEBUG */
2223 }
2224 break;
2225
2226 case 5: /* References: */
2227 if (*ptr)
2228 art->refs = my_strdup(ptr);
2229 else
2230 art->refs = NULL;
2231 break;
2232
2233 case 6: /* :bytes || Bytes: */
2234 if (*ptr) {
2235 #ifdef DEBUG
2236 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2237 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2238 #endif /* DEBUG */
2239 }
2240 break;
2241
2242 case 7: /* :lines || Lines: */
2243 if (*ptr) {
2244 if (isdigit((unsigned char) *ptr))
2245 art->line_count = atoi(ptr);
2246 else {
2247 art->line_count = 0;
2248 #ifdef DEBUG
2249 if ((debug & DEBUG_NNTP) && verbose > 1)
2250 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2251 #endif /* DEBUG */
2252 }
2253 } else
2254 art->line_count = 0;
2255 break;
2256
2257 default:
2258 break;
2259 }
2260 }
2261
2262 /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2263 if (ofmt[count].type == OVER_T_FSTRING) {
2264 if (*ptr) {
2265 if (!strcasecmp(ofmt[count].name, "Xref:")) {
2266 if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2267 FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2268 art->xref = my_strdup(q);
2269 }
2270 #ifdef DEBUG
2271 else {
2272 if ((debug & DEBUG_NNTP) && verbose > 1)
2273 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2274 }
2275 #endif /* DEBUG */
2276 continue;
2277 }
2278 /*
2279 * handling of addition overview fields
2280 * goes here
2281 */
2282 #ifdef DEBUG
2283 if ((debug & DEBUG_NNTP) && verbose > 1)
2284 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2285 #endif /* DEBUG */
2286 /* if we're lucky we've Path in NOV */
2287 /*
2288 * if reading locally cached overview data try
2289 * path regardless of the server OVERVIEW.FMT
2290 */
2291 if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2292 if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2293 if (!path_found)
2294 path_found = TRUE;
2295 FreeIfNeeded(art->path);
2296 art->path = my_strdup(q);
2297 #ifdef DEBUG
2298 if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2299 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2300 #endif /* DEBUG */
2301
2302 }
2303 continue;
2304 }
2305 }
2306 continue;
2307 }
2308 }
2309
2310 /*
2311 * RFC says Message-ID is mandatory in newsgroups (but not in
2312 * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2313 * threading in mailgroups
2314 */
2315 if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2316 continue;
2317
2318 /* we might lose accuracy here, but that shouldn't hurt */
2319 if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2320 show_progress(group_msg, artnum - min, max - min);
2321
2322 top_art++; /* Basically this statement commits the article */
2323 }
2324
2325 free(group_msg);
2326 TIN_FCLOSE(fp);
2327
2328 if (tin_errno)
2329 return -1;
2330
2331 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2332 if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
2333 char cbuf[HEADER_LEN];
2334 int i;
2335 static t_bool found;
2336 static t_bool first = TRUE;
2337
2338 if (first) {
2339 found = TRUE;
2340 /*
2341 * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2342 */
2343 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
2344 if (!*nntp_caps.headers_range) {
2345 i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2346
2347 found = FALSE;
2348 switch (i) {
2349 case 215:
2350 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2351 # ifdef DEBUG
2352 if (debug & DEBUG_NNTP)
2353 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2354 # endif /* DEBUG */
2355 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2356 found = TRUE;
2357 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
2358 strcat(nntp_caps.headers_range, ptr);
2359 strcat(nntp_caps.headers_range, "\n");
2360 }
2361 break;
2362
2363 default:
2364 break;
2365 }
2366 first = FALSE;
2367 } else {
2368 found = FALSE;
2369 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
2370 do {
2371 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2372 found = TRUE;
2373 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2374 }
2375 }
2376 }
2377 }
2378
2379 if (found) {
2380 snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2381 group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2382 if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2383 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2384 # ifdef DEBUG
2385 if ((debug & DEBUG_NNTP) && verbose)
2386 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2387 # endif /* DEBUG */
2388
2389 artnum = atoartnum(ptr);
2390 if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2391 continue;
2392 art = &arts[top_art];
2393 set_article(art);
2394 if (!art->xref && !strstr(ptr, "(none)")) {
2395 if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2396 continue;
2397 ptr = q;
2398 while (*ptr && isspace((int) *ptr))
2399 ptr++;
2400 q = strchr(ptr, '\n');
2401 if (q)
2402 *q = '\0';
2403 art->xref = my_strdup(ptr);
2404 }
2405 /* we might lose accuracy here, but that shouldn't hurt */
2406 if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2407 show_progress(group_msg, artnum - min, max - min);
2408 }
2409 }
2410 free(group_msg);
2411 }
2412 }
2413 #endif /* NNTP_ABLE && XHDR_XREF */
2414
2415 if (local) {
2416 #ifdef NNTP_ABLE
2417 if (filter_on_path(group)) {
2418 struct t_article_range *ranges, *curr;
2419 t_bool supported = TRUE;
2420 int curr_range, range_cnt;
2421
2422 /*
2423 * Get the ranges without Path: header and try to fetch the
2424 * headers
2425 */
2426 if ((ranges = build_range_list(min, *top, &range_cnt))) {
2427 curr = ranges;
2428 curr_range = 1;
2429 while (curr && supported) {
2430 if (curr->cnt)
2431 supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2432 curr = curr->next;
2433 }
2434 if (!supported && path_in_ofmt) {
2435 /*
2436 * fetching Path: headers via [X]HDR or XPAT has failed
2437 * Path: is in the servers overview so let the next
2438 * read_overview() fetch them
2439 */
2440 free_art_array();
2441 free_msgids();
2442 top_art = 0;
2443 *top = T_ARTNUM_CONST(0);
2444 expired = 0;
2445 }
2446 *rebuild_cache = TRUE;
2447 while (ranges) {
2448 curr = ranges;
2449 ranges = curr->next;
2450 free(curr);
2451 }
2452 }
2453 }
2454 #endif /* NNTP_ABLE */
2455 } else
2456 if (!path_found && filter_on_path(group)) {
2457 #ifdef NNTP_ABLE
2458 if (!get_path_header(1, 1, group, min, *top))
2459 #endif /* NNTP_ABLE */
2460 wait_message(2, _(txt_cannot_filter_on_path));
2461 }
2462 return expired;
2463 }
2464
2465
2466 /*
2467 * Write an Nov/Xover index file. Fields are separated by '\t'.
2468 *
2469 * Format:
2470 * 1. article number (ie. 183) [mandatory]
2471 * 2. Subject: line (ie. Which newsreader?) [mandatory]
2472 * 3. From: line (ie. iain@ecrc.de) [mandatory]
2473 * 4. Date: line (rfc822 format) [mandatory]
2474 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
2475 * 6. References: (ie. <message-id> ....) [optional]
2476 * 7. Byte count (Skipped - not used) [mandatory]
2477 * 8. Line count (ie. 23) [mandatory]
2478 * 9. Xref: line (ie. alt.test:389) [optional]
2479 *
2480 * TODO: as we don't use the original data, we currently can't store
2481 * the data (from/subject) in the original charset (we don't store
2482 * that info). this has the advantage that we can avoid raw 8bit data
2483 * in our overviews, but the disadvantage that we might store the data
2484 * with a wrong charset and thus lose information. a similar problem
2485 * exists with the data for the from:-line, we don't store it in the
2486 * original format, whenever our from-parser (partially) fails we'll
2487 * lose information in our overviews (but those couldn't be handled
2488 * by tin anyway, so this is not a real problem).
2489 * long-term solution: store the original data in the overview
2490 * (tin has to handle raw 8bit data and other ugly stuff in the
2491 * overviews anyway and thus we preserver as much info as possible)
2492 * this would require some changes in read_overview() and
2493 * parse_headers(): don't do the decoding/unfolding there, but in a
2494 * second pass right after write_overview(), or two additional fields
2495 * which hold the raw data for from/subject. the latter has the
2496 * disadvantage that it costs (much) more memory.
2497 */
2498 static void
2499 write_overview(
2500 struct t_group *group)
2501 {
2502 FILE *fp;
2503 int i;
2504 struct t_article *article;
2505 #ifdef CHARSET_CONVERSION
2506 int c = -1;
2507 #endif /* CHARSET_CONVERSION */
2508
2509 /*
2510 * Can't write or caching is off or getart_limit is set
2511 */
2512 if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
2513 return;
2514
2515 if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2516 return;
2517
2518 if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
2519 SortBy(artnum_comp);
2520
2521 /*
2522 * Needed to preserve uniqueness in hashed private overview files
2523 */
2524 fprintf(fp, "%s\n", group->name);
2525
2526 #ifdef CHARSET_CONVERSION
2527 /* get undeclared_charset number if required */
2528 if (group->attribute->undeclared_charset) {
2529 for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2530 if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2531 c = i;
2532 break;
2533 }
2534 }
2535 }
2536 #endif /* CHARSET_CONVERSION */
2537
2538 if (verbose && batch_mode) /* -> lang.c */
2539 wait_message(0, _("Writing %s\n"), group->name);
2540
2541 for_each_art(i) {
2542 char *p;
2543 char *q, *ref;
2544
2545 article = &arts[i];
2546
2547 if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2548 ref = NULL;
2549
2550 if (!group->attribute->post_8bit_header) { /* write encoded data */
2551 /*
2552 * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2553 * here and in print_from() in the CHARSET_CONVERSION case.
2554 * note that this requires something like
2555 * buffer_to_network(article->subject, "UTF-8");
2556 * right before the rfc1522_encode() call.
2557 *
2558 * if we would cache the original undecoded data, we could
2559 * ignore stuff like this.
2560 */
2561 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
2562 /* as the subject might now be folded we have to unfold it */
2563 unfold_header(p);
2564 } else { /* raw data */
2565 p = my_strdup(article->subject);
2566 #ifdef CHARSET_CONVERSION
2567 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2568 buffer_to_network(p, c);
2569 #endif /* CHARSET_CONVERSION */
2570 }
2571
2572 /*
2573 * replace any '\t's with ' ' in the references-data
2574 *
2575 * TODO: nntpext-draft might come up with a new scheme:
2576 * For all fields, the value is processed by first
2577 * removing all US-ASCII CRLF pairs and then replacing
2578 * each remaining US-ASCII NUL, TAB, CR, or LF character
2579 * with a single US-ASCII space (for example, CR LF LF TAB
2580 * will become two spaces).
2581 */
2582 if (article->refs) {
2583 ref = q = my_strdup(article->refs);
2584 while (*q) {
2585 if (*q == '\t')
2586 *q = ' ';
2587 q++;
2588 }
2589 }
2590
2591 fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2592 article->artnum,
2593 p,
2594 #ifdef CHARSET_CONVERSION
2595 print_from(group, article, c),
2596 #else
2597 print_from(group, article, -1),
2598 #endif /* CHARSET_CONVERSION */
2599 print_date(article->date),
2600 BlankIfNull(article->msgid),
2601 BlankIfNull(ref),
2602 0, /* bytes */
2603 article->line_count);
2604
2605 if (article->xref)
2606 fprintf(fp, "\tXref: %s", article->xref);
2607
2608 if (article->path)
2609 fprintf(fp, "\tPath: %s", article->path);
2610
2611 fprintf(fp, "\n");
2612 free(p);
2613 if (article->refs) {
2614 FreeIfNeeded(ref);
2615 q = NULL;
2616 }
2617 }
2618 if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
2619 show_progress(_("Writing overview cache..."), i, top_art);
2620 }
2621 #ifdef HAVE_FCHMOD
2622 fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2623 /*
2624 * TODO:
2625 * add code for !HAVE_FCHMOD && HAVE_CHMOD
2626 */
2627 #endif /* HAVE_FCHMOD */
2628 fclose(fp);
2629 }
2630
2631
2632 /*
2633 * A complex little function to determine the correct overview index file
2634 * according to 'mode' (read or write)
2635 * NULL is returned if the current setup dictates otherwise
2636 *
2637 * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2638 * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2639 *
2640 * Both of these are hashed
2641 *
2642 * GROUP_TYPE_NEWS index files are a little bit more complex
2643 *
2644 * When hashing the index filename will be in format number.number.
2645 * Hashing the groupname gets a number. See if that #.1 file exists;
2646 * if so, read first line. Is this the group we want? If no, try #.2.
2647 * Repeat until no such file or we find an existing file that matches
2648 * our group. Return pointer to path or NULL if not found.
2649 */
2650 static char *
2651 find_nov_file(
2652 struct t_group *group,
2653 int mode)
2654 {
2655 FILE *fp;
2656 const char *dir;
2657 char buf[PATH_LEN];
2658 int i;
2659 struct stat sb;
2660 unsigned long hash;
2661 static char nov_file[PATH_LEN];
2662 static t_bool once_only = FALSE; /* Trap things that are done only 1 time */
2663
2664 if (group == NULL || (mode != R_OK && mode != W_OK))
2665 return NULL;
2666
2667 switch (group->type) {
2668 case GROUP_TYPE_MAIL:
2669 dir = index_maildir;
2670 break;
2671
2672 case GROUP_TYPE_SAVE:
2673 dir = index_savedir;
2674 break;
2675
2676 case GROUP_TYPE_NEWS:
2677 /*
2678 * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2679 * about [X]OVER are handled in nntp_open()
2680 */
2681
2682 /*
2683 * When reading via NNTP, system wide overviews are irrelevant, of
2684 * course, and the private overview filename will be the same for
2685 * both reading and writing.
2686 *
2687 * When working locally, we only use a private cache for reading
2688 * if requested and when system wide overviews don't already exist.
2689 * When writing then only private overviews can be used since
2690 * updating system wide overviews is not safe wrt locking etc.
2691 *
2692 * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2693 */
2694 #ifndef NNTP_ONLY
2695 if (!read_news_via_nntp) {
2696 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2697 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2698 if (access(nov_file, R_OK) == 0) {
2699 if (mode == R_OK)
2700 return nov_file; /* Use system wide overviews */
2701 else
2702 return NULL; /* Don't write cache in this case */
2703 }
2704 }
2705 #endif /* !NNTP_ONLY */
2706
2707 /*
2708 * We only get here when private overviews are going to be used
2709 * Go no further if they are explicitly turned off
2710 */
2711 if (!tinrc.cache_overview_files)
2712 return NULL;
2713
2714 /*
2715 * Append -<nntpserver> to private cache dir
2716 */
2717 if (!once_only && nntp_server) {
2718 size_t sp, ln = strlen(index_newsdir);
2719
2720 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
2721 char *srv = my_strdup(nntp_server);
2722
2723 strcat(index_newsdir, "-");
2724 sp--;
2725 ln++;
2726 str_lwr(srv);
2727 my_strncpy(index_newsdir + ln, srv, sp);
2728 free(srv);
2729 }
2730 once_only = TRUE;
2731 }
2732
2733 /*
2734 * Only try to set up the private cache when writing. If it
2735 * doesn't exist yet, then ergo we can't read from it.
2736 * The cache will be checked/created on every write; a previous
2737 * bug report complained that this was not the case
2738 */
2739 if (stat(index_newsdir, &sb) == -1) { /* Private cache doesn't exist */
2740 if (mode == R_OK)
2741 return NULL;
2742 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2743 return NULL;
2744 } else {
2745 if (!S_ISDIR(sb.st_mode))
2746 return NULL;
2747 }
2748
2749 /*
2750 * Update the newsgroups cache to point to the new location
2751 * now that we know it is valid
2752 */
2753 if (!once_only)
2754 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
2755
2756 dir = index_newsdir;
2757 break;
2758
2759 default: /* not reached */
2760 return NULL;
2761 }
2762
2763 /*
2764 * We only get here if writing to a private overview.
2765 * These always have hashed filenames.
2766 * Try <hash>.<seqno> and check the group name tagline until
2767 * matching index file is found. If not found return next unused
2768 * filename
2769 */
2770 hash = hash_groupname(group->name);
2771
2772 for (i = 1; ; i++) {
2773 char *ptr;
2774
2775 snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2776 joinpath(nov_file, sizeof(nov_file), dir, buf);
2777
2778 if ((fp = fopen(nov_file, "r")) == NULL)
2779 break;
2780
2781 /*
2782 * No group name header, so not a valid index file => overwrite it
2783 */
2784 if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2785 fclose(fp);
2786 break;
2787 }
2788 fclose(fp);
2789
2790 if ((ptr = strrchr(buf, '\n')) != NULL)
2791 *ptr = '\0';
2792
2793 if (strcmp(buf, group->name) == 0)
2794 break;
2795 }
2796
2797 return nov_file;
2798 }
2799
2800
2801 /*
2802 * Run the index file updater only for the groups we've loaded.
2803 */
2804 void
2805 do_update(
2806 t_bool catchup)
2807 {
2808 int i, j, k = 0;
2809 time_t beg_epoch = 0;
2810 struct t_article *art;
2811 struct t_group *group;
2812
2813 if (verbose)
2814 (void) time(&beg_epoch);
2815
2816 /*
2817 * loop through groups and update any required index files
2818 */
2819 for (i = 0; i < selmenu.max; i++) {
2820 group = &active[my_group[i]];
2821 /*
2822 * FIXME: workaround to get a valid CURR_GROUP
2823 * it also points to the currently processed group so that
2824 * the correct attributes are used
2825 * The correct fix is to get rid of CURR_GROUP
2826 */
2827 selmenu.curr = i;
2828
2829 if (group->bogus || !group->subscribed)
2830 continue;
2831
2832 if (!index_group(group)) {
2833 for_each_art(j) {
2834 art = &arts[j];
2835 FreeAndNull(art->refs);
2836 FreeAndNull(art->msgid);
2837 }
2838 continue;
2839 }
2840
2841 k++;
2842
2843 if (verbose) {
2844 my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2845 my_flush();
2846 }
2847
2848 if (catchup) {
2849 for_each_art(j)
2850 art_mark(group, &arts[j], ART_READ);
2851 }
2852 }
2853
2854 if (verbose) {
2855 wait_message(0, _(txt_catchup_update_info),
2856 (catchup ? _(txt_caughtup) : _(txt_updated)), k,
2857 PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2858 }
2859 }
2860
2861
2862 static int
2863 artnum_comp(
2864 t_comptype p1,
2865 t_comptype p2)
2866 {
2867 const struct t_article *s1 = (const struct t_article *) p1;
2868 const struct t_article *s2 = (const struct t_article *) p2;
2869
2870 /*
2871 * s1->artnum less than s2->artnum
2872 */
2873 if (s1->artnum < s2->artnum)
2874 return -1;
2875
2876 /*
2877 * s1->artnum greater than s2->artnum
2878 */
2879 if (s1->artnum > s2->artnum)
2880 return 1;
2881
2882 return 0;
2883 }
2884
2885
2886 /*
2887 * return result of strcmp (reversed for descending)
2888 */
2889 static int
2890 subj_comp_asc(
2891 t_comptype p1,
2892 t_comptype p2)
2893 {
2894 int retval;
2895 const struct t_article *s1 = (const struct t_article *) p1;
2896 const struct t_article *s2 = (const struct t_article *) p2;
2897
2898 if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2899 return retval;
2900
2901 return s1->date - s2->date > 0 ? 1 : -1;
2902 }
2903
2904
2905 static int
2906 subj_comp_desc(
2907 t_comptype p1,
2908 t_comptype p2)
2909 {
2910 int retval;
2911 const struct t_article *s1 = (const struct t_article *) p1;
2912 const struct t_article *s2 = (const struct t_article *) p2;
2913
2914 if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2915 return retval;
2916
2917 return s1->date - s2->date > 0 ? 1 : -1;
2918 }
2919
2920
2921 /*
2922 * return result of strcmp (reversed for descending)
2923 */
2924 static int
2925 from_comp_asc(
2926 t_comptype p1,
2927 t_comptype p2)
2928 {
2929 int retval;
2930 const struct t_article *s1 = (const struct t_article *) p1;
2931 const struct t_article *s2 = (const struct t_article *) p2;
2932
2933 if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2934 return retval;
2935
2936 return s1->date - s2->date > 0 ? 1 : -1;
2937 }
2938
2939
2940 static int
2941 from_comp_desc(
2942 t_comptype p1,
2943 t_comptype p2)
2944 {
2945 int retval;
2946 const struct t_article *s1 = (const struct t_article *) p1;
2947 const struct t_article *s2 = (const struct t_article *) p2;
2948
2949 if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2950 return retval;
2951
2952 return s1->date - s2->date > 0 ? 1 : -1;
2953 }
2954
2955
2956 /*
2957 * Works like strcmp() for comparing time_t type values
2958 * Return codes:
2959 * -1: If p1 is before p2
2960 * 0: If they are the same time
2961 * 1: If p1 is after p2
2962 * If the sort order is _not_ DATE_ASCEND then the sense of the above
2963 * is reversed.
2964 */
2965 static int
2966 date_comp_asc(
2967 t_comptype p1,
2968 t_comptype p2)
2969 {
2970 const struct t_article *s1 = (const struct t_article *) p1;
2971 const struct t_article *s2 = (const struct t_article *) p2;
2972
2973 /*
2974 * s1->date less than s2->date
2975 */
2976 if (s1->date < s2->date)
2977 return -1;
2978
2979 /*
2980 * s1->date greater than s2->date
2981 */
2982 if (s1->date > s2->date)
2983 return 1;
2984
2985 return 0;
2986 }
2987
2988
2989 static int
2990 date_comp_desc(
2991 t_comptype p1,
2992 t_comptype p2)
2993 {
2994 const struct t_article *s1 = (const struct t_article *) p1;
2995 const struct t_article *s2 = (const struct t_article *) p2;
2996
2997 /*
2998 * s2->date less than s1->date
2999 */
3000 if (s2->date < s1->date)
3001 return -1;
3002
3003 /*
3004 * s2->date greater than s1->date
3005 */
3006 if (s2->date > s1->date)
3007 return 1;
3008
3009 return 0;
3010 }
3011
3012
3013 /*
3014 * Same again, but for art[].score
3015 */
3016 static int
3017 score_comp_asc(
3018 t_comptype p1,
3019 t_comptype p2)
3020 {
3021 const struct t_article *s1 = (const struct t_article *) p1;
3022 const struct t_article *s2 = (const struct t_article *) p2;
3023
3024 if (s1->score < s2->score)
3025 return -1;
3026
3027 if (s1->score > s2->score)
3028 return 1;
3029
3030 return s1->date - s2->date > 0 ? 1 : -1;
3031 }
3032
3033
3034 static int
3035 score_comp_desc(
3036 t_comptype p1,
3037 t_comptype p2)
3038 {
3039 const struct t_article *s1 = (const struct t_article *) p1;
3040 const struct t_article *s2 = (const struct t_article *) p2;
3041
3042 if (s2->score < s1->score)
3043 return -1;
3044
3045 if (s2->score > s1->score)
3046 return 1;
3047
3048 return s1->date - s2->date > 0 ? 1 : -1;
3049 }
3050
3051
3052 /*
3053 * Same again, but for art[].line_count
3054 */
3055 static int
3056 lines_comp_asc(
3057 t_comptype p1,
3058 t_comptype p2)
3059 {
3060 const struct t_article *s1 = (const struct t_article *) p1;
3061 const struct t_article *s2 = (const struct t_article *) p2;
3062
3063 if (s1->line_count < s2->line_count)
3064 return -1;
3065
3066 if (s1->line_count > s2->line_count)
3067 return 1;
3068
3069 return s1->date - s2->date > 0 ? 1 : -1;
3070 }
3071
3072
3073 static int
3074 lines_comp_desc(
3075 t_comptype p1,
3076 t_comptype p2)
3077 {
3078 const struct t_article *s1 = (const struct t_article *) p1;
3079 const struct t_article *s2 = (const struct t_article *) p2;
3080
3081 if (s2->line_count < s1->line_count)
3082 return -1;
3083
3084 if (s2->line_count > s1->line_count)
3085 return 1;
3086
3087 return s1->date - s2->date > 0 ? 1 : -1;
3088 }
3089
3090
3091 /*
3092 * Compares the total score of two threads. Used for sorting base[].
3093 */
3094 static int
3095 score_comp_base(
3096 t_comptype p1,
3097 t_comptype p2)
3098 {
3099 int a = get_score_of_thread(*(const long *) p1);
3100 int b = get_score_of_thread(*(const long *) p2);
3101
3102 /* If scores are equal, compare using the article sort order.
3103 * This determines the order in a group of equally scored threads.
3104 */
3105 if (a == b) {
3106 const struct t_article *s1 = &arts[*(const long *) p1];
3107 const struct t_article *s2 = &arts[*(const long *) p2];
3108 t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3109
3110 if (comp_func)
3111 return (*comp_func)(s1, s2);
3112 return 0;
3113 }
3114
3115 if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3116 return a > b ? 1 : -1;
3117 return a < b ? 1 : -1;
3118 }
3119
3120
3121 /*
3122 * Compare the date of the last posted article of two threads.
3123 * Used for sorting base[].
3124 */
3125 static int
3126 last_date_comp_base_desc(
3127 t_comptype p1,
3128 t_comptype p2)
3129 {
3130 time_t s1_last = get_last_posting_date(*(const long *) p1);
3131 time_t s2_last = get_last_posting_date(*(const long *) p2);
3132
3133 if (s2_last < s1_last)
3134 return -1;
3135
3136 if (s2_last > s1_last)
3137 return 1;
3138
3139 return 0;
3140 }
3141
3142
3143 static int
3144 last_date_comp_base_asc(
3145 t_comptype p1,
3146 t_comptype p2)
3147 {
3148 time_t s1_last = get_last_posting_date(*(const long *) p1);
3149 time_t s2_last = get_last_posting_date(*(const long *) p2);
3150
3151 if (s2_last > s1_last)
3152 return -1;
3153
3154 if (s2_last < s1_last)
3155 return 1;
3156
3157 return 0;
3158 }
3159
3160
3161 static time_t
3162 get_last_posting_date(
3163 long n)
3164 {
3165 long i;
3166 time_t last = (time_t) 0;
3167
3168 for (i = n; i >= 0; i = arts[i].thread) {
3169 if (arts[i].date > last)
3170 last = arts[i].date;
3171 }
3172
3173 return last;
3174 }
3175
3176
3177 void
3178 set_article(
3179 struct t_article *art)
3180 {
3181 art->subject = NULL;
3182 art->from = NULL;
3183 art->name = NULL;
3184 art->date = (time_t) 0;
3185 art->xref = NULL;
3186 art->msgid = NULL;
3187 art->refs = NULL;
3188 art->refptr = NULL;
3189 art->line_count = -1;
3190 art->archive = NULL;
3191 art->tagged = 0;
3192 art->thread = ART_EXPIRED;
3193 art->prev = ART_NORMAL;
3194 art->score = 0;
3195 art->status = ART_UNREAD;
3196 art->killed = ART_NOTKILLED;
3197 art->zombie = FALSE;
3198 art->delete_it = FALSE;
3199 art->selected = FALSE;
3200 art->inrange = FALSE;
3201 art->matched = FALSE;
3202 art->keep_in_base = FALSE;
3203 art->multipart_subj = FALSE;
3204 }
3205
3206
3207 /*
3208 * Do a binary chop to see if 'art' (an article number) exists in arts[]
3209 * Naturally arts[] must be sorted on artnum
3210 * Return index into arts[] or -1
3211 */
3212 static int
3213 valid_artnum(
3214 t_artnum art)
3215 {
3216 int prev, range;
3217 int dctop = top_art;
3218 int cur = 1;
3219
3220 while ((dctop >>= 1))
3221 cur <<= 1;
3222
3223 range = cur >> 1;
3224 cur--;
3225
3226 forever {
3227 if (arts[cur].artnum == art)
3228 return cur;
3229
3230 prev = cur;
3231 cur += ((arts[cur].artnum < art) ? range : -range);
3232 if (prev == cur)
3233 break;
3234
3235 if (cur >= top_art)
3236 cur = top_art - 1;
3237
3238 range >>= 1;
3239 }
3240 return -1;
3241 }
3242
3243
3244 /*
3245 * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3246 * Needed if arts[] is not sorted on artnum
3247 * Return index into arts[] or -1
3248 */
3249 int
3250 find_artnum(
3251 t_artnum art)
3252 {
3253 int i;
3254
3255 for_each_art(i) {
3256 if (arts[i].artnum == art)
3257 return i;
3258 }
3259 return -1;
3260 }
3261
3262
3263 /*----------------------------- Overview handling -----------------------*/
3264 /* TODO: use
3265 * setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3266 * my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3267 * instead?
3268 */
3269 static char *
3270 print_date(
3271 time_t secs)
3272 {
3273 static char date[25];
3274 struct tm *tm;
3275 static const char *const months_a[] = {
3276 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3277 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3278 };
3279
3280 if ((tm = gmtime(&secs)) != NULL)
3281 snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3282 tm->tm_mday,
3283 months_a[tm->tm_mon],
3284 tm->tm_year + 1900,
3285 tm->tm_hour, tm->tm_min, tm->tm_sec);
3286 else
3287 snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3288
3289 return date;
3290 }
3291
3292
3293 static char *
3294 print_from(
3295 struct t_group *group,
3296 struct t_article *article,
3297 int charset)
3298 {
3299 char *p, *q;
3300 static char from[PATH_LEN];
3301
3302 *from = '\0';
3303
3304 if (article->name != NULL) {
3305 q = my_strdup(article->name);
3306 #ifdef CHARSET_CONVERSION
3307 if (charset != -1)
3308 buffer_to_network(q, charset);
3309 #endif /* CHARSET_CONVERSION */
3310 p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
3311 unfold_header(p);
3312 if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3313 snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3314 else
3315 snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3316
3317 free(p);
3318 free(q);
3319 } else
3320 snprintf(from, sizeof(from), "<%s>", article->from);
3321
3322 return from;
3323 }
3324
3325
3326 /*
3327 * Open a group news overview file
3328 * Use NNTP XOVER where possible unless 'local' is set
3329 */
3330 static FILE *
3331 open_xover_fp(
3332 struct t_group *group,
3333 const char *mode,
3334 t_artnum min,
3335 t_artnum max,
3336 t_bool local)
3337 {
3338 #ifdef NNTP_ABLE
3339 if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3340 char line[NNTP_STRLEN];
3341
3342 if (!max)
3343 return NULL;
3344 if (min == max)
3345 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3346 else
3347 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3348 return (nntp_command(line, OK_XOVER, NULL, 0));
3349 }
3350 #endif /* NNTP_ABLE */
3351 {
3352 FILE *fp;
3353 char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3354
3355 if (nov_file != NULL) {
3356 if ((fp = fopen(nov_file, mode)) != NULL)
3357 return fp;
3358
3359 if (*mode != 'r')
3360 error_message(2, _(txt_cannot_open), nov_file);
3361 }
3362 }
3363 return NULL;
3364 }
3365
3366
3367 #ifdef USE_HEAPSORT
3368 int
3369 tin_sort(
3370 void *sbase,
3371 size_t nel,
3372 size_t width,
3373 t_compfunc compar)
3374 {
3375 int rc;
3376
3377 switch (tinrc.sort_function) {
3378 case 0:
3379 qsort(sbase, nel, width, compar);
3380 rc = 0;
3381 break;
3382
3383 case 1:
3384 rc = heapsort(sbase, nel, width, compar);
3385 break;
3386
3387 default:
3388 rc = -1;
3389 break;
3390 }
3391 return rc;
3392 }
3393 #endif /* USE_HEAPSORT */