"Fossies" - the Fresh Open Source Software Archive 
Member "tin-2.4.3/src/art.c" (18 Dec 2018, 89022 Bytes) of package /linux/misc/tin-2.4.3.tar.xz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "art.c" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
2.4.2_vs_2.4.3.
1 /*
2 * Project : tin - a Usenet reader
3 * Module : art.c
4 * Author : I.Lea & R.Skrenta
5 * Created : 1991-04-01
6 * Updated : 2018-12-18
7 * Notes :
8 *
9 * Copyright (c) 1991-2019 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10 * All rights reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. The name of the author may not be used to endorse or promote
21 * products derived from this software without specific prior written
22 * permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
25 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
26 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
28 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
30 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
33 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
34 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 */
36
37
38 #ifndef TIN_H
39 # include "tin.h"
40 #endif /* !TIN_H */
41 #ifndef TCURSES_H
42 # include "tcurses.h"
43 #endif /* !TCURSES_H */
44 #ifndef NEWSRC_H
45 # include "newsrc.h"
46 #endif /* !NEWSRC_H */
47
48 #ifndef STPWATCH_H
49 # include "stpwatch.h"
50 #endif /* !STPWATCH_H */
51
52 /*
53 * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
54 */
55 #define SortBy(func) tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
56
57 int top_art = 0; /* # of articles in arts[] */
58
59 /*
60 * Local prototypes
61 */
62 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
63 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
64 static char *find_nov_file(struct t_group *group, int mode);
65 static char *print_date(time_t secs);
66 static char *print_from(struct t_group *group, struct t_article *article, int charset);
67 static int artnum_comp(t_comptype p1, t_comptype p2);
68 static int base_comp(t_comptype p1, t_comptype p2);
69 static int date_comp_asc(t_comptype p1, t_comptype p2);
70 static int date_comp_desc(t_comptype p1, t_comptype p2);
71 static int from_comp_asc(t_comptype p1, t_comptype p2);
72 static int from_comp_desc(t_comptype p1, t_comptype p2);
73 static int global_get_multiparts(int aindex, MultiPartInfo **malloc_and_setme_info);
74 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
75 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
76 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
77 static int lines_comp_asc(t_comptype p1, t_comptype p2);
78 static int lines_comp_desc(t_comptype p1, t_comptype p2);
79 static int read_art_headers(struct t_group *group, int total, t_artnum top);
80 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
81 static int score_comp_asc(t_comptype p1, t_comptype p2);
82 static int score_comp_desc(t_comptype p1, t_comptype p2);
83 static int score_comp_base(t_comptype p1, t_comptype p2);
84 static int subj_comp_asc(t_comptype p1, t_comptype p2);
85 static int subj_comp_desc(t_comptype p1, t_comptype p2);
86 static int valid_artnum(t_artnum art);
87 static t_artnum find_first_unread(struct t_group *group);
88 static t_artnum setup_hard_base(struct t_group *group);
89 static t_bool parse_headers(FILE *fp, struct t_article *h);
90 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
91 static time_t get_last_posting_date(long n);
92 static void sort_base(unsigned int sort_threads_type);
93 static void thread_by_multipart(void);
94 static void thread_by_percentage(struct t_group *group);
95 static void thread_by_subject(void);
96 static void write_overview(struct t_group *group);
97 #ifdef NNTP_ABLE
98 static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
99 static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
100 #endif /* NNTP_ABLE */
101
102
103 /*
104 * Display a suitable 'entering group' message if screen needs redrawing
105 * Allow for the non-printing %s, and the %-age counter
106 */
107 void
108 show_art_msg(
109 const char *group)
110 {
111 wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
112 }
113
114
115 /*
116 * Construct the pointers to the first (base) article in each thread.
117 * If we are showing only unread, then point to the first unread. I have
118 * no idea why this should be so, it causes problems elsewhere [which_response]
119 */
120 void
121 find_base(
122 struct t_group *group)
123 {
124 int i, j;
125
126 grpmenu.max = 0;
127
128 #ifdef DEBUG
129 debug_print_arts();
130 #endif /* DEBUG */
131
132 for_each_art(i) {
133 /*
134 * .prev will be set on each article that is after the first article in
135 * the thread. Invalid articles which have been expired will have
136 * .thread set to ART_EXPIRED
137 */
138 if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
139 continue;
140
141 if (grpmenu.max >= max_base)
142 expand_base();
143
144 if (group->attribute->show_only_unread_arts) {
145 if (arts[i].status != ART_READ || arts[i].keep_in_base)
146 base[grpmenu.max++] = i;
147 else {
148 /* Find 1st unread art in thread */
149 for (j = i; j >= 0; j = arts[j].thread) {
150 if (arts[j].status != ART_READ || arts[j].keep_in_base) {
151 base[grpmenu.max++] = i;
152 break;
153 }
154 }
155 }
156 } else
157 base[grpmenu.max++] = i;
158 }
159 /* sort base[] */
160 if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
161 sort_base(group->attribute->sort_threads_type);
162 }
163
164
165 /*
166 * Longword comparison routine for the tin_sort()
167 */
168 static int
169 base_comp(
170 t_comptype p1,
171 t_comptype p2)
172 {
173 const t_artnum *a = (const t_artnum *) p1;
174 const t_artnum *b = (const t_artnum *) p2;
175
176 if (*a < *b)
177 return -1;
178
179 if (*a > *b)
180 return 1;
181
182 return 0;
183 }
184
185
186 /*
187 * via NNTP:
188 * Issue a LISTGROUP command
189 * Read the article numbers existing in the group into base[]
190 * If the LISTGROUP failed, issue a GROUP command. Use the results to
191 * create a less accurate version of base[]
192 * This data will already be sorted
193 *
194 * on local spool:
195 * Read the spool dir to populate base[] as above. Sort it.
196 *
197 * Grow the arts[] and bitmaps as needed.
198 * NB: the output will be sorted on artnum
199 *
200 * grpmenu.max is one past top.
201 * Returns total number of articles in group, or -1 on error
202 */
203 static t_artnum
204 setup_hard_base(
205 struct t_group *group)
206 {
207 t_artnum total = 0;
208
209 grpmenu.max = 0;
210
211 /*
212 * If reading with NNTP, issue a LISTGROUP
213 */
214 if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
215 #ifdef NNTP_ABLE
216 char buf[NNTP_STRLEN];
217 char line[NNTP_STRLEN];
218 int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
219 FILE *fp;
220 t_artnum last, start, count = 0, j = 0;
221 static t_bool skip_listgroup = FALSE;
222
223 /*
224 * Some nntp servers are broken and need an extra GROUP command
225 * (reported by reorx@irc.pl). This affects (old?) versions of
226 * nntpcache, leafnode and SurgeNews. Usually this should not be
227 * needed.
228 *
229 * For getart_limit recheck lowwatermark as at least giganews gives
230 * very different results for LIST ACTIVE (3 year retention for all)
231 * and GROUP (based on the clients contract).
232 * Calculate range and prepare base[] not to lose unread arts.
233 */
234 if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
235 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
236 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
237 return -1;
238
239 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
240 return -1;
241
242 if (getart_limit > 0) {
243 j = group->xmax - getart_limit;
244 count = MAX(find_first_unread(group), start);
245 }
246 if (getart_limit < 0) {
247 j = getart_limit + find_first_unread(group);
248 count = group->xmin;
249 }
250 if (j < group->xmin)
251 j = group->xmin;
252
253 for (; count < j; count++) {
254 if (grpmenu.max >= max_base)
255 expand_base();
256 base[grpmenu.max++] = count;
257 }
258 }
259
260 /*
261 * See if LISTGROUP works
262 */
263 if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
264 if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
265 /* RFC 3977 allows ranges in LISTGROUP */
266 if (getart_limit > 0)
267 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
268 else /* getart_limit < 0; fetch till newest art */
269 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
270
271 } else /* for RFC 977 just use GROUP */
272 skip_listgroup = TRUE;
273
274 } else /* get all article numbers */
275 snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
276
277 if (!skip_listgroup) {
278 if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
279 char *ptr;
280
281 # ifdef DEBUG
282 if ((debug & DEBUG_NNTP) && verbose > 1)
283 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
284 # endif /* DEBUG */
285
286 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
287 if (grpmenu.max >= max_base)
288 expand_base();
289 base[grpmenu.max++] = atoartnum(ptr);
290 total++;
291 }
292
293 if (tin_errno)
294 return -1;
295 } else
296 skip_listgroup = TRUE;
297 }
298
299 if (skip_listgroup) { /* LISTGROUP was skipped or failed */
300 /*
301 * Handle the obscure case that the user aborted before the LISTGROUP
302 * had a chance to respond
303 */
304 if (tin_errno)
305 return -1;
306
307 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
308 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
309 return -1;
310
311 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
312 return -1;
313
314 # ifdef DEBUG
315 if ((debug & DEBUG_NNTP) && verbose > 1)
316 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
317 # endif /* DEBUG */
318 total = count;
319 grpmenu.max = 0;
320 if (getart_limit > 0) {
321 if ((j = find_first_unread(group)) > start) {
322 if (group->xmax > getart_limit) {
323 start = MIN(j, group->xmax - getart_limit);
324 total = getart_limit;
325 } else
326 start = j;
327 }
328 }
329 if (getart_limit < 0) {
330 if ((j = (getart_limit + find_first_unread(group))) > start)
331 start = j;
332 }
333 while (start <= last) {
334 if (grpmenu.max >= max_base)
335 expand_base();
336 base[grpmenu.max++] = start++;
337 }
338 }
339 #endif /* NNTP_ABLE */
340 /*
341 * Reading off local spool, read the directory files
342 */
343 } else {
344 DIR *d;
345 DIR_BUF *e;
346 char group_path[PATH_LEN];
347 t_artnum art;
348
349 make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
350
351 if ((d = opendir(group_path)) != NULL) {
352 while ((e = readdir(d)) != NULL) {
353 art = atoartnum(e->d_name);
354 if (art >= 1) {
355 total++;
356 if (grpmenu.max >= max_base)
357 expand_base();
358 base[grpmenu.max++] = art;
359 }
360 }
361 CLOSEDIR(d);
362 tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
363 } else {
364 perror_message(_(txt_cannot_open), group_path);
365 #if 0
366 if (access(group_path, R_OK) != 0)
367 error_message(2, _(txt_not_exist));
368 #endif /* 0 */
369 return -1;
370 }
371 }
372
373 if (grpmenu.max) {
374 if (base[grpmenu.max - 1] > group->xmax)
375 group->xmax = base[grpmenu.max - 1];
376 expand_bitmap(group, base[0]);
377 }
378
379 return total;
380 }
381
382
383 /*
384 * Main group indexing routine.
385 *
386 * Will read any existing index, create or incrementally update
387 * the index by looking at the articles in the spool directory,
388 * and attempt to write a new index if necessary.
389 *
390 * Returns FALSE if the user aborted the indexing, otherwise TRUE
391 */
392 t_bool
393 index_group(
394 struct t_group *group)
395 {
396 int i;
397 int changed; /* Count of articles whose overview has changed */
398 int getart_limit;
399 int respnum;
400 int total;
401 t_artnum last_read_article;
402 t_artnum min, new_min, max;
403 t_bool caching_xover;
404 t_bool filtered;
405 t_bool path_in_nov = FALSE;
406 t_bool rebuild_cache = FALSE;
407
408 if (group == NULL)
409 return TRUE;
410
411 if (!batch_mode)
412 show_art_msg(group->name);
413
414 signal_context = cArt; /* Set this only once curr_group is valid */
415
416 hash_reclaim();
417 free_art_array();
418 free_msgids();
419
420 BegStopWatch("setup_hard_base()");
421
422 /*
423 * Get list of valid article numbers
424 */
425 if (setup_hard_base(group) < 0)
426 return FALSE;
427
428 EndStopWatch();
429 PrintStopWatch();
430
431 #ifdef DEBUG
432 if (debug & DEBUG_NEWSRC) {
433 debug_print_comment("Before read_overview");
434 debug_print_bitmap(group, NULL);
435 }
436 #endif /* DEBUG */
437
438 min = grpmenu.max ? base[0] : group->xmin;
439 max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
440
441 getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
442
443 if (getart_limit > 0) {
444 if (grpmenu.max && (grpmenu.max > getart_limit))
445 min = base[grpmenu.max - getart_limit];
446 else
447 getart_limit = 0;
448 } else if (getart_limit < 0) {
449 t_artnum first_unread = find_first_unread(group);
450
451 if (min - first_unread < getart_limit)
452 min = first_unread + getart_limit;
453 else
454 getart_limit = 0;
455 }
456
457 /*
458 * Quit now if no articles
459 */
460 if (max < 0)
461 return FALSE;
462
463 top_art = 0;
464 last_read_article = T_ARTNUM_CONST(0);
465
466 /*
467 * Read in the existing overview data for min..max
468 * This read has local=TRUE set if locally caching XOVER records to ensure
469 * we pull in any private overview caches in preference to using OVER
470 *
471 * When reading local spool, this will pull in the system wide overview
472 * cache (if found) otherwise the private overview cache will be read
473 */
474 caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
475 if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
476 return FALSE; /* user aborted indexing */
477
478 /*
479 * Fill in the range last_read_article...max using XOVER
480 * Only do this if the previous read_overview() was against private cache
481 */
482 if ((last_read_article < max) && caching_xover) {
483 new_min = (last_read_article >= min) ? last_read_article + 1 : min;
484
485 if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
486 return FALSE; /* user aborted indexing */
487 else
488 changed += i;
489 } else
490 caching_xover = FALSE;
491
492 /*
493 * Mark as UNTHREADED all articles that have been verified as valid
494 * Get num of new arts to index so the user will have an idea of index time
495 */
496 for (i = 0, total = 0; i < grpmenu.max; i++) {
497 if ((respnum = valid_artnum(base[i])) >= 0) {
498 arts[respnum].thread = ART_UNTHREADED;
499 continue;
500 }
501 if (base[i] <= last_read_article) /* It is vital this test be done last */
502 continue;
503 total++;
504 }
505
506 /*
507 * Add any articles to arts[] that are new or were killed
508 */
509 if (total > 0) {
510 new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
511
512 if ((i = read_art_headers(group, total, new_min)) == -1)
513 return FALSE; /* user aborted indexing */
514 else
515 changed += i;
516 }
517
518 #ifdef DEBUG
519 if (debug & DEBUG_NEWSRC) {
520 debug_print_comment("Before parse_unread_arts()");
521 debug_print_bitmap(group, NULL);
522 }
523 #endif /* DEBUG */
524 /*
525 * Do this before calling art_mark(,, ART_READ) if you want
526 * the unread count to be correct.
527 */
528 min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
529 parse_unread_arts(group, min);
530 #ifdef DEBUG
531 if (debug & DEBUG_NEWSRC) {
532 debug_print_comment("After parse_unread_arts()");
533 debug_print_bitmap(group, NULL);
534 }
535 #endif /* DEBUG */
536
537 /*
538 * Stat all articles to see if any have expired
539 */
540 for_each_art(i) {
541 if (arts[i].thread == ART_EXPIRED) {
542 changed++;
543 #ifdef DEBUG
544 if (debug & DEBUG_NEWSRC)
545 debug_print_comment("art.c: index_group() purging...");
546 #endif /* DEBUG */
547 art_mark(group, &arts[i], ART_READ);
548 if (group->attribute->show_only_unread_arts)
549 arts[i].keep_in_base = FALSE;
550 }
551 if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
552 path_in_nov = TRUE;
553 }
554
555 /*
556 * Only rewrite the index if it has changed
557 * TODO review the exact logic behind "|| caching_xover"
558 */
559 if (changed || caching_xover || rebuild_cache)
560 write_overview(group);
561
562 /*
563 * Create the reference tree. The msgid and ref ptrs will
564 * be free()d now that the NovFile has been written.
565 */
566 build_references(group);
567
568 /*
569 * Needs access to the reference tree
570 */
571 filtered = filter_articles(group);
572
573 BegStopWatch("make_threads()");
574
575 /*
576 * Thread the group
577 */
578 make_threads(group, FALSE);
579
580 EndStopWatch();
581 PrintStopWatch();
582
583 if ((changed > 0 || filtered) && !batch_mode)
584 clear_message();
585
586 return TRUE;
587 }
588
589
590 /*
591 * Returns number of first unread article
592 */
593 static t_artnum
594 find_first_unread(
595 struct t_group *group)
596 {
597 unsigned char *p;
598 unsigned char *end = group->newsrc.xbitmap;
599 t_artnum first = group->newsrc.xmin; /* initial value */
600
601 if ((p = group->newsrc.xbitmap)) {
602 end += group->newsrc.xbitlen / NBITS;
603 for (; *p == '\0' && p < end; p++, first += NBITS)
604 ;
605 }
606 return first;
607 }
608
609
610 /*
611 * Open an article for reading just the header
612 * 'next' is used/updated with the next article number
613 * to optimise the number of 'HEAD' commands issued on
614 * groups with holes.
615 */
616 static FILE *
617 open_art_header(
618 char *groupname,
619 t_artnum art,
620 t_artnum *next)
621 {
622 char buf[NNTP_STRLEN];
623 #ifdef NNTP_ABLE
624 FILE *fp;
625 int i;
626
627 if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
628 /*
629 * Don't bother requesting if we have not got there yet.
630 * This is a big win if the group has got holes in it (ie. if 000's
631 * of articles have expired between active files min & max values).
632 */
633 if (art < *next)
634 return NULL;
635
636 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
637 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
638 return fp;
639
640 /*
641 * TODO:
642 * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
643 * "503 Access denied" instead of 480 but NEXT still works,
644 * so tin loops over all articles without getting useful data
645 */
646
647 /*
648 * HEAD failed, try to find NEXT
649 * Should return "223 artno message-id more text...."
650 */
651 i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
652 switch (i) {
653 case OK_NOTEXT:
654 *next = atoartnum(buf); /* Set next art number */
655 break;
656
657 # ifndef BROKEN_LISTGROUP
658 /*
659 * might happen if LISTGROUP doesn't select group, but
660 * we are not -DBROKEN_LISTGROUP
661 */
662 case ERR_NCING:
663 nntp_caps.broken_listgroup = TRUE;
664 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
665 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
666 return NULL;
667 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
668 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
669 return fp;
670 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
671 *next = atoartnum(buf);
672 break;
673 # endif /* !BROKEN_LISTGROUP */
674
675 default:
676 /*
677 * TODO: abort loop over all arts on ERR_NONEXT
678 */
679 # ifndef BROKEN_LISTGROUP
680 /*
681 * to avoid out of sync responses
682 * (listgroup seems to work, but didn't select new group,
683 * so xover seems to work but returns old data)
684 * we set listgroup_broken = TRUE; once we saw a
685 * ERR_NOARTIG / ERR_NONEXT or the like - even if
686 * ERR_NOARTIG may occur on servers where listgroup
687 * isn't broken...
688 */
689 nntp_caps.broken_listgroup = TRUE;
690 # endif /* !BROKEN_LISTGROUP */
691 break;
692 }
693
694 return NULL;
695 }
696 #endif /* NNTP_ABLE */
697
698 snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
699 return (fopen(buf, "r"));
700 }
701
702
703 /*
704 * Called after XOVER/local/private overview databases have been loaded
705 * Read and parse in headers for any arts not already found (usually
706 * new articles that have not been indexed yet)
707 * Any new articles that are added have ->thread set to ART_UNTHREADED
708 * 'top' is the current highest artnum read
709 *
710 * Return values are:
711 * >0 Number of additional articles read in
712 * 0 No additional (new) articles were found
713 * -1 user aborted during read
714 */
715 static int
716 read_art_headers(
717 struct t_group *group,
718 int total,
719 t_artnum top)
720 {
721 FILE *fp;
722 char dir[PATH_LEN];
723 char group_msg[LEN];
724 int i;
725 int modified = 0;
726 t_artnum art;
727 t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
728 t_bool res;
729
730 /*
731 * Change to groups spooldir to optimize fopen()'s on local articles
732 * NB open_art_header() requires this
733 */
734 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
735 char buf[PATH_LEN];
736
737 get_cwd(dir);
738 make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
739 if (chdir(buf) != 0) {
740 #ifdef DEBUG
741 if (debug & DEBUG_MISC)
742 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
743 #endif /* DEBUG */
744 return -1;
745 }
746 }
747
748 snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
749
750 for (i = 0; i < grpmenu.max; i++) { /* for each article number */
751 art = base[i];
752
753 /*
754 * Skip articles that are below the low water mark or are
755 * already present
756 */
757 if (valid_artnum(art) >= 0)
758 continue;
759 if (art <= top)
760 continue;
761
762 /*
763 * Try and open the article
764 */
765 if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
766 continue;
767
768 /*
769 * Add article to arts[]
770 */
771 if (top_art >= max_art)
772 expand_art();
773
774 set_article(&arts[top_art]);
775 arts[top_art].artnum = art;
776 arts[top_art].thread = ART_UNTHREADED;
777
778 res = parse_headers(fp, &arts[top_art]);
779
780 TIN_FCLOSE(fp);
781 if (tin_errno) {
782 modified = -1;
783 break;
784 }
785
786 if (!res) {
787 #ifdef DEBUG
788 if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
789 char buf[PATH_LEN];
790
791 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
792 debug_print_file("ARTS", "read_art_headers() %s", buf);
793 }
794 #endif /* DEBUG */
795 arts[top_art].artnum = T_ARTNUM_CONST(0);
796 arts[top_art].date = (time_t) 0;
797 FreeAndNull(arts[top_art].xref);
798 FreeAndNull(arts[top_art].path);
799 FreeAndNull(arts[top_art].refs);
800 FreeAndNull(arts[top_art].msgid);
801 if (arts[top_art].archive) {
802 FreeAndNull(arts[top_art].archive->partnum);
803 FreeAndNull(arts[top_art].archive);
804 }
805 arts[top_art].tagged = 0;
806 arts[top_art].thread = ART_EXPIRED;
807 arts[top_art].prev = ART_NORMAL;
808 arts[top_art].status = ART_UNREAD;
809 arts[top_art].killed = ART_NOTKILLED;
810 arts[top_art].selected = FALSE;
811 continue;
812 }
813
814 top = arts[top_art].artnum; /* used if arts are killed */
815 top_art++;
816
817 if (++modified % MODULO_COUNT_NUM == 0)
818 show_progress(group_msg, modified, total);
819 }
820
821 /*
822 * Change back to previous dir before indexing started
823 */
824 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
825 chdir(dir);
826
827 return modified;
828 }
829
830
831 /*
832 * The algorithm is elegant, using the fact that identical Subject lines
833 * are hashed to the same node in table[] (see hashstr.c)
834 *
835 * Mark i as being in j's thread list if
836 * . The article is _not_ being ignored
837 * . The article is not already threaded
838 * . One of the following is true:
839 * 1) The subject lines are the same
840 * 2) Both are part of the same archive (name's match and arch bit set)
841 * IMHO the tests for archive name are redundant and have been for years
842 */
843 static void
844 thread_by_subject(
845 void)
846 {
847 int i, j;
848 struct t_hashnode *h;
849
850 for_each_art(i) {
851 if (IGNORE_ART_THREAD(i))
852 continue;
853
854 /*
855 * Get the contents of the magic marker in the hashnode
856 */
857 h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
858
859 j = h->aptr;
860
861 if (j != -1 && j < i) {
862 #if 1
863 if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
864 #else
865 /* see also refs.c:collate_subjects() */
866 if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
867 #endif /* 1 */
868 {
869 arts[j].thread = i;
870 arts[i].prev = j;
871 }
872 }
873
874 /*
875 * Update the magic marker with the highest numbered mesg in
876 * arts[] that has been used in this thread so far
877 */
878 h->aptr = i;
879 }
880
881 #if 0
882 fprintf(stderr, "Subj dump\n");
883 fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
884 for_each_art(i) {
885 fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
886 (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
887 (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
888 (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
889 arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
890 }
891 #endif /* 0 */
892 }
893
894
895 /*
896 * This Threading algorithm threads articles into 'buckets' where each bucket
897 * contains all the articles which match the root article's subject line to
898 * the configured percentage. Eg, if the root article had the subject "asdf"
899 * and the match percentage was configured to be 75% then any article would
900 * match if its subject was no different in more than a single character.
901 */
902 static void
903 thread_by_percentage(
904 struct t_group *group)
905 {
906 int i, j, k;
907 int root_num = 0; /* The index number of the root we are currently working on. */
908 unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
909 unsigned int percentage = 100 - group->attribute->thread_perc;
910 size_t slen;
911
912 /* First we need to sort art[] to simplify and speed up the matching. */
913 SortBy(subj_comp_asc);
914
915 /*
916 * Now we put all the articles which match enough into the thread. If
917 * an article doesn't match enough we create a new thread and then add
918 * to that and so on.
919 */
920 base[0] = 0;
921 arts[0].prev = ART_NORMAL;
922 for_each_art(i) {
923 if (i == 0)
924 continue;
925
926 /* Check each character to see if it matched enough */
927 k = 0;
928 unmatched = 0;
929 for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
930 if (arts[base[root_num]].subject[j] != arts[i].subject[k])
931 unmatched++;
932 }
933
934 /*
935 * By getting here we have a number of unmatched characters
936 * between the two strings. We also have the length of the
937 * strings available to us easily.
938 * All we need to do is see if the match is good enough, but
939 * we count differences in the length of the strings against
940 * them matching.
941 */
942 if (!(slen = strlen(arts[base[root_num]].subject)))
943 slen++;
944 unmatched += slen - strlen(arts[i].subject);
945 if (unmatched * 100 / slen > percentage) {
946 /*
947 * If there is less greater than percentage% different start a
948 * new thread.
949 */
950 base[++root_num] = i;
951 arts[i].prev = ART_NORMAL;
952 continue;
953 } else {
954 /*
955 * The subject lines match enough to consider them part of a single
956 * thread, so add the current article to the thread.
957 */
958 if (arts[base[root_num]].thread < 0)
959 arts[base[root_num]].thread = i;
960 arts[i].prev = i - 1;
961 arts[i - 1].thread = i;
962 continue;
963 }
964 }
965 }
966
967
968 /*
969 * This was brought over from tags.c, however this version doesn't
970 * operate on base_index
971 *
972 * Parses a subject header of the type "multipart message subject (01/42)"
973 * into a MultiPartInfo struct, or fails if the message subject isn't in the
974 * right form.
975 *
976 * @return nonzero on success
977 */
978 int
979 global_get_multipart_info(
980 int aindex,
981 MultiPartInfo *setme)
982 {
983 int i, j, offi, offj;
984 MultiPartInfo setmei, setmej;
985
986 i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
987 j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
988
989 /* Ok i hits first */
990 if (offi > offj) {
991 *setme = setmei;
992 return i;
993 }
994
995 /* Its j or they are both the same (which must be zero!) so we don't care */
996 *setme = setmej;
997 return j;
998 }
999
1000
1001 /*
1002 * Again this was taken from tags.c, but works on global indices into arts
1003 * rather than on base_index.
1004 */
1005 static int
1006 global_look_for_multipart_info(
1007 int aindex,
1008 MultiPartInfo* setme,
1009 char start,
1010 char stop,
1011 int *offset)
1012 {
1013 char *subj;
1014 char *pch;
1015 MultiPartInfo tmp;
1016
1017 *offset = 0;
1018
1019 /* entry assertions */
1020 assert(0 <= aindex && aindex < top_art && "invalid index");
1021 assert(setme != NULL && "setme must not be NULL");
1022
1023 /* parse the message */
1024 subj = arts[aindex].subject;
1025 pch = strrchr(subj, start);
1026 if (!pch || !isdigit((int) pch[1]))
1027 return 0;
1028
1029 tmp.base_index = aindex; /* This could be confusing because we are actually storing the index into arts, not the base_index. */
1030 tmp.subject_compare_len = pch - subj;
1031 tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1032 if (*pch != '/' && *pch != '|')
1033 return 0;
1034
1035 if (!isdigit((int) pch[1]))
1036 return 0;
1037
1038 tmp.total = (int) strtol(pch + 1, &pch, 10);
1039 if (*pch != stop)
1040 return 0;
1041
1042 tmp.subject = subj;
1043 *setme = tmp;
1044 *offset = pch - subj;
1045 return 1;
1046 }
1047
1048
1049 /*
1050 * Taken from tags.c but changed to use indices into arts[] instead of
1051 * base_index. Changed so that even when we don't have all the parts we
1052 * return a valid array.
1053 *
1054 * Tries to find all the parts to the multipart message pointed to by
1055 * base_index.
1056 *
1057 * @return on success, the number of parts found. On failure, zero if not a
1058 * multipart or the negative value of the first missing part.
1059 * @param base_index index pointing to one of the messages in a multipart
1060 * message.
1061 * @param malloc_and_setme_info on success, set to a malloced array the
1062 * parts found.
1063 */
1064 static int
1065 global_get_multiparts(
1066 int aindex,
1067 MultiPartInfo **malloc_and_setme_info)
1068 {
1069 int i;
1070 int part_index;
1071 MultiPartInfo tmp, tmp2;
1072 MultiPartInfo *info = NULL;
1073
1074 /* entry assertions */
1075 assert(0 <= aindex && aindex < top_art && "Invalid index");
1076 assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1077
1078 /* make sure this is a multipart message... */
1079 if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1080 return 0;
1081
1082 /* make a temporary buffer to hold the multipart info... */
1083 info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
1084
1085 /* zero out part-number for the repost check below */
1086 for (i = 0; i < tmp.total; ++i) {
1087 info[i].total = tmp.total; /* Added this for thread_by_multipart */
1088 info[i].part_number = -1;
1089 }
1090
1091 /* try to find all the multiparts... */
1092 for_each_art(i) {
1093 if (strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
1094 continue;
1095
1096 if (!global_get_multipart_info(i, &tmp2))
1097 continue;
1098
1099 /* 'test (1/5)' is not the same as 'test (1/15)' */
1100 if (tmp.total != tmp2.total)
1101 continue;
1102
1103 part_index = tmp2.part_number - 1;
1104
1105 /*
1106 * skip "blah (00/102)" or "blah (103/102)" subjects
1107 */
1108 if (part_index < 0 || part_index >= tmp.total)
1109 continue;
1110
1111 /* repost check: do we already have this part? */
1112 if (info[part_index].part_number != -1) {
1113 assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1114 continue;
1115 }
1116
1117 /* we have a match, hooray! */
1118 info[part_index] = tmp2;
1119 }
1120
1121 /* see if we got them all. */
1122 for (i = 0; i < tmp.total; ++i) {
1123 if (info[i].part_number != i + 1) {
1124 *malloc_and_setme_info = info;
1125 return -(i + 1); /* missing part #(i+1) */
1126 }
1127 }
1128
1129 /* looks like a success .. */
1130 *malloc_and_setme_info = info;
1131 return tmp.total;
1132 }
1133
1134
1135 /*
1136 * The algorithm uses the tag multipart searches to thread articles together.
1137 */
1138 static void
1139 thread_by_multipart(
1140 void)
1141 {
1142 int i, j, threadNum;
1143 MultiPartInfo *minfo = NULL;
1144
1145 for_each_art(i) {
1146 if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo))
1147 continue;
1148
1149 threadNum = -1;
1150 for (j = minfo[0].total - 1; j >= 0; j--) {
1151 if (minfo[j].part_number != -1) {
1152 if (threadNum != -1) {
1153 arts[minfo[j].base_index].thread = threadNum;
1154 arts[threadNum].prev = minfo[j].base_index;
1155 }
1156
1157 threadNum = minfo[j].base_index;
1158 }
1159 }
1160 }
1161 free(minfo);
1162 }
1163
1164
1165 /*
1166 * Go through the articles in arts[] and create threads. There are
1167 * 5 strategies currently defined :
1168 *
1169 * THREAD_NONE No threading
1170 * THREAD_SUBJ Threads are created using like Subject lines
1171 * THREAD_REFS Threads are created using the References headers
1172 * THREAD_BOTH Threads created using References and then Subject
1173 * THREAD_MULTI Threads created using Subject to search for Multiparts
1174 * THREAD_PERC Threads based upon a char for char match of greater than x%
1175 *
1176 * .thread and .prev are used to hold the threading information, see tin.h for
1177 * more information
1178 * Only process valid (unexpired) articles we haven't visited yet
1179 * (ie arts[].thread == ART_UNTHREADED)
1180 *
1181 * The rethread parameter is used to force the deletion of existing threading
1182 * information before threading which happens anyway expect when using
1183 * THREAD_NONE (I don't immediately see how this is useful)
1184 */
1185 /* TODO: rewrite that user can easily combine different 'threading'
1186 * methods, i.e:
1187 * - thread_by_multipart() + collate_subjects()
1188 */
1189 void
1190 make_threads(
1191 struct t_group *group,
1192 t_bool rethread)
1193 {
1194 if (!cmd_line && !batch_mode)
1195 info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
1196
1197 #ifdef DEBUG
1198 if (debug & DEBUG_MISC)
1199 error_message(2, "rethread=[%d] thread_articles=[%d] attr_thread_articles=[%d]",
1200 rethread, tinrc.thread_articles, group->attribute->thread_articles);
1201 #endif /* DEBUG */
1202
1203 /*
1204 * Sort all the articles using the preferred method
1205 * When find_base() is called, the bases are created ordered
1206 * on arts[] and so the base messages under all threading systems
1207 * will be sorted in this way.
1208 */
1209 sort_arts(group->attribute->sort_article_type);
1210
1211 /*
1212 * Reset all the ptrs to articles following the above sort
1213 */
1214 clear_art_ptrs();
1215
1216 /*
1217 * The threading pointers need to be reset if re-threading
1218 * If using ref threading, revector the links back to the articles
1219 */
1220 if (rethread || group->attribute->thread_articles) {
1221 int i;
1222
1223 for_each_art(i) {
1224 if (arts[i].thread >= 0)
1225 arts[i].thread = ART_UNTHREADED;
1226
1227 arts[i].prev = ART_NORMAL;
1228
1229 /* Should never happen if tree is built properly */
1230 if (arts[i].refptr == NULL) {
1231 #ifdef DEBUG
1232 if (debug & DEBUG_REFS) {
1233 my_fprintf(stderr, "\nError : art->refptr is NULL\n");
1234 my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1235 my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1236 my_fprintf(stderr, "From : %s\n", arts[i].from);
1237 assert(arts[i].refptr != NULL);
1238 } else
1239 #endif /* DEBUG */
1240 continue;
1241 }
1242 arts[i].refptr->article = i;
1243 }
1244 }
1245
1246 /*
1247 * Do the right thing according to the threading strategy
1248 */
1249 switch (group->attribute->thread_articles) {
1250 case THREAD_NONE:
1251 break;
1252
1253 case THREAD_SUBJ:
1254 thread_by_subject();
1255 break;
1256
1257 case THREAD_REFS:
1258 thread_by_reference();
1259 break;
1260
1261 case THREAD_BOTH:
1262 thread_by_reference();
1263 collate_subjects();
1264 break;
1265
1266 case THREAD_MULTI:
1267 thread_by_multipart();
1268 break;
1269
1270 case THREAD_PERC:
1271 thread_by_percentage(group);
1272 break;
1273
1274 default: /* not reached */
1275 break;
1276 }
1277
1278 /*
1279 * Rebuild base[]
1280 */
1281 find_base(group);
1282 }
1283
1284
1285 static t_compfunc
1286 eval_sort_arts_func(
1287 unsigned int sort_art_type)
1288 {
1289 switch (sort_art_type) {
1290 case SORT_ARTICLES_BY_NOTHING: /* don't sort at all */
1291 return artnum_comp;
1292
1293 case SORT_ARTICLES_BY_SUBJ_DESCEND:
1294 return subj_comp_desc;
1295
1296 case SORT_ARTICLES_BY_SUBJ_ASCEND:
1297 return subj_comp_asc;
1298
1299 case SORT_ARTICLES_BY_FROM_DESCEND:
1300 return from_comp_desc;
1301
1302 case SORT_ARTICLES_BY_FROM_ASCEND:
1303 return from_comp_asc;
1304
1305 case SORT_ARTICLES_BY_DATE_DESCEND:
1306 return date_comp_desc;
1307
1308 case SORT_ARTICLES_BY_DATE_ASCEND:
1309 return date_comp_asc;
1310
1311 case SORT_ARTICLES_BY_SCORE_DESCEND:
1312 return score_comp_desc;
1313
1314 case SORT_ARTICLES_BY_SCORE_ASCEND:
1315 return score_comp_asc;
1316
1317 case SORT_ARTICLES_BY_LINES_DESCEND:
1318 return lines_comp_desc;
1319
1320 case SORT_ARTICLES_BY_LINES_ASCEND:
1321 return lines_comp_asc;
1322
1323 default:
1324 break;
1325 }
1326 return NULL;
1327 }
1328
1329
1330 void
1331 sort_arts(
1332 unsigned int sort_art_type)
1333 {
1334 t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1335
1336 if (comp_func)
1337 SortBy(comp_func);
1338 }
1339
1340
1341 static void
1342 sort_base(
1343 unsigned int sort_threads_type)
1344 {
1345 switch (sort_threads_type) {
1346 case SORT_THREADS_BY_SCORE_DESCEND:
1347 case SORT_THREADS_BY_SCORE_ASCEND:
1348 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1349 break;
1350
1351 case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
1352 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
1353 break;
1354
1355 case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
1356 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
1357 break;
1358 }
1359 }
1360
1361
1362 /*
1363 * This is called to get header info for articles not already found in the
1364 * overview files.
1365 * Code reads (max_lineno) lines of article to catch headers like Archive-name:
1366 * which are not normally included in XOVER or even the normal block of headers.
1367 * How this is supposed to be useful when 99% of the time we'll have overview
1368 * data I don't know...
1369 * TODO: move Archive-name: parsing to article body parsing, remove the
1370 * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
1371 * TODO: currgrp->auto_save is set, otherwise it is redundant info
1372 */
1373 static t_bool
1374 parse_headers(
1375 FILE *fp,
1376 struct t_article *h)
1377 {
1378 char art_from_addr[HEADER_LEN];
1379 char art_full_name[HEADER_LEN];
1380 char *hdr, *ptr;
1381 unsigned int lineno = 0;
1382 unsigned int max_lineno = 25;
1383 t_bool got_from, got_lines, got_received;
1384
1385 got_from = got_lines = got_received = FALSE;
1386
1387 while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1388 /*
1389 * Look for the end of information which tin wants to get.
1390 * Applies when reading local spool and via NNTP.
1391 */
1392
1393 /*
1394 * as Archive-name: is placed in the body it's safe to exit
1395 * the loop if it was found. Don't mix up with Archive: from
1396 * RFC 5536.
1397 */
1398 if (lineno++ > max_lineno || h->archive)
1399 break;
1400
1401 unfold_header(ptr);
1402 switch (toupper((unsigned char) *ptr)) {
1403 case 'A': /* Archive-name: optional */
1404 /*
1405 * Archive-name: {name}/{part|patch}{number}
1406 * eg, acorn/faq/part01
1407 */
1408 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
1409 char *s;
1410
1411 if ((s = strrchr(hdr, '/')) != NULL) {
1412 struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
1413
1414 if (STRNCASECMPEQ(s + 1, "part", 4)) {
1415 archptr->partnum = my_strdup(s + 5);
1416 archptr->ispart = TRUE;
1417 } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
1418 archptr->partnum = my_strdup(s + 6);
1419 archptr->ispart = FALSE;
1420 } else { /* part or patch must be present */
1421 free(archptr);
1422 continue;
1423 }
1424 strtok(archptr->partnum, "\n");
1425 *s = '\0';
1426 archptr->name = hash_str(hdr);
1427 h->archive = archptr;
1428 }
1429 }
1430 break;
1431
1432 case 'D': /* Date: mandatory */
1433 if (!h->date) {
1434 if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1435 h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1436 }
1437 break;
1438
1439 case 'F': /* From: mandatory */
1440 if (!got_from) {
1441 if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1442 h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1443 h->from = hash_str(buffer_to_ascii(art_from_addr));
1444 if (*art_full_name)
1445 h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
1446 got_from = TRUE;
1447 }
1448 }
1449 break;
1450
1451 case 'L': /* Lines: optional */
1452 if (!got_lines) {
1453 if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1454 h->line_count = atoi(hdr);
1455 got_lines = TRUE;
1456 }
1457 }
1458 break;
1459
1460 case 'M': /* Message-ID: mandatory */
1461 if (!h->msgid) {
1462 if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1463 h->msgid = my_strdup(hdr);
1464 }
1465 break;
1466
1467 /* for Path:-filter when reading from local spool */
1468 case 'P': /* Path: */
1469 if (!h->path) {
1470 if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1471 h->path = my_strdup(hdr);
1472 }
1473 break;
1474
1475 case 'R': /* References: optional */
1476 if (!h->refs) {
1477 if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1478 h->refs = my_strdup(hdr);
1479 }
1480
1481 /* Received: If found it's probably a mail article */
1482 if (!got_received) {
1483 if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
1484 max_lineno <<= 1; /* double the max number of line to read for mails */
1485 got_received = TRUE;
1486 }
1487 }
1488 break;
1489
1490 case 'S': /* Subject: mandatory */
1491 if (!h->subject) {
1492 if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE)))
1493 h->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(hdr), FALSE)), FALSE));
1494 }
1495 break;
1496
1497 case 'X': /* Xref: optional */
1498 if (!h->xref) {
1499 if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1500 h->xref = my_strdup(hdr);
1501 }
1502 break;
1503
1504 default:
1505 break;
1506 } /* switch */
1507
1508 } /* while */
1509
1510 #ifdef NNTP_ABLE
1511 if (ptr)
1512 drain_buffer(fp);
1513 #endif /* NNTP_ABLE */
1514
1515 if (tin_errno)
1516 return FALSE;
1517
1518 /*
1519 * The son of RFC 1036 states that the following hdrs are mandatory. It
1520 * also states that Subject, Newsgroups and Path are too. Ho hum.
1521 *
1522 * What about reading mail from local spool via ~/.tin/active.mail,
1523 * they might not have a Message-ID but got_received is very likely to
1524 * be true.
1525 */
1526 if (got_from && h->date && h->msgid) {
1527 if (!h->subject)
1528 h->subject = hash_str("<No subject>");
1529
1530 #ifdef DEBUG
1531 debug_print_header(h);
1532 #endif /* DEBUG */
1533 return TRUE;
1534 }
1535
1536 return FALSE;
1537 }
1538
1539 #ifdef NNTP_ABLE
1540 /*
1541 * Loop over arts[] and find ranges without Path: header
1542 * If there are any try to optimize the ranges regarding traffic consumption
1543 * Start optimization if at least MIN_CNT ranges exist
1544 * If there are more than MAX_CNT ranges after optimization, fetch all in one
1545 * big range
1546 */
1547 #define MIN_CNT 10
1548 #define MAX_CNT 50
1549 static struct t_article_range *
1550 build_range_list(
1551 t_artnum min,
1552 t_artnum max,
1553 int *range_cnt)
1554 {
1555 int i, gap_cnt = 0;
1556 struct t_article_range *res = NULL, *gap_list, *curr, *from;
1557 t_artnum new_end;
1558
1559 new_end = T_ARTNUM_CONST(0);
1560 gap_list = my_malloc(sizeof(struct t_article_range));
1561 curr = gap_list;
1562 curr->start = min;
1563 curr->end = max;
1564 curr->cnt = T_ARTNUM_CONST(0);
1565 curr->next = NULL;
1566
1567 for_each_art(i) {
1568 if (arts[i].artnum < min)
1569 continue;
1570 if (arts[i].artnum > max)
1571 break;
1572 if (arts[i].path) {
1573 for (; i < top_art && arts[i].path; i++)
1574 ;
1575 /*
1576 * the current art has no path -> we use this one
1577 * if we reached top_art all arts have path
1578 * so we use max
1579 */
1580 curr->start = i == top_art ? max : arts[i--].artnum;
1581 } else {
1582 for (; i < top_art && !arts[i].path; i++)
1583 ;
1584 /* the current art has path -> we use the last one */
1585 new_end = curr->end = arts[--i].artnum;
1586 }
1587 if (new_end) {
1588 curr->cnt = curr->end - curr->start + 1;
1589 curr->next = my_malloc(sizeof(struct t_article_range));
1590 curr = curr->next;
1591 curr->start = new_end;
1592 curr->end = max;
1593 curr->cnt = T_ARTNUM_CONST(0);
1594 curr->next = NULL;
1595 new_end = T_ARTNUM_CONST(0);
1596 }
1597 }
1598
1599 curr = gap_list;
1600 while (curr && curr->cnt) {
1601 ++gap_cnt;
1602 # ifdef DEBUG
1603 if ((debug & DEBUG_NNTP) && verbose > 1)
1604 debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1605 # endif /* DEBUG */
1606 curr = curr->next;
1607 }
1608
1609 /*
1610 * Optimize only if there are at least MIN_CNT ranges
1611 */
1612 if (gap_cnt >= MIN_CNT) {
1613 res = my_malloc(sizeof(struct t_article_range));
1614 res->start = T_ARTNUM_CONST(0);
1615 res->end = T_ARTNUM_CONST(0);
1616 res->cnt = T_ARTNUM_CONST(0);
1617 res->next = NULL;
1618
1619 from = gap_list;
1620 curr = res;
1621 while (from) {
1622 curr->start = from->start;
1623 curr->end = from->end;
1624 curr->cnt = from->cnt;
1625 if ((from = from->next)) {
1626 /*
1627 * If the next range is grater then the gap between the current
1628 * one and the next one we build a new range including the
1629 * current one, the next one and the gap between
1630 */
1631 while (from && from->cnt >= from->start - curr->end - 1) {
1632 curr->end = from->end;
1633 from = from->next;
1634 }
1635 curr->cnt = curr->end - curr->start + 1;
1636 curr->next = my_malloc(sizeof(struct t_article_range));
1637 curr = curr->next;
1638 curr->start = T_ARTNUM_CONST(0);
1639 curr->end = T_ARTNUM_CONST(0);
1640 curr->cnt = T_ARTNUM_CONST(0);
1641 curr->next = NULL;
1642 }
1643 }
1644 }
1645
1646 /*
1647 * If there are less then MIN_CNT ranges
1648 * no res is build -> return the original list
1649 */
1650 if (res) {
1651 while (gap_list) {
1652 curr = gap_list;
1653 gap_list = curr->next;
1654 free(curr);
1655 }
1656 } else
1657 res = gap_list;
1658
1659 curr = res;
1660 gap_cnt = 0;
1661 while (curr && curr->cnt) {
1662 ++gap_cnt;
1663 # ifdef DEBUG
1664 if ((debug & DEBUG_NNTP) && verbose > 1)
1665 debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1666 # endif /* DEBUG */
1667 curr = curr->next;
1668 }
1669
1670 if (gap_cnt >= MAX_CNT) {
1671 curr = res;
1672 while (curr->next && curr->next->cnt) {
1673 res->end = curr->next->end;
1674 curr->next->cnt = 0;
1675 curr = curr->next;
1676 }
1677 res->cnt = res->end - res->start + 1;
1678 gap_cnt = 1;
1679 # ifdef DEBUG
1680 if ((debug & DEBUG_NNTP) && verbose > 1) {
1681 debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1682 }
1683 # endif /* DEBUG */
1684 }
1685 *range_cnt = gap_cnt;
1686
1687 return res;
1688 }
1689
1690
1691 /*
1692 * Fetch the Path header in case we want to filter on that in the given group
1693 *
1694 * Try [X]HDR first, then XPAT
1695 */
1696 static t_bool
1697 get_path_header(
1698 int cur,
1699 int cnt,
1700 struct t_group *group,
1701 t_artnum min,
1702 t_artnum max)
1703 {
1704 FILE *fp = NULL;
1705 char *prep_msg;
1706 char *buf, *ptr;
1707 char cmd[NNTP_STRLEN];
1708 t_artnum artnum, i;
1709 t_bool found = FALSE;
1710 static t_bool supported = TRUE; /* assume HDR || XPAT works */
1711
1712 if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1713 return FALSE;
1714
1715 # ifdef DEBUG
1716 if ((debug & DEBUG_NNTP) && verbose > 1)
1717 debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1718 # endif /* DEBUG */
1719
1720 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
1721 int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1722 switch (j) {
1723 case 215:
1724 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1725 # ifdef DEBUG
1726 if (debug & DEBUG_NNTP)
1727 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1728 # endif /* DEBUG */
1729 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
1730 strcat(nntp_caps.headers_range, ptr);
1731 strcat(nntp_caps.headers_range, "\n");
1732 }
1733 break;
1734
1735 default:
1736 break;
1737 }
1738 }
1739
1740 /* does HDR return Path? */
1741 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1742 do {
1743 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1744 found = TRUE;
1745 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1746 }
1747
1748 if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1749 if (min == max)
1750 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1751 else
1752 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1753 fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1754 if (!nntp_caps.hdr && fp)
1755 nntp_caps.hdr = TRUE;
1756 } else if (nntp_caps.xpat) {
1757 if (min == max)
1758 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1759 else
1760 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1761 fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1762 }
1763
1764 if (fp) {
1765 prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1766 while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1767 # ifdef DEBUG
1768 if ((debug & DEBUG_NNTP) && verbose)
1769 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1770 # endif /* DEBUG */
1771 if ((ptr = tin_strtok(buf, " ")) == NULL)
1772 continue;
1773 artnum = atoartnum(ptr);
1774 if ((ptr = tin_strtok(NULL, " ")) == NULL)
1775 continue;
1776 for_each_art(i) {
1777 if (arts[i].artnum == artnum) {
1778 FreeIfNeeded(arts[i].path);
1779 arts[i].path = my_strdup(ptr);
1780 }
1781 }
1782 if (++artnum % MODULO_COUNT_NUM == 0)
1783 show_progress(prep_msg, artnum - min, max - min);
1784 }
1785 free(prep_msg);
1786 return supported;
1787 }
1788
1789 /* !fp */
1790 supported = FALSE;
1791 if (nntp_caps.xpat)
1792 nntp_caps.xpat = FALSE;
1793 /* as nntp_caps.hdr may work with other headers we don't disable it*/
1794
1795 #ifdef DEBUG
1796 if ((debug & DEBUG_NNTP) && verbose > 1)
1797 debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supportet.", group->name);
1798 #endif /* DEBUG */
1799 return supported;
1800 }
1801 #endif /* NNTP_ABLE */
1802
1803
1804 /*
1805 * Read in an overview index file. Fields are separated by TAB.
1806 * return the number of expired articles encountered or -1 if the user aborted
1807 * the read
1808 * 'top' is set to the highest artnum read
1809 * If 'local' is set then always open local overview cache in preference to
1810 * using NNTP XOVER
1811 *
1812 * Format (mandatory as far as line count [RFC2980]):
1813 * 1. article number (ie. 183) [mandatory]
1814 * 2. Subject: line (ie. Which newsreader?) [mandatory]
1815 * 3. From: line (ie. iain@ecrc.de) [mandatory]
1816 * 4. Date: line (rfc822 format) [mandatory]
1817 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
1818 * 6. References: (ie. <message-id> ....) [optional]
1819 * 7. Byte count (Skipped - not used) [mandatory]
1820 * 8. Line count (ie. 23) [mandatory]
1821 * 9. Xref: line (ie. alt.test:389) [optional]
1822 */
1823 static int
1824 read_overview(
1825 struct t_group *group,
1826 t_artnum min,
1827 t_artnum max,
1828 t_artnum *top,
1829 t_bool local,
1830 t_bool *rebuild_cache)
1831 {
1832 FILE *fp;
1833 char *ptr;
1834 char *q;
1835 char *buf;
1836 char *group_msg;
1837 char art_full_name[HEADER_LEN];
1838 char art_from_addr[HEADER_LEN];
1839 unsigned int count;
1840 int expired = 0;
1841 t_artnum artnum;
1842 t_bool path_found = FALSE, path_in_ofmt = FALSE;
1843 struct t_article *art;
1844 size_t over_fields = 1;
1845
1846 /*
1847 * open the overview file (whether it be local or via nntp)
1848 */
1849 if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1850 return expired;
1851
1852 if (group->xmax > max)
1853 group->xmax = max;
1854
1855 group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1856
1857 /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1858 if (ofmt) {
1859 for (; ofmt[over_fields].name; over_fields++) {
1860 if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1861 path_in_ofmt = TRUE;
1862 }
1863 }
1864
1865 if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1866 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1867 ofmt[0].type = OVER_T_INT;
1868 ofmt[0].name = my_strdup("Artnum:");
1869 ofmt[1].type = OVER_T_STRING;
1870 ofmt[1].name = my_strdup("Subject:");
1871 ofmt[2].type = OVER_T_STRING;
1872 ofmt[2].name = my_strdup("From:");
1873 ofmt[3].type = OVER_T_STRING;
1874 ofmt[3].name = my_strdup("Date:");
1875 ofmt[4].type = OVER_T_STRING;
1876 ofmt[4].name = my_strdup("Message-ID:");
1877 ofmt[5].type = OVER_T_STRING;
1878 ofmt[5].name = my_strdup("References:");
1879 ofmt[6].type = OVER_T_INT;
1880 ofmt[6].name = my_strdup("Bytes:");
1881 ofmt[7].type = OVER_T_INT;
1882 ofmt[7].name = my_strdup("Lines:");
1883 ofmt[8].type = OVER_T_ERROR;
1884 ofmt[8].name = NULL;
1885 over_fields = 7;
1886 }
1887
1888 while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1889 #ifdef DEBUG
1890 if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1891 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1892 #endif /* DEBUG */
1893
1894 if (need_resize) {
1895 handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
1896 need_resize = cNo;
1897 }
1898
1899 /*
1900 * Read artnum
1901 */
1902 if ((ptr = tin_strtok(buf, "\t")) == NULL)
1903 continue;
1904
1905 /*
1906 * read the article number, guaranteed to be the first field
1907 */
1908 artnum = atoartnum(ptr);
1909
1910 /*
1911 * artnum field invalid/corrupt or is 1st line of local cached overview
1912 * (group name)
1913 */
1914 if (artnum <= 0)
1915 continue;
1916
1917 /*
1918 * skip artnums below the given minimum (getart_limit)
1919 */
1920 if (artnum < min)
1921 continue;
1922
1923 /*
1924 * Check to make sure article in nov file has not expired in group
1925 */
1926 if (artnum < group->xmin) {
1927 expired++;
1928 continue;
1929 }
1930
1931 /*
1932 * artnum in overview data higher than groups high mark
1933 *
1934 * TODO: - warn user about broken overviews?
1935 * - try to parse the Xref:-line to get the correct artnum
1936 * - see also parse_unread_arts()
1937 */
1938 if (artnum > group->xmax)
1939 continue;
1940
1941 if (top_art >= max_art)
1942 expand_art();
1943
1944 art = &arts[top_art];
1945 set_article(art);
1946 art->artnum = *top = artnum;
1947
1948 /*
1949 * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
1950 * to check for additions like we do with xref_supported
1951 */
1952 for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
1953 /* skip unexpected tailing fields */
1954 if (count > over_fields) {
1955 #ifdef DEBUG
1956 if ((debug & DEBUG_NNTP) && verbose > 1)
1957 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
1958 #endif /* DEBUG */
1959
1960 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
1961 if (count == over_fields + 1) {
1962 if (!strncasecmp(ptr, "Xref: ", 6)) {
1963 #ifdef DEBUG
1964 if ((debug & DEBUG_NNTP) && verbose > 1)
1965 debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
1966 #endif /* DEBUG */
1967 over_fields++;
1968 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1969 ofmt[over_fields].type = OVER_T_FSTRING;
1970 ofmt[over_fields].name = my_strdup("Xref:");
1971 ofmt[over_fields + 1].type = OVER_T_ERROR;
1972 ofmt[over_fields + 1].name = NULL;
1973 xref_supported = TRUE;
1974 } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
1975 #ifdef DEBUG
1976 if ((debug & DEBUG_NNTP) && verbose > 1)
1977 debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
1978 #endif /* DEBUG */
1979 over_fields++;
1980 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1981 ofmt[over_fields].type = OVER_T_FSTRING;
1982 ofmt[over_fields].name = my_strdup("Path:");
1983 ofmt[over_fields + 1].type = OVER_T_ERROR;
1984 ofmt[over_fields + 1].name = NULL;
1985 xref_supported = TRUE;
1986 } else
1987 continue;
1988 } else
1989 continue;
1990 }
1991
1992 /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
1993 if (expensive_over_parse) { /* strange order */
1994 /* madatory fields */
1995 if (ofmt[count].type == OVER_T_STRING) {
1996 if (!strcasecmp(ofmt[count].name, "Subject:")) {
1997 if (*ptr)
1998 art->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
1999 else {
2000 art->subject = hash_str("");
2001 #ifdef DEBUG
2002 if ((debug & DEBUG_NNTP) && verbose > 1)
2003 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2004 #endif /* DEBUG */
2005 }
2006 continue;
2007 }
2008
2009 if (!strcasecmp(ofmt[count].name, "From:")) {
2010 if (*ptr) {
2011 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2012 art->from = hash_str(buffer_to_ascii(art_from_addr));
2013 if (*art_full_name)
2014 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2015 } else {
2016 art->from = hash_str("");
2017 #ifdef DEBUG
2018 if ((debug & DEBUG_NNTP) && verbose > 1)
2019 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2020 #endif /* DEBUG */
2021 }
2022 continue;
2023 }
2024
2025 if (!strcasecmp(ofmt[count].name, "Date:")) {
2026 art->date = parsedate(ptr, (TIMEINFO *) 0);
2027 #ifdef DEBUG
2028 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2029 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2030 #endif /* DEBUG */
2031 continue;
2032 }
2033
2034 if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2035 if (*ptr) {
2036 FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2037 art->msgid = my_strdup(ptr);
2038 } else {
2039 art->msgid = NULL;
2040 #ifdef DEBUG
2041 if ((debug & DEBUG_NNTP) && verbose > 1)
2042 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2043 #endif /* DEBUG */
2044 }
2045 continue;
2046 }
2047
2048 if (!strcasecmp(ofmt[count].name, "References:")) {
2049 if (*ptr) {
2050 FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2051 art->refs = my_strdup(ptr);
2052 } else
2053 art->refs = NULL;
2054 continue;
2055 }
2056
2057 /*
2058 * non std. fields when doing
2059 * expensive overview parsing (very
2060 * rare, just happens if RFC 3977
2061 * 8.4.2 is violated) go here
2062 */
2063 /* for Path:-filter */
2064 if (!strcasecmp(ofmt[count].name, "Path:")) {
2065 if (!path_found)
2066 path_found = TRUE;
2067 if (*ptr) {
2068 FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2069 art->path = my_strdup(ptr);
2070 } else
2071 art->path = NULL;
2072 continue;
2073 }
2074 }
2075 /* metadata fields */
2076 if (ofmt[count].type == OVER_T_INT) {
2077 if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2078 if (*ptr) {
2079 #ifdef DEBUG
2080 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2081 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2082 #endif /* DEBUG */
2083 }
2084 continue;
2085 }
2086
2087 if (!strcasecmp(ofmt[count].name, "Lines:")) {
2088 if (*ptr) {
2089 if (isdigit((unsigned char) *ptr))
2090 art->line_count = atoi(ptr);
2091 else {
2092 art->line_count = 0;
2093 #ifdef DEBUG
2094 if ((debug & DEBUG_NNTP) && verbose > 1)
2095 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2096 #endif /* DEBUG */
2097 }
2098 } else
2099 art->line_count = 0;
2100 continue;
2101 }
2102 }
2103 } else { /* first 7 fields are in RFC 3977 order */
2104 switch (count) {
2105 case 1: /* Subject: */
2106 if (*ptr)
2107 art->subject = hash_str(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2108 else {
2109 art->subject = hash_str("");
2110 #ifdef DEBUG
2111 if ((debug & DEBUG_NNTP) && verbose > 1)
2112 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2113 #endif /* DEBUG */
2114 }
2115 break;
2116
2117 case 2: /* From: */
2118 if (*ptr) {
2119 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2120 art->from = hash_str(buffer_to_ascii(art_from_addr));
2121 if (*art_full_name)
2122 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2123 } else {
2124 art->from = hash_str("");
2125 #ifdef DEBUG
2126 if ((debug & DEBUG_NNTP) && verbose > 1)
2127 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2128 #endif /* DEBUG */
2129 }
2130 break;
2131
2132 case 3: /* Date: */
2133 art->date = parsedate(ptr, (TIMEINFO *) 0);
2134 #ifdef DEBUG
2135 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2136 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2137 #endif /* DEBUG */
2138 break;
2139
2140 case 4: /* Message-ID: */
2141 if (*ptr)
2142 art->msgid = my_strdup(ptr);
2143 else {
2144 art->msgid = NULL;
2145 #ifdef DEBUG
2146 if ((debug & DEBUG_NNTP) && verbose > 1)
2147 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2148 #endif /* DEBUG */
2149 }
2150 break;
2151
2152 case 5: /* References: */
2153 if (*ptr)
2154 art->refs = my_strdup(ptr);
2155 else
2156 art->refs = NULL;
2157 break;
2158
2159 case 6: /* :bytes || Bytes: */
2160 if (*ptr) {
2161 #ifdef DEBUG
2162 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2163 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2164 #endif /* DEBUG */
2165 }
2166 break;
2167
2168 case 7: /* :lines || Lines: */
2169 if (*ptr) {
2170 if (isdigit((unsigned char) *ptr))
2171 art->line_count = atoi(ptr);
2172 else {
2173 art->line_count = 0;
2174 #ifdef DEBUG
2175 if ((debug & DEBUG_NNTP) && verbose > 1)
2176 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2177 #endif /* DEBUG */
2178 }
2179 } else
2180 art->line_count = 0;
2181 break;
2182
2183 default:
2184 break;
2185 }
2186 }
2187
2188 /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2189 if (ofmt[count].type == OVER_T_FSTRING) {
2190 if (*ptr) {
2191 if (!strcasecmp(ofmt[count].name, "Xref:")) {
2192 if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2193 FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2194 art->xref = my_strdup(q);
2195 }
2196 #ifdef DEBUG
2197 else {
2198 if ((debug & DEBUG_NNTP) && verbose > 1)
2199 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2200 }
2201 #endif /* DEBUG */
2202 continue;
2203 }
2204 /*
2205 * handling of addition overview fields
2206 * goes here
2207 */
2208 #ifdef DEBUG
2209 if ((debug & DEBUG_NNTP) && verbose > 1)
2210 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2211 #endif /* DEBUG */
2212 /* if we're lucky we've Path in NOV */
2213 /*
2214 * if reading locally cached overview data try
2215 * path regardless of the server OVERVIEW.FMT
2216 */
2217 if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2218 if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2219 if (!path_found)
2220 path_found = TRUE;
2221 FreeIfNeeded(art->path);
2222 art->path = my_strdup(q);
2223 #ifdef DEBUG
2224 if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2225 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2226 #endif /* DEBUG */
2227
2228 }
2229 continue;
2230 }
2231 }
2232 continue;
2233 }
2234 }
2235
2236 /*
2237 * RFC says Message-ID is mandatory in newsgroups (but not in
2238 * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2239 * threading in mailgroups
2240 */
2241 if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2242 continue;
2243
2244 /* we might lose accuracy here, but that shouldn't hurt */
2245 if (artnum % MODULO_COUNT_NUM == 0)
2246 show_progress(group_msg, artnum - min, max - min);
2247
2248 top_art++; /* Basically this statement commits the article */
2249 }
2250
2251 free(group_msg);
2252 TIN_FCLOSE(fp);
2253
2254 if (tin_errno)
2255 return -1;
2256
2257 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2258 if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
2259 char cbuf[HEADER_LEN];
2260 int i;
2261 static t_bool found;
2262 static t_bool first = TRUE;
2263
2264 if (first) {
2265 found = TRUE;
2266 /*
2267 * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2268 */
2269 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
2270 if (!*nntp_caps.headers_range) {
2271 i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2272
2273 found = FALSE;
2274 switch (i) {
2275 case 215:
2276 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2277 # ifdef DEBUG
2278 if (debug & DEBUG_NNTP)
2279 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2280 # endif /* DEBUG */
2281 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2282 found = TRUE;
2283 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
2284 strcat(nntp_caps.headers_range, ptr);
2285 strcat(nntp_caps.headers_range, "\n");
2286 }
2287 break;
2288
2289 default:
2290 break;
2291 }
2292 first = FALSE;
2293 } else {
2294 found = FALSE;
2295 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
2296 do {
2297 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2298 found = TRUE;
2299 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2300 }
2301 }
2302 }
2303 }
2304
2305 if (found) {
2306 snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2307 group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2308 if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2309 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2310 # ifdef DEBUG
2311 if ((debug & DEBUG_NNTP) && verbose)
2312 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2313 # endif /* DEBUG */
2314
2315 artnum = atoartnum(ptr);
2316 if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2317 continue;
2318 art = &arts[top_art];
2319 set_article(art);
2320 if (!art->xref && !strstr(ptr, "(none)")) {
2321 if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2322 continue;
2323 ptr = q;
2324 while (*ptr && isspace((int) *ptr))
2325 ptr++;
2326 q = strchr(ptr, '\n');
2327 if (q)
2328 *q = '\0';
2329 art->xref = my_strdup(ptr);
2330 }
2331 /* we might lose accuracy here, but that shouldn't hurt */
2332 if (artnum % MODULO_COUNT_NUM == 0)
2333 show_progress(group_msg, artnum - min, max - min);
2334 }
2335 }
2336 free(group_msg);
2337 }
2338 }
2339 #endif /* NNTP_ABLE && XHDR_XREF */
2340
2341 if (local) {
2342 #ifdef NNTP_ABLE
2343 if (filter_on_path(group)) {
2344 struct t_article_range *ranges, *curr;
2345 t_bool supported = TRUE;
2346 int curr_range, range_cnt;
2347
2348 /*
2349 * Get the ranges without Path: header and try to fetch the
2350 * headers
2351 */
2352 if ((ranges = build_range_list(min, *top, &range_cnt))) {
2353 curr = ranges;
2354 curr_range = 1;
2355 while (curr && supported) {
2356 if (curr->cnt)
2357 supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2358 curr = curr->next;
2359 }
2360 if (!supported && path_in_ofmt) {
2361 /*
2362 * fetching Path: headers via [X]HDR or XPAT has failed
2363 * Path: is in the servers overview so let the next
2364 * read_overview() fetch them
2365 */
2366 free_art_array();
2367 free_msgids();
2368 top_art = 0;
2369 *top = T_ARTNUM_CONST(0);
2370 expired = 0;
2371 }
2372 *rebuild_cache = TRUE;
2373 while (ranges) {
2374 curr = ranges;
2375 ranges = curr->next;
2376 free(curr);
2377 }
2378 }
2379 }
2380 #endif /* NNTP_ABLE */
2381 } else
2382 if (!path_found && filter_on_path(group)) {
2383 #ifdef NNTP_ABLE
2384 if (!get_path_header(1, 1, group, min, *top))
2385 #endif /* NNTP_ABLE */
2386 wait_message(2, _(txt_cannot_filter_on_path));
2387 }
2388 return expired;
2389 }
2390
2391
2392 /*
2393 * Write an Nov/Xover index file. Fields are separated by '\t'.
2394 *
2395 * Format:
2396 * 1. article number (ie. 183) [mandatory]
2397 * 2. Subject: line (ie. Which newsreader?) [mandatory]
2398 * 3. From: line (ie. iain@ecrc.de) [mandatory]
2399 * 4. Date: line (rfc822 format) [mandatory]
2400 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
2401 * 6. References: (ie. <message-id> ....) [optional]
2402 * 7. Byte count (Skipped - not used) [mandatory]
2403 * 8. Line count (ie. 23) [mandatory]
2404 * 9. Xref: line (ie. alt.test:389) [optional]
2405 *
2406 * TODO: as we don't use the original data, we currently can't store
2407 * the data (from/subject) in the original charset (we don't store
2408 * that info). this has the advantage that we can avoid raw 8bit data
2409 * in our overviews, but the disadvantage that we might store the data
2410 * with a wrong charset and thus lose information. a similar problem
2411 * exists with the data for the from:-line, we don't store it in the
2412 * original format, whenever our from-parser (partially) fails we'll
2413 * lose information in our overviews (but those couldn't be handled
2414 * by tin anyway, so this is not a real problem).
2415 * long-term solution: store the original data in the overview
2416 * (tin has to handle raw 8bit data and other ugly stuff in the
2417 * overviews anyway and thus we preserver as much info as possible)
2418 * this would require some changes in read_overview() and
2419 * parse_headers(): don't do the decoding/unfolding there, but in a
2420 * second pass right after write_overview(), or two additional fields
2421 * which hold the raw data for from/subject. the latter has the
2422 * disadvantage that it costs (much) more memory.
2423 */
2424 static void
2425 write_overview(
2426 struct t_group *group)
2427 {
2428 FILE *fp;
2429 int i;
2430 struct t_article *article;
2431 #ifdef CHARSET_CONVERSION
2432 int c = -1;
2433 #endif /* CHARSET_CONVERSION */
2434
2435 /*
2436 * Can't write or caching is off or getart_limit is set
2437 */
2438 if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
2439 return;
2440
2441 if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2442 return;
2443
2444 if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
2445 SortBy(artnum_comp);
2446
2447 /*
2448 * Needed to preserve uniqueness in hashed private overview files
2449 */
2450 fprintf(fp, "%s\n", group->name);
2451
2452 #ifdef CHARSET_CONVERSION
2453 /* get undeclared_charset number if required */
2454 if (group->attribute->undeclared_charset) {
2455 for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2456 if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2457 c = i;
2458 break;
2459 }
2460 }
2461 }
2462 #endif /* CHARSET_CONVERSION */
2463
2464 for_each_art(i) {
2465 char *p;
2466 char *q, *ref;
2467
2468 article = &arts[i];
2469
2470 if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2471 ref = NULL;
2472
2473 if (!group->attribute->post_8bit_header) { /* write encoded data */
2474 /*
2475 * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2476 * here and in print_from() in the CHARSET_CONVERSION case.
2477 * note that this requires something like
2478 * buffer_to_network(article->subject, "UTF-8");
2479 * right before the rfc1522_encode() call.
2480 *
2481 * if we would cache the original undecoded data, we could
2482 * ignore stuff like this.
2483 */
2484 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
2485 /* as the subject might now be folded we have to unfold it */
2486 unfold_header(p);
2487 } else { /* raw data */
2488 p = my_strdup(article->subject);
2489 #ifdef CHARSET_CONVERSION
2490 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2491 buffer_to_network(p, c);
2492 #endif /* CHARSET_CONVERSION */
2493 }
2494
2495 /*
2496 * replace any '\t's with ' ' in the references-data
2497 *
2498 * TODO: nntpext-draft might come up with a new scheme:
2499 * For all fields, the value is processed by first
2500 * removing all US-ASCII CRLF pairs and then replacing
2501 * each remaining US-ASCII NUL, TAB, CR, or LF character
2502 * with a single US-ASCII space (for example, CR LF LF TAB
2503 * will become two spaces).
2504 */
2505 if (article->refs) {
2506 ref = q = my_strdup(article->refs);
2507 while (*q) {
2508 if (*q == '\t')
2509 *q = ' ';
2510 q++;
2511 }
2512 }
2513
2514 fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2515 article->artnum,
2516 p,
2517 #ifdef CHARSET_CONVERSION
2518 print_from(group, article, c),
2519 #else
2520 print_from(group, article, -1),
2521 #endif /* CHARSET_CONVERSION */
2522 print_date(article->date),
2523 BlankIfNull(article->msgid),
2524 BlankIfNull(ref),
2525 0, /* bytes */
2526 article->line_count);
2527
2528 if (article->xref)
2529 fprintf(fp, "\tXref: %s", article->xref);
2530
2531 if (article->path)
2532 fprintf(fp, "\tPath: %s", article->path);
2533
2534 fprintf(fp, "\n");
2535 free(p);
2536 if (article->refs) {
2537 FreeIfNeeded(ref);
2538 q = NULL;
2539 }
2540 }
2541 }
2542 fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2543 fclose(fp);
2544 }
2545
2546
2547 /*
2548 * A complex little function to determine the correct overview index file
2549 * according to 'mode' (read or write)
2550 * NULL is returned if the current setup dictates otherwise
2551 *
2552 * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2553 * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2554 *
2555 * Both of these are hashed
2556 *
2557 * GROUP_TYPE_NEWS index files are a little bit more complex
2558 *
2559 * When hashing the index filename will be in format number.number.
2560 * Hashing the groupname gets a number. See if that #.1 file exists;
2561 * if so, read first line. Is this the group we want? If no, try #.2.
2562 * Repeat until no such file or we find an existing file that matches
2563 * our group. Return pointer to path or NULL if not found.
2564 */
2565 static char *
2566 find_nov_file(
2567 struct t_group *group,
2568 int mode)
2569 {
2570 FILE *fp;
2571 const char *dir;
2572 char buf[PATH_LEN];
2573 int i;
2574 struct stat sb;
2575 unsigned long hash;
2576 static char nov_file[PATH_LEN];
2577 static t_bool once_only = FALSE; /* Trap things that are done only 1 time */
2578
2579 if (group == NULL || (mode != R_OK && mode != W_OK))
2580 return NULL;
2581
2582 switch (group->type) {
2583 case GROUP_TYPE_MAIL:
2584 dir = index_maildir;
2585 break;
2586
2587 case GROUP_TYPE_SAVE:
2588 dir = index_savedir;
2589 break;
2590
2591 case GROUP_TYPE_NEWS:
2592 /*
2593 * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2594 * about [X]OVER are handled in nntp_open()
2595 */
2596
2597 /*
2598 * When reading via NNTP, system wide overviews are irrelevant, of
2599 * course, and the private overview filename will be the same for
2600 * both reading and writing.
2601 *
2602 * When working locally, we only use a private cache for reading
2603 * if requested and when system wide overviews don't already exist.
2604 * When writing then only private overviews can be used since
2605 * updating system wide overviews is not safe wrt locking etc.
2606 *
2607 * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2608 */
2609 #ifndef NNTP_ONLY
2610 if (!read_news_via_nntp) {
2611 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2612 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2613 if (access(nov_file, R_OK) == 0) {
2614 if (mode == R_OK)
2615 return nov_file; /* Use system wide overviews */
2616 else
2617 return NULL; /* Don't write cache in this case */
2618 }
2619 }
2620 #endif /* !NNTP_ONLY */
2621
2622 /*
2623 * We only get here when private overviews are going to be used
2624 * Go no further if they are explicitly turned off
2625 */
2626 if (!tinrc.cache_overview_files)
2627 return NULL;
2628
2629 /*
2630 * Append -<nntpserver> to private cache dir
2631 */
2632 if (!once_only && nntp_server) {
2633 size_t sp, ln = strlen(index_newsdir);
2634
2635 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
2636 char *srv = my_strdup(nntp_server);
2637
2638 strcat(index_newsdir, "-");
2639 sp--;
2640 ln++;
2641 str_lwr(srv);
2642 my_strncpy(index_newsdir + ln, srv, sp);
2643 free(srv);
2644 }
2645 once_only = TRUE;
2646 }
2647
2648 /*
2649 * Only try to set up the private cache when writing. If it
2650 * doesn't exist yet, then ergo we can't read from it.
2651 * The cache will be checked/created on every write; a previous
2652 * bug report complained that this was not the case
2653 */
2654 if (stat(index_newsdir, &sb) == -1) { /* Private cache doesn't exist */
2655 if (mode == R_OK)
2656 return NULL;
2657 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2658 return NULL;
2659 } else {
2660 if (!S_ISDIR(sb.st_mode))
2661 return NULL;
2662 }
2663
2664 /*
2665 * Update the newsgroups cache to point to the new location
2666 * now that we know it is valid
2667 */
2668 if (!once_only)
2669 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
2670
2671 dir = index_newsdir;
2672 break;
2673
2674 default: /* not reached */
2675 return NULL;
2676 }
2677
2678 /*
2679 * We only get here if writing to a private overview.
2680 * These always have hashed filenames.
2681 * Try <hash>.<seqno> and check the group name tagline until
2682 * matching index file is found. If not found return next unused
2683 * filename
2684 */
2685 hash = hash_groupname(group->name);
2686
2687 for (i = 1; ; i++) {
2688 char *ptr;
2689
2690 snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2691 joinpath(nov_file, sizeof(nov_file), dir, buf);
2692
2693 if ((fp = fopen(nov_file, "r")) == NULL)
2694 break;
2695
2696 /*
2697 * No group name header, so not a valid index file => overwrite it
2698 */
2699 if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2700 fclose(fp);
2701 break;
2702 }
2703 fclose(fp);
2704
2705 if ((ptr = strrchr(buf, '\n')) != NULL)
2706 *ptr = '\0';
2707
2708 if (strcmp(buf, group->name) == 0)
2709 break;
2710 }
2711
2712 return nov_file;
2713 }
2714
2715
2716 /*
2717 * Run the index file updater only for the groups we've loaded.
2718 */
2719 void
2720 do_update(
2721 t_bool catchup)
2722 {
2723 int i, j, k = 0;
2724 time_t beg_epoch = 0;
2725 struct t_article *art;
2726 struct t_group *group;
2727
2728 if (verbose)
2729 (void) time(&beg_epoch);
2730
2731 /*
2732 * loop through groups and update any required index files
2733 */
2734 for (i = 0; i < selmenu.max; i++) {
2735 group = &active[my_group[i]];
2736 /*
2737 * FIXME: workaround to get a valid CURR_GROUP
2738 * it also points to the currently processed group so that
2739 * the correct attributes are used
2740 * The correct fix is to get rid of CURR_GROUP
2741 */
2742 selmenu.curr = i;
2743
2744 if (group->bogus || !group->subscribed)
2745 continue;
2746
2747 if (!index_group(group)) {
2748 for_each_art(j) {
2749 art = &arts[j];
2750 FreeAndNull(art->refs);
2751 FreeAndNull(art->msgid);
2752 }
2753 continue;
2754 }
2755
2756 k++;
2757
2758 if (verbose) {
2759 my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2760 my_flush();
2761 }
2762
2763 if (catchup) {
2764 for_each_art(j)
2765 art_mark(group, &arts[j], ART_READ);
2766 }
2767 }
2768
2769 if (verbose) {
2770 wait_message(0, _(txt_catchup_update_info),
2771 (catchup ? _(txt_caughtup) : _(txt_updated)), k,
2772 PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2773 }
2774 }
2775
2776
2777 static int
2778 artnum_comp(
2779 t_comptype p1,
2780 t_comptype p2)
2781 {
2782 const struct t_article *s1 = (const struct t_article *) p1;
2783 const struct t_article *s2 = (const struct t_article *) p2;
2784
2785 /*
2786 * s1->artnum less than s2->artnum
2787 */
2788 if (s1->artnum < s2->artnum)
2789 return -1;
2790
2791 /*
2792 * s1->artnum greater than s2->artnum
2793 */
2794 if (s1->artnum > s2->artnum)
2795 return 1;
2796
2797 return 0;
2798 }
2799
2800
2801 /*
2802 * return result of strcmp (reversed for descending)
2803 */
2804 static int
2805 subj_comp_asc(
2806 t_comptype p1,
2807 t_comptype p2)
2808 {
2809 int retval;
2810 const struct t_article *s1 = (const struct t_article *) p1;
2811 const struct t_article *s2 = (const struct t_article *) p2;
2812
2813 if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2814 return retval;
2815
2816 return s1->date - s2->date > 0 ? 1 : -1;
2817 }
2818
2819
2820 static int
2821 subj_comp_desc(
2822 t_comptype p1,
2823 t_comptype p2)
2824 {
2825 int retval;
2826 const struct t_article *s1 = (const struct t_article *) p1;
2827 const struct t_article *s2 = (const struct t_article *) p2;
2828
2829 if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2830 return retval;
2831
2832 return s1->date - s2->date > 0 ? 1 : -1;
2833 }
2834
2835
2836 /*
2837 * return result of strcmp (reversed for descending)
2838 */
2839 static int
2840 from_comp_asc(
2841 t_comptype p1,
2842 t_comptype p2)
2843 {
2844 int retval;
2845 const struct t_article *s1 = (const struct t_article *) p1;
2846 const struct t_article *s2 = (const struct t_article *) p2;
2847
2848 if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2849 return retval;
2850
2851 return s1->date - s2->date > 0 ? 1 : -1;
2852 }
2853
2854
2855 static int
2856 from_comp_desc(
2857 t_comptype p1,
2858 t_comptype p2)
2859 {
2860 int retval;
2861 const struct t_article *s1 = (const struct t_article *) p1;
2862 const struct t_article *s2 = (const struct t_article *) p2;
2863
2864 if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2865 return retval;
2866
2867 return s1->date - s2->date > 0 ? 1 : -1;
2868 }
2869
2870
2871 /*
2872 * Works like strcmp() for comparing time_t type values
2873 * Return codes:
2874 * -1: If p1 is before p2
2875 * 0: If they are the same time
2876 * 1: If p1 is after p2
2877 * If the sort order is _not_ DATE_ASCEND then the sense of the above
2878 * is reversed.
2879 */
2880 static int
2881 date_comp_asc(
2882 t_comptype p1,
2883 t_comptype p2)
2884 {
2885 const struct t_article *s1 = (const struct t_article *) p1;
2886 const struct t_article *s2 = (const struct t_article *) p2;
2887
2888 /*
2889 * s1->date less than s2->date
2890 */
2891 if (s1->date < s2->date)
2892 return -1;
2893
2894 /*
2895 * s1->date greater than s2->date
2896 */
2897 if (s1->date > s2->date)
2898 return 1;
2899
2900 return 0;
2901 }
2902
2903
2904 static int
2905 date_comp_desc(
2906 t_comptype p1,
2907 t_comptype p2)
2908 {
2909 const struct t_article *s1 = (const struct t_article *) p1;
2910 const struct t_article *s2 = (const struct t_article *) p2;
2911
2912 /*
2913 * s2->date less than s1->date
2914 */
2915 if (s2->date < s1->date)
2916 return -1;
2917
2918 /*
2919 * s2->date greater than s1->date
2920 */
2921 if (s2->date > s1->date)
2922 return 1;
2923
2924 return 0;
2925 }
2926
2927
2928 /*
2929 * Same again, but for art[].score
2930 */
2931 static int
2932 score_comp_asc(
2933 t_comptype p1,
2934 t_comptype p2)
2935 {
2936 const struct t_article *s1 = (const struct t_article *) p1;
2937 const struct t_article *s2 = (const struct t_article *) p2;
2938
2939 if (s1->score < s2->score)
2940 return -1;
2941
2942 if (s1->score > s2->score)
2943 return 1;
2944
2945 return s1->date - s2->date > 0 ? 1 : -1;
2946 }
2947
2948
2949 static int
2950 score_comp_desc(
2951 t_comptype p1,
2952 t_comptype p2)
2953 {
2954 const struct t_article *s1 = (const struct t_article *) p1;
2955 const struct t_article *s2 = (const struct t_article *) p2;
2956
2957 if (s2->score < s1->score)
2958 return -1;
2959
2960 if (s2->score > s1->score)
2961 return 1;
2962
2963 return s1->date - s2->date > 0 ? 1 : -1;
2964 }
2965
2966
2967 /*
2968 * Same again, but for art[].line_count
2969 */
2970 static int
2971 lines_comp_asc(
2972 t_comptype p1,
2973 t_comptype p2)
2974 {
2975 const struct t_article *s1 = (const struct t_article *) p1;
2976 const struct t_article *s2 = (const struct t_article *) p2;
2977
2978 if (s1->line_count < s2->line_count)
2979 return -1;
2980
2981 if (s1->line_count > s2->line_count)
2982 return 1;
2983
2984 return s1->date - s2->date > 0 ? 1 : -1;
2985 }
2986
2987
2988 static int
2989 lines_comp_desc(
2990 t_comptype p1,
2991 t_comptype p2)
2992 {
2993 const struct t_article *s1 = (const struct t_article *) p1;
2994 const struct t_article *s2 = (const struct t_article *) p2;
2995
2996 if (s2->line_count < s1->line_count)
2997 return -1;
2998
2999 if (s2->line_count > s1->line_count)
3000 return 1;
3001
3002 return s1->date - s2->date > 0 ? 1 : -1;
3003 }
3004
3005
3006 /*
3007 * Compares the total score of two threads. Used for sorting base[].
3008 */
3009 static int
3010 score_comp_base(
3011 t_comptype p1,
3012 t_comptype p2)
3013 {
3014 int a = get_score_of_thread(*(const long *) p1);
3015 int b = get_score_of_thread(*(const long *) p2);
3016
3017 /* If scores are equal, compare using the article sort order.
3018 * This determines the order in a group of equally scored threads.
3019 */
3020 if (a == b) {
3021 const struct t_article *s1 = &arts[*(const long *) p1];
3022 const struct t_article *s2 = &arts[*(const long *) p2];
3023 t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3024
3025 if (comp_func)
3026 return (*comp_func)(s1, s2);
3027 return 0;
3028 }
3029
3030 if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3031 return a > b ? 1 : -1;
3032 return a < b ? 1 : -1;
3033 }
3034
3035
3036 /*
3037 * Compare the date of the last posted article of two threads.
3038 * Used for sorting base[].
3039 */
3040 static int
3041 last_date_comp_base_desc(
3042 t_comptype p1,
3043 t_comptype p2)
3044 {
3045 time_t s1_last = get_last_posting_date(*(const long *) p1);
3046 time_t s2_last = get_last_posting_date(*(const long *) p2);
3047
3048 if (s2_last < s1_last)
3049 return -1;
3050
3051 if (s2_last > s1_last)
3052 return 1;
3053
3054 return 0;
3055 }
3056
3057
3058 static int
3059 last_date_comp_base_asc(
3060 t_comptype p1,
3061 t_comptype p2)
3062 {
3063 time_t s1_last = get_last_posting_date(*(const long *) p1);
3064 time_t s2_last = get_last_posting_date(*(const long *) p2);
3065
3066 if (s2_last > s1_last)
3067 return -1;
3068
3069 if (s2_last < s1_last)
3070 return 1;
3071
3072 return 0;
3073 }
3074
3075
3076 static time_t
3077 get_last_posting_date(
3078 long n)
3079 {
3080 long i;
3081 time_t last = (time_t) 0;
3082
3083 for (i = n; i >= 0; i = arts[i].thread) {
3084 if (arts[i].date > last)
3085 last = arts[i].date;
3086 }
3087
3088 return last;
3089 }
3090
3091
3092 void
3093 set_article(
3094 struct t_article *art)
3095 {
3096 art->subject = NULL;
3097 art->from = NULL;
3098 art->name = NULL;
3099 art->date = (time_t) 0;
3100 art->xref = NULL;
3101 art->msgid = NULL;
3102 art->refs = NULL;
3103 art->refptr = NULL;
3104 art->line_count = -1;
3105 art->archive = NULL;
3106 art->tagged = 0;
3107 art->thread = ART_EXPIRED;
3108 art->prev = ART_NORMAL;
3109 art->score = 0;
3110 art->status = ART_UNREAD;
3111 art->killed = ART_NOTKILLED;
3112 art->zombie = FALSE;
3113 art->delete_it = FALSE;
3114 art->selected = FALSE;
3115 art->inrange = FALSE;
3116 art->matched = FALSE;
3117 art->keep_in_base = FALSE;
3118 }
3119
3120
3121 /*
3122 * Do a binary chop to see if 'art' (an article number) exists in arts[]
3123 * Naturally arts[] must be sorted on artnum
3124 * Return index into arts[] or -1
3125 */
3126 static int
3127 valid_artnum(
3128 t_artnum art)
3129 {
3130 int prev, range;
3131 int dctop = top_art;
3132 int cur = 1;
3133
3134 while ((dctop >>= 1))
3135 cur <<= 1;
3136
3137 range = cur >> 1;
3138 cur--;
3139
3140 forever {
3141 if (arts[cur].artnum == art)
3142 return cur;
3143
3144 prev = cur;
3145 cur += ((arts[cur].artnum < art) ? range : -range);
3146 if (prev == cur)
3147 break;
3148
3149 if (cur >= top_art)
3150 cur = top_art - 1;
3151
3152 range >>= 1;
3153 }
3154 return -1;
3155 }
3156
3157
3158 /*
3159 * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3160 * Needed if arts[] is not sorted on artnum
3161 * Return index into arts[] or -1
3162 */
3163 int
3164 find_artnum(
3165 t_artnum art)
3166 {
3167 int i;
3168
3169 for_each_art(i) {
3170 if (arts[i].artnum == art)
3171 return i;
3172 }
3173 return -1;
3174 }
3175
3176
3177 /*----------------------------- Overview handling -----------------------*/
3178 /* TODO: use
3179 * setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3180 * my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3181 * instead?
3182 */
3183 static char *
3184 print_date(
3185 time_t secs)
3186 {
3187 static char date[25];
3188 struct tm *tm;
3189 static const char *const months_a[] = {
3190 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3191 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3192 };
3193
3194 if ((tm = gmtime(&secs)) != NULL)
3195 snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3196 tm->tm_mday,
3197 months_a[tm->tm_mon],
3198 tm->tm_year + 1900,
3199 tm->tm_hour, tm->tm_min, tm->tm_sec);
3200 else
3201 snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3202
3203 return date;
3204 }
3205
3206
3207 static char *
3208 print_from(
3209 struct t_group *group,
3210 struct t_article *article,
3211 int charset)
3212 {
3213 char *p, *q;
3214 static char from[PATH_LEN];
3215
3216 *from = '\0';
3217
3218 if (article->name != NULL) {
3219 q = my_strdup(article->name);
3220 #ifdef CHARSET_CONVERSION
3221 if (charset != -1) {
3222 buffer_to_network(q, charset);
3223 }
3224 #endif /* CHARSET_CONVERSION */
3225 p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
3226 unfold_header(p);
3227 if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3228 snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3229 else
3230 snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3231
3232 free(p);
3233 free(q);
3234 } else
3235 STRCPY(from, article->from);
3236
3237 return from;
3238 }
3239
3240
3241 /*
3242 * Open a group news overview file
3243 * Use NNTP XOVER where possible unless 'local' is set
3244 */
3245 static FILE *
3246 open_xover_fp(
3247 struct t_group *group,
3248 const char *mode,
3249 t_artnum min,
3250 t_artnum max,
3251 t_bool local)
3252 {
3253 #ifdef NNTP_ABLE
3254 if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3255 char line[NNTP_STRLEN];
3256
3257 if (!max)
3258 return NULL;
3259 if (min == max)
3260 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3261 else
3262 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3263 return (nntp_command(line, OK_XOVER, NULL, 0));
3264 }
3265 #endif /* NNTP_ABLE */
3266 {
3267 FILE *fp;
3268 char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3269
3270 if (nov_file != NULL) {
3271 if ((fp = fopen(nov_file, mode)) != NULL)
3272 return fp;
3273
3274 if (*mode != 'r')
3275 error_message(2, _(txt_cannot_open), nov_file);
3276 }
3277 }
3278 return NULL;
3279 }
3280
3281
3282 #ifdef USE_HEAPSORT
3283 int
3284 tin_sort(
3285 void *sbase,
3286 size_t nel,
3287 size_t width,
3288 t_compfunc compar)
3289 {
3290 int rc;
3291
3292 switch (tinrc.sort_function) {
3293 case 0:
3294 qsort(sbase, nel, width, compar);
3295 rc = 0;
3296 break;
3297
3298 case 1:
3299 rc = heapsort(sbase, nel, width, compar);
3300 break;
3301
3302 default:
3303 rc = -1;
3304 break;
3305 }
3306 return rc;
3307 }
3308 #endif /* USE_HEAPSORT */