"Fossies" - the Fresh Open Source Software Archive 
Member "tin-2.6.1/src/art.c" (22 Dec 2021, 89594 Bytes) of package /linux/misc/tin-2.6.1.tar.xz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "art.c" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
2.6.0_vs_2.6.1.
1 /*
2 * Project : tin - a Usenet reader
3 * Module : art.c
4 * Author : I.Lea & R.Skrenta
5 * Created : 1991-04-01
6 * Updated : 2021-04-10
7 * Notes :
8 *
9 * Copyright (c) 1991-2022 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10 * All rights reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * 1. Redistributions of source code must retain the above copyright notice,
17 * this list of conditions and the following disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 *
23 * 3. Neither the name of the copyright holder nor the names of its
24 * contributors may be used to endorse or promote products derived from
25 * this software without specific prior written permission.
26 *
27 * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
31 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40
41 #ifndef TIN_H
42 # include "tin.h"
43 #endif /* !TIN_H */
44 #ifndef TCURSES_H
45 # include "tcurses.h"
46 #endif /* !TCURSES_H */
47 #ifndef NEWSRC_H
48 # include "newsrc.h"
49 #endif /* !NEWSRC_H */
50
51 #ifndef STPWATCH_H
52 # include "stpwatch.h"
53 #endif /* !STPWATCH_H */
54
55 /*
56 * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
57 */
58 #define SortBy(func) tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
59
60 int top_art = 0; /* # of articles in arts[] */
61
62 /*
63 * Local prototypes
64 */
65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
67 static char *find_nov_file(struct t_group *group, int mode);
68 static char *print_date(time_t secs);
69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
70 static int artnum_comp(t_comptype p1, t_comptype p2);
71 static int base_comp(t_comptype p1, t_comptype p2);
72 static int date_comp_asc(t_comptype p1, t_comptype p2);
73 static int date_comp_desc(t_comptype p1, t_comptype p2);
74 static int from_comp_asc(t_comptype p1, t_comptype p2);
75 static int from_comp_desc(t_comptype p1, t_comptype p2);
76 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
77 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
78 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
79 static int lines_comp_asc(t_comptype p1, t_comptype p2);
80 static int lines_comp_desc(t_comptype p1, t_comptype p2);
81 static int read_art_headers(struct t_group *group, int total, t_artnum top);
82 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
83 static int score_comp_asc(t_comptype p1, t_comptype p2);
84 static int score_comp_desc(t_comptype p1, t_comptype p2);
85 static int score_comp_base(t_comptype p1, t_comptype p2);
86 static int subj_comp_asc(t_comptype p1, t_comptype p2);
87 static int subj_comp_desc(t_comptype p1, t_comptype p2);
88 static int valid_artnum(t_artnum art);
89 static t_artnum find_first_unread(struct t_group *group);
90 static t_artnum setup_hard_base(struct t_group *group);
91 static t_bool parse_headers(FILE *fp, struct t_article *h);
92 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
93 static time_t get_last_posting_date(long n);
94 static void sort_base(unsigned int sort_threads_type);
95 static void thread_by_multipart(void);
96 static void thread_by_percentage(unsigned int percentage);
97 static void thread_by_subject(void);
98 static void write_overview(struct t_group *group);
99 #ifdef NNTP_ABLE
100 static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
101 static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
102 #endif /* NNTP_ABLE */
103
104
105 /*
106 * Display a suitable 'entering group' message if screen needs redrawing
107 * Allow for the non-printing %s, and the %-age counter
108 */
109 void
110 show_art_msg(
111 const char *group)
112 {
113 wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
114 }
115
116
117 /*
118 * Construct the pointers to the first (base) article in each thread.
119 * If we are showing only unread, then point to the first unread. I have
120 * no idea why this should be so, it causes problems elsewhere [which_response]
121 */
122 void
123 find_base(
124 struct t_group *group)
125 {
126 int i, j;
127
128 grpmenu.max = 0;
129
130 #ifdef DEBUG
131 if (debug & DEBUG_FILTER)
132 debug_print_arts();
133 #endif /* DEBUG */
134
135 for_each_art(i) {
136 /*
137 * .prev will be set on each article that is after the first article in
138 * the thread. Invalid articles which have been expired will have
139 * .thread set to ART_EXPIRED
140 */
141 if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
142 continue;
143
144 if (grpmenu.max >= max_base)
145 expand_base();
146
147 if (group->attribute->show_only_unread_arts) {
148 if (arts[i].status != ART_READ || arts[i].keep_in_base)
149 base[grpmenu.max++] = i;
150 else {
151 /* Find 1st unread art in thread */
152 for (j = i; j >= 0; j = arts[j].thread) {
153 if (arts[j].status != ART_READ || arts[j].keep_in_base) {
154 base[grpmenu.max++] = i;
155 break;
156 }
157 }
158 }
159 } else
160 base[grpmenu.max++] = i;
161 }
162 /* sort base[] */
163 if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
164 sort_base(group->attribute->sort_threads_type);
165 }
166
167
168 /*
169 * Longword comparison routine for the tin_sort()
170 */
171 static int
172 base_comp(
173 t_comptype p1,
174 t_comptype p2)
175 {
176 const t_artnum *a = (const t_artnum *) p1;
177 const t_artnum *b = (const t_artnum *) p2;
178
179 if (*a < *b)
180 return -1;
181
182 if (*a > *b)
183 return 1;
184
185 return 0;
186 }
187
188
189 /*
190 * via NNTP:
191 * Issue a LISTGROUP command
192 * Read the article numbers existing in the group into base[]
193 * If the LISTGROUP failed, issue a GROUP command. Use the results to
194 * create a less accurate version of base[]
195 * This data will already be sorted
196 *
197 * on local spool:
198 * Read the spool dir to populate base[] as above. Sort it.
199 *
200 * Grow the arts[] and bitmaps as needed.
201 * NB: the output will be sorted on artnum
202 *
203 * grpmenu.max is one past top.
204 * Returns total number of articles in group, or -1 on error
205 */
206 static t_artnum
207 setup_hard_base(
208 struct t_group *group)
209 {
210 t_artnum total = 0;
211
212 grpmenu.max = 0;
213
214 /*
215 * If reading with NNTP, issue a LISTGROUP
216 */
217 if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
218 #ifdef NNTP_ABLE
219 char buf[NNTP_STRLEN];
220 char line[NNTP_STRLEN];
221 int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
222 FILE *fp;
223 t_artnum last, start, count = 0, j = 0;
224 static t_bool skip_listgroup = FALSE;
225
226 /*
227 * Some nntp servers are broken and need an extra GROUP command
228 * (reported by reorx@irc.pl). This affects (old?) versions of
229 * nntpcache, leafnode and SurgeNews. Usually this should not be
230 * needed.
231 *
232 * For getart_limit recheck lowwatermark as at least giganews gives
233 * very different results for LIST ACTIVE (3 year retention for all)
234 * and GROUP (based on the clients contract).
235 * Calculate range and prepare base[] not to lose unread arts.
236 */
237 if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
238 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
239 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
240 return -1;
241
242 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
243 return -1;
244
245 if (getart_limit > 0) {
246 j = group->xmax - getart_limit;
247 count = MAX(find_first_unread(group), start);
248 }
249 if (getart_limit < 0) {
250 j = getart_limit + find_first_unread(group);
251 count = group->xmin;
252 }
253 if (j < group->xmin)
254 j = group->xmin;
255
256 for (; count < j; count++) {
257 if (grpmenu.max >= max_base)
258 expand_base();
259 base[grpmenu.max++] = count;
260 }
261 }
262
263 /*
264 * See if LISTGROUP works
265 */
266 if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
267 if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
268 /* RFC 3977 allows ranges in LISTGROUP */
269 if (getart_limit > 0)
270 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
271 else /* getart_limit < 0; fetch till newest art */
272 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
273
274 } else /* for RFC 977 just use GROUP */
275 skip_listgroup = TRUE;
276
277 } else /* get all article numbers */
278 snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
279
280 if (!skip_listgroup) {
281 if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
282 char *ptr;
283
284 # ifdef DEBUG
285 if ((debug & DEBUG_NNTP) && verbose > 1)
286 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
287 # endif /* DEBUG */
288
289 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
290 if (grpmenu.max >= max_base)
291 expand_base();
292 base[grpmenu.max++] = atoartnum(ptr);
293 total++;
294 }
295
296 if (tin_errno)
297 return -1;
298 } else
299 skip_listgroup = TRUE;
300 }
301
302 if (skip_listgroup) { /* LISTGROUP was skipped or failed */
303 /*
304 * Handle the obscure case that the user aborted before the LISTGROUP
305 * had a chance to respond
306 */
307 if (tin_errno)
308 return -1;
309
310 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
311 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
312 return -1;
313
314 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
315 return -1;
316
317 # ifdef DEBUG
318 if ((debug & DEBUG_NNTP) && verbose > 1)
319 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
320 # endif /* DEBUG */
321 total = count;
322 grpmenu.max = 0;
323 if (getart_limit > 0) {
324 if ((j = find_first_unread(group)) > start) {
325 if (group->xmax > getart_limit) {
326 start = MIN(j, group->xmax - getart_limit);
327 total = getart_limit;
328 } else
329 start = j;
330 }
331 }
332 if (getart_limit < 0) {
333 if ((j = (getart_limit + find_first_unread(group))) > start)
334 start = j;
335 }
336 while (start <= last) {
337 if (grpmenu.max >= max_base)
338 expand_base();
339 base[grpmenu.max++] = start++;
340 }
341 }
342 #endif /* NNTP_ABLE */
343 /*
344 * Reading off local spool, read the directory files
345 */
346 } else {
347 DIR *d;
348 DIR_BUF *e;
349 char group_path[PATH_LEN];
350 t_artnum art;
351
352 make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
353
354 if ((d = opendir(group_path)) != NULL) {
355 while ((e = readdir(d)) != NULL) {
356 art = atoartnum(e->d_name);
357 if (art >= 1) {
358 total++;
359 if (grpmenu.max >= max_base)
360 expand_base();
361 base[grpmenu.max++] = art;
362 }
363 }
364 CLOSEDIR(d);
365 tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
366 } else {
367 perror_message(_(txt_cannot_open), group_path);
368 #if 0
369 if (access(group_path, R_OK) != 0)
370 error_message(2, _(txt_not_exist));
371 #endif /* 0 */
372 return -1;
373 }
374 }
375
376 if (grpmenu.max) {
377 if (base[grpmenu.max - 1] > group->xmax)
378 group->xmax = base[grpmenu.max - 1];
379 expand_bitmap(group, base[0]);
380 }
381
382 return total;
383 }
384
385
386 /*
387 * Main group indexing routine.
388 *
389 * Will read any existing index, create or incrementally update
390 * the index by looking at the articles in the spool directory,
391 * and attempt to write a new index if necessary.
392 *
393 * Returns FALSE if the user aborted the indexing, otherwise TRUE
394 */
395 t_bool
396 index_group(
397 struct t_group *group)
398 {
399 int i;
400 int changed; /* Count of articles whose overview has changed */
401 int getart_limit;
402 int respnum;
403 int total;
404 t_artnum last_read_article;
405 t_artnum min, new_min, max;
406 t_bool caching_xover;
407 t_bool filtered;
408 t_bool path_in_nov = FALSE;
409 t_bool rebuild_cache = FALSE;
410
411 if (group == NULL)
412 return TRUE;
413
414 if (!batch_mode)
415 show_art_msg(group->name);
416 else {
417 if (verbose) /* -> lang.c */
418 wait_message(0, _("Reading %s\n"), group->name);
419 }
420
421 signal_context = cArt; /* Set this only once curr_group is valid */
422
423 hash_reclaim();
424 free_art_array();
425 free_msgids();
426
427 BegStopWatch("setup_hard_base()");
428
429 /*
430 * Get list of valid article numbers
431 */
432 if (setup_hard_base(group) < 0)
433 return FALSE;
434
435 EndStopWatch();
436 PrintStopWatch();
437
438 #ifdef DEBUG
439 if (debug & DEBUG_NEWSRC) {
440 debug_print_comment("Before read_overview");
441 debug_print_bitmap(group, NULL);
442 }
443 #endif /* DEBUG */
444
445 min = grpmenu.max ? base[0] : group->xmin;
446 max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
447
448 getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
449
450 if (getart_limit > 0) {
451 if (grpmenu.max && (grpmenu.max > getart_limit))
452 min = base[grpmenu.max - getart_limit];
453 else
454 getart_limit = 0;
455 } else if (getart_limit < 0) {
456 t_artnum first_unread = find_first_unread(group);
457
458 if (min - first_unread < getart_limit)
459 min = first_unread + getart_limit;
460 else
461 getart_limit = 0;
462 }
463
464 /*
465 * Quit now if no articles
466 */
467 if (max < 0)
468 return FALSE;
469
470 top_art = 0;
471 last_read_article = T_ARTNUM_CONST(0);
472
473 /*
474 * Read in the existing overview data for min..max
475 * This read has local=TRUE set if locally caching XOVER records to ensure
476 * we pull in any private overview caches in preference to using OVER
477 *
478 * When reading local spool, this will pull in the system wide overview
479 * cache (if found) otherwise the private overview cache will be read
480 */
481 caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
482 if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
483 return FALSE; /* user aborted indexing */
484
485 /*
486 * Fill in the range last_read_article...max using XOVER
487 * Only do this if the previous read_overview() was against private cache
488 */
489 if ((last_read_article < max) && caching_xover) {
490 new_min = (last_read_article >= min) ? last_read_article + 1 : min;
491
492 if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
493 return FALSE; /* user aborted indexing */
494 else
495 changed += i;
496 } else
497 caching_xover = FALSE;
498
499 /*
500 * Mark as UNTHREADED all articles that have been verified as valid
501 * Get num of new arts to index so the user will have an idea of index time
502 */
503 for (i = 0, total = 0; i < grpmenu.max; i++) {
504 if ((respnum = valid_artnum(base[i])) >= 0) {
505 arts[respnum].thread = ART_UNTHREADED;
506 continue;
507 }
508 if (base[i] <= last_read_article) /* It is vital this test be done last */
509 continue;
510 total++;
511 }
512
513 /*
514 * Add any articles to arts[] that are new or were killed
515 */
516 if (total > 0) {
517 new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
518
519 if ((i = read_art_headers(group, total, new_min)) == -1)
520 return FALSE; /* user aborted indexing */
521 else
522 changed += i;
523 }
524
525 #ifdef DEBUG
526 if (debug & DEBUG_NEWSRC) {
527 debug_print_comment("Before parse_unread_arts()");
528 debug_print_bitmap(group, NULL);
529 }
530 #endif /* DEBUG */
531 /*
532 * Do this before calling art_mark(,, ART_READ) if you want
533 * the unread count to be correct.
534 */
535 min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
536 parse_unread_arts(group, min);
537 #ifdef DEBUG
538 if (debug & DEBUG_NEWSRC) {
539 debug_print_comment("After parse_unread_arts()");
540 debug_print_bitmap(group, NULL);
541 }
542 #endif /* DEBUG */
543
544 /*
545 * Stat all articles to see if any have expired
546 */
547 for_each_art(i) {
548 if (arts[i].thread == ART_EXPIRED) {
549 changed++;
550 #ifdef DEBUG
551 if (debug & DEBUG_NEWSRC)
552 debug_print_comment("art.c: index_group() purging...");
553 #endif /* DEBUG */
554 art_mark(group, &arts[i], ART_READ);
555 if (group->attribute->show_only_unread_arts)
556 arts[i].keep_in_base = FALSE;
557 }
558 if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
559 path_in_nov = TRUE;
560 }
561
562 /*
563 * Only rewrite the index if it has changed
564 * TODO review the exact logic behind "|| caching_xover"
565 */
566 if (changed || caching_xover || rebuild_cache)
567 write_overview(group);
568
569 /*
570 * Create the reference tree. The msgid and ref ptrs will
571 * be free()d now that the NovFile has been written.
572 */
573 build_references(group);
574
575 /*
576 * Needs access to the reference tree
577 */
578 filtered = filter_articles(group);
579
580 BegStopWatch("make_threads()");
581
582 /*
583 * Thread the group
584 */
585 make_threads(group, FALSE);
586
587 EndStopWatch();
588 PrintStopWatch();
589
590 if ((changed > 0 || filtered) && !batch_mode)
591 clear_message();
592
593 return TRUE;
594 }
595
596
597 /*
598 * Returns number of first unread article
599 */
600 static t_artnum
601 find_first_unread(
602 struct t_group *group)
603 {
604 unsigned char *p;
605 unsigned char *end = group->newsrc.xbitmap;
606 t_artnum first = group->newsrc.xmin; /* initial value */
607
608 if ((p = group->newsrc.xbitmap)) {
609 end += group->newsrc.xbitlen / NBITS;
610 for (; p < end && *p == '\0'; p++, first += NBITS)
611 ;
612 }
613 return first;
614 }
615
616
617 /*
618 * Open an article for reading just the header
619 * 'next' is used/updated with the next article number
620 * to optimise the number of 'HEAD' commands issued on
621 * groups with holes.
622 */
623 static FILE *
624 open_art_header(
625 char *groupname,
626 t_artnum art,
627 t_artnum *next)
628 {
629 char buf[NNTP_STRLEN];
630 #ifdef NNTP_ABLE
631 FILE *fp;
632 int i;
633
634 if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
635 /*
636 * Don't bother requesting if we have not got there yet.
637 * This is a big win if the group has got holes in it (ie. if 000's
638 * of articles have expired between active files min & max values).
639 */
640 if (art < *next)
641 return NULL;
642
643 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
644 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
645 return fp;
646
647 /*
648 * TODO:
649 * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
650 * "503 Access denied" instead of 480 but NEXT still works,
651 * so tin loops over all articles without getting useful data
652 */
653
654 /*
655 * HEAD failed, try to find NEXT
656 * Should return "223 artno message-id more text...."
657 */
658 i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
659 switch (i) {
660 case OK_NOTEXT:
661 *next = atoartnum(buf); /* Set next art number */
662 break;
663
664 # ifndef BROKEN_LISTGROUP
665 /*
666 * might happen if LISTGROUP doesn't select group, but
667 * we are not -DBROKEN_LISTGROUP
668 */
669 case ERR_NCING:
670 nntp_caps.broken_listgroup = TRUE;
671 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
672 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
673 return NULL;
674 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
675 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
676 return fp;
677 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
678 *next = atoartnum(buf);
679 break;
680 # endif /* !BROKEN_LISTGROUP */
681
682 default:
683 /*
684 * TODO: abort loop over all arts on ERR_NONEXT
685 */
686 # ifndef BROKEN_LISTGROUP
687 /*
688 * to avoid out of sync responses
689 * (listgroup seems to work, but didn't select new group,
690 * so xover seems to work but returns old data)
691 * we set listgroup_broken = TRUE; once we saw a
692 * ERR_NOARTIG / ERR_NONEXT or the like - even if
693 * ERR_NOARTIG may occur on servers where listgroup
694 * isn't broken...
695 */
696 nntp_caps.broken_listgroup = TRUE;
697 # endif /* !BROKEN_LISTGROUP */
698 break;
699 }
700
701 return NULL;
702 }
703 #endif /* NNTP_ABLE */
704
705 snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
706 return (fopen(buf, "r"));
707 }
708
709
710 /*
711 * Called after XOVER/local/private overview databases have been loaded
712 * Read and parse in headers for any arts not already found (usually
713 * new articles that have not been indexed yet)
714 * Any new articles that are added have ->thread set to ART_UNTHREADED
715 * 'top' is the current highest artnum read
716 *
717 * Return values are:
718 * >0 Number of additional articles read in
719 * 0 No additional (new) articles were found
720 * -1 user aborted during read
721 */
722 static int
723 read_art_headers(
724 struct t_group *group,
725 int total,
726 t_artnum top)
727 {
728 FILE *fp;
729 char dir[PATH_LEN];
730 char group_msg[LEN];
731 int i;
732 int modified = 0;
733 t_artnum art;
734 t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
735 t_bool res;
736
737 /*
738 * Change to groups spooldir to optimize fopen()'s on local articles
739 * NB open_art_header() requires this
740 */
741 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
742 char buf[PATH_LEN];
743
744 get_cwd(dir);
745 make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
746 if (chdir(buf) != 0) {
747 #ifdef DEBUG
748 if (debug & DEBUG_MISC)
749 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
750 #endif /* DEBUG */
751 return -1;
752 }
753 }
754
755 snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
756
757 for (i = 0; i < grpmenu.max; i++) { /* for each article number */
758 art = base[i];
759
760 /*
761 * Skip articles that are below the low water mark or are
762 * already present
763 */
764 if (valid_artnum(art) >= 0)
765 continue;
766 if (art <= top)
767 continue;
768
769 /*
770 * Try and open the article
771 */
772 if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
773 continue;
774
775 /*
776 * Add article to arts[]
777 */
778 if (top_art >= max_art)
779 expand_art();
780
781 set_article(&arts[top_art]);
782 arts[top_art].artnum = art;
783 arts[top_art].thread = ART_UNTHREADED;
784
785 res = parse_headers(fp, &arts[top_art]);
786
787 TIN_FCLOSE(fp);
788 if (tin_errno) {
789 modified = -1;
790 break;
791 }
792
793 if (!res) {
794 #ifdef DEBUG
795 if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
796 char buf[PATH_LEN];
797
798 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
799 debug_print_file("ARTS", "read_art_headers() %s", buf);
800 }
801 #endif /* DEBUG */
802 arts[top_art].artnum = T_ARTNUM_CONST(0);
803 arts[top_art].date = (time_t) 0;
804 FreeAndNull(arts[top_art].xref);
805 FreeAndNull(arts[top_art].path);
806 FreeAndNull(arts[top_art].refs);
807 FreeAndNull(arts[top_art].msgid);
808 arts[top_art].tagged = 0;
809 arts[top_art].thread = ART_EXPIRED;
810 arts[top_art].prev = ART_NORMAL;
811 arts[top_art].status = ART_UNREAD;
812 arts[top_art].killed = ART_NOTKILLED;
813 arts[top_art].selected = FALSE;
814 continue;
815 }
816
817 top = arts[top_art].artnum; /* used if arts are killed */
818 top_art++;
819
820 if (++modified % (MODULO_COUNT_NUM * 20) == 0)
821 show_progress(group_msg, modified, total);
822 }
823
824 /*
825 * Change back to previous dir before indexing started
826 */
827 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
828 if (chdir(dir) == -1) {
829 #ifdef DEBUG
830 int e = errno;
831 if (debug & DEBUG_MISC)
832 error_message(2, "chdir(%s): Error: %s", dir, strerror(e));
833 #endif /* DEBUG */
834 }
835 }
836
837 return modified;
838 }
839
840
841 /*
842 * The algorithm is elegant, using the fact that identical Subject lines
843 * are hashed to the same node in table[] (see hashstr.c)
844 *
845 * Mark i as being in j's thread list if
846 * . The article is _not_ being ignored
847 * . The article is not already threaded
848 * . The subject lines are the same
849 */
850 static void
851 thread_by_subject(
852 void)
853 {
854 int i, j;
855 struct t_hashnode *h;
856
857 for_each_art(i) {
858 if (IGNORE_ART_THREAD(i))
859 continue;
860
861 /*
862 * Get the contents of the magic marker in the hashnode
863 */
864 h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
865
866 j = h->aptr;
867
868 if (j != -1 && j < i) {
869 if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject)) {
870 arts[j].thread = i;
871 arts[i].prev = j;
872 }
873 }
874
875 /*
876 * Update the magic marker with the highest numbered mesg in
877 * arts[] that has been used in this thread so far
878 */
879 h->aptr = i;
880 }
881
882 #if 0
883 fprintf(stderr, "Subj dump\n");
884 fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
885 for_each_art(i) {
886 fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
887 (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
888 (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
889 (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
890 arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
891 }
892 #endif /* 0 */
893 }
894
895
896 /*
897 * This Threading algorithm threads articles into 'buckets' where each bucket
898 * contains all the articles which match the root article's subject line to
899 * the configured percentage. Eg, if the root article had the subject "asdf"
900 * and the match percentage was configured to be 75% then any article would
901 * match if its subject was no different in more than a single character.
902 */
903 static void
904 thread_by_percentage(
905 unsigned int percentage)
906 {
907 int i, j, k;
908 int root_num = 0; /* The index number of the root we are currently working on. */
909 unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
910 size_t slen;
911
912 /* First we need to sort art[] to simplify and speed up the matching. */
913 SortBy(subj_comp_asc);
914
915 /*
916 * Now we put all the articles which match enough into the thread. If
917 * an article doesn't match enough we create a new thread and then add
918 * to that and so on.
919 */
920 base[0] = 0;
921 arts[0].prev = ART_NORMAL;
922 for_each_art(i) {
923 if (i == 0)
924 continue;
925
926 /* Check each character to see if it matched enough */
927 k = 0;
928 unmatched = 0;
929 for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
930 if (arts[base[root_num]].subject[j] != arts[i].subject[k])
931 unmatched++;
932 }
933
934 /*
935 * By getting here we have a number of unmatched characters
936 * between the two strings. We also have the length of the
937 * strings available to us easily.
938 * All we need to do is see if the match is good enough, but
939 * we count differences in the length of the strings against
940 * them matching.
941 */
942 if (!(slen = strlen(arts[base[root_num]].subject)))
943 slen++;
944 unmatched += (unsigned) (slen - strlen(arts[i].subject));
945 if (unmatched * 100 / slen > percentage) {
946 /*
947 * If there is less greater than percentage% different start a
948 * new thread.
949 */
950 base[++root_num] = i;
951 arts[i].prev = ART_NORMAL;
952 continue;
953 } else {
954 /*
955 * The subject lines match enough to consider them part of a single
956 * thread, so add the current article to the thread.
957 */
958 if (arts[base[root_num]].thread < 0)
959 arts[base[root_num]].thread = i;
960 arts[i].prev = i - 1;
961 arts[i - 1].thread = i;
962 continue;
963 }
964 }
965 }
966
967
968 /*
969 * Parses a subject header of the type "multipart message subject (01/42)"
970 * into a MultiPartInfo struct, or fails if the message subject isn't in the
971 * right form.
972 *
973 * @return nonzero on success
974 */
975 int
976 global_get_multipart_info(
977 int aindex,
978 MultiPartInfo *setme)
979 {
980 int i, j, offi, offj;
981 MultiPartInfo setmei, setmej;
982
983 i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
984 j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
985
986 /* Ok i hits first */
987 if (offi > offj) {
988 *setme = setmei;
989 return i;
990 }
991
992 /* Its j or they are both the same (which must be zero!) so we don't care */
993 *setme = setmej;
994 return j;
995 }
996
997
998 static int
999 global_look_for_multipart_info(
1000 int aindex,
1001 MultiPartInfo* setme,
1002 char start,
1003 char stop,
1004 int *offset)
1005 {
1006 char *subj;
1007 char *pch;
1008 MultiPartInfo tmp;
1009
1010 *offset = 0;
1011
1012 /* entry assertions */
1013 assert(0 <= aindex && aindex < top_art && "invalid index");
1014 assert(setme != NULL && "setme must not be NULL");
1015
1016 /* parse the message */
1017 subj = arts[aindex].subject;
1018 pch = strrchr(subj, start);
1019 if (!pch || !isdigit((int) pch[1]))
1020 return 0;
1021
1022 tmp.arts_index = aindex;
1023 tmp.subject_compare_len = (int) (pch - subj);
1024 tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1025 if (*pch != '/' && *pch != '|')
1026 return 0;
1027
1028 if (!isdigit((int) pch[1]))
1029 return 0;
1030
1031 tmp.total = (int) strtol(pch + 1, &pch, 10);
1032 if (*pch != stop)
1033 return 0;
1034
1035 /*
1036 * skip "blah (00/102)" or "blah (103/102)" subjects
1037 */
1038 if (tmp.part_number < 1 || tmp.part_number > tmp.total)
1039 return 0;
1040
1041 tmp.subject = subj;
1042 *setme = tmp;
1043 *offset = (int) (pch - subj);
1044 return 1;
1045 }
1046
1047
1048 t_bool
1049 global_look_for_multipart(
1050 int aindex,
1051 char start,
1052 char stop)
1053 {
1054 char *pch;
1055
1056 pch = strrchr(arts[aindex].subject, start);
1057 if (!pch || !isdigit((int) pch[1]))
1058 return FALSE;
1059
1060 strtol(pch + 1, &pch, 10);
1061 if (*pch != '/' && *pch != '|')
1062 return FALSE;
1063
1064 if (!isdigit((int) pch[1]))
1065 return FALSE;
1066
1067 strtol(pch + 1, &pch, 10);
1068 if (*pch != stop)
1069 return FALSE;
1070
1071 arts[aindex].multipart_subj = TRUE;
1072 return TRUE;
1073 }
1074
1075
1076 /*
1077 * Tries to find all the parts to the multipart message pointed to by
1078 * aindex.
1079 *
1080 * @return on success, the number of parts found. On failure, zero if not
1081 * a multipart or the negative value of the first missing part in case of
1082 * tagging.
1083 * @param aindex index pointing to one of the messages in a multipart
1084 * message.
1085 * @param malloc_and_setme_info on success, set to a malloced array the
1086 * parts found.
1087 */
1088 int
1089 global_get_multiparts(
1090 int aindex,
1091 MultiPartInfo **malloc_and_setme_info,
1092 t_bool tagging)
1093 {
1094 int i, part_index, part_cnt = 0;
1095 MultiPartInfo tmp, tmp2;
1096 MultiPartInfo *info = NULL;
1097
1098 /* entry assertions */
1099 assert(0 <= aindex && aindex < top_art && "Invalid index");
1100 assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1101
1102 /* make sure this is a multipart message... */
1103 if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1104 return 0;
1105
1106 /* make a temporary buffer to hold the multipart info... */
1107 info = my_malloc(sizeof(MultiPartInfo) * (size_t) tmp.total);
1108
1109 /* zero out part-number for the repost check below */
1110 for (i = 0; i < tmp.total; ++i) {
1111 info[i].total = tmp.total; /* Added this for thread_by_multipart */
1112 info[i].part_number = -1;
1113 }
1114
1115 /* try to find all the multiparts... */
1116 for (i = (tagging ? 0 : aindex); i < top_art; i++) {
1117 if (!arts[i].multipart_subj || strncmp(arts[i].subject, tmp.subject, (size_t) tmp.subject_compare_len))
1118 continue;
1119
1120 if (!global_get_multipart_info(i, &tmp2))
1121 continue;
1122
1123 /* 'test (1/5)' is not the same as 'test (1/15)' */
1124 if (tmp.total != tmp2.total)
1125 continue;
1126
1127 part_index = tmp2.part_number - 1;
1128
1129 /* repost check: do we already have this part? */
1130 if (info[part_index].part_number != -1) {
1131 assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1132 continue;
1133 }
1134
1135 /* we have a match, hooray! */
1136 info[part_index] = tmp2;
1137
1138 arts[i].multipart_subj = FALSE;
1139
1140 /* all parts found? */
1141 if (++part_cnt == tmp.total)
1142 break;
1143 }
1144
1145 /* see if we got them all. */
1146 if (tagging) {
1147 for (i = 0; i < tmp.total; ++i) {
1148 if (info[i].part_number != i + 1) {
1149 free(info);
1150 return -(i + 1); /* missing part #(i+1) */
1151 }
1152 }
1153 }
1154
1155 /* looks like a success .. */
1156 *malloc_and_setme_info = info;
1157 return tmp.total;
1158 }
1159
1160
1161 /*
1162 * The algorithm uses the tag multipart searches to thread articles together.
1163 */
1164 static void
1165 thread_by_multipart(
1166 void)
1167 {
1168 int i, j, threadNum;
1169 MultiPartInfo *minfo = NULL;
1170
1171 for_each_art(i) {
1172 if (!global_look_for_multipart(i, '[', ']'))
1173 global_look_for_multipart(i, '(', ')');
1174 }
1175
1176 for_each_art(i) {
1177 if (!arts[i].multipart_subj)
1178 continue;
1179
1180 if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo, FALSE)) {
1181 FreeAndNull(minfo);
1182 arts[i].multipart_subj = FALSE;
1183 continue;
1184 }
1185
1186 threadNum = -1;
1187 for (j = minfo[0].total - 1; j >= 0; j--) {
1188 if (minfo[j].part_number != -1) {
1189 if (threadNum != -1) {
1190 arts[minfo[j].arts_index].thread = threadNum;
1191 arts[threadNum].prev = minfo[j].arts_index;
1192 }
1193 threadNum = minfo[j].arts_index;
1194 }
1195 }
1196 FreeAndNull(minfo);
1197 arts[i].multipart_subj = FALSE;
1198 if (i % MODULO_COUNT_NUM == 0) /* TODO: -> lang.c */
1199 show_progress(_("Threading by multipart"), i, top_art);
1200 }
1201 }
1202
1203
1204 /*
1205 * Go through the articles in arts[] and create threads. There are
1206 * 5 strategies currently defined :
1207 *
1208 * THREAD_NONE No threading
1209 * THREAD_SUBJ Threads are created using like Subject lines
1210 * THREAD_REFS Threads are created using the References headers
1211 * THREAD_BOTH Threads created using References and then Subject
1212 * THREAD_MULTI Threads created using Subject to search for Multiparts
1213 * THREAD_PERC Threads based upon a char for char match of greater than x%
1214 *
1215 * .thread and .prev are used to hold the threading information, see tin.h for
1216 * more information
1217 * Only process valid (unexpired) articles we haven't visited yet
1218 * (ie arts[].thread == ART_UNTHREADED)
1219 *
1220 * The rethread parameter is used to force the deletion of existing threading
1221 * information before threading which happens anyway expect when using
1222 * THREAD_NONE (I don't immediately see how this is useful)
1223 */
1224 /* TODO: rewrite that user can easily combine different 'threading'
1225 * methods, i.e:
1226 * - thread_by_multipart() + collate_subjects()
1227 */
1228 void
1229 make_threads(
1230 struct t_group *group,
1231 t_bool rethread)
1232 {
1233 if (!cmd_line && !batch_mode) {
1234 info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
1235 my_flush();
1236 }
1237
1238 #ifdef DEBUG
1239 if (debug & DEBUG_MISC)
1240 error_message(2, "rethread=[%d] thread_articles=[%d] attr_thread_articles=[%d]",
1241 rethread, tinrc.thread_articles, group->attribute->thread_articles);
1242 #endif /* DEBUG */
1243
1244 /*
1245 * Sort all the articles using the preferred method
1246 * When find_base() is called, the bases are created ordered
1247 * on arts[] and so the base messages under all threading systems
1248 * will be sorted in this way.
1249 */
1250 sort_arts(group->attribute->sort_article_type);
1251
1252 /*
1253 * Reset all the ptrs to articles following the above sort
1254 */
1255 clear_art_ptrs();
1256
1257 /*
1258 * The threading pointers need to be reset if re-threading
1259 * If using ref threading, revector the links back to the articles
1260 */
1261 if (rethread || group->attribute->thread_articles) {
1262 int i;
1263
1264 for_each_art(i) {
1265 if (arts[i].thread >= 0)
1266 arts[i].thread = ART_UNTHREADED;
1267
1268 arts[i].prev = ART_NORMAL;
1269
1270 /* Should never happen if tree is built properly */
1271 if (arts[i].refptr == NULL) {
1272 #ifdef DEBUG
1273 if (debug & DEBUG_REFS) {
1274 my_fprintf(stderr, "\nError : art->refptr is NULL\n");
1275 my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1276 my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1277 my_fprintf(stderr, "From : %s\n", arts[i].from);
1278 assert(arts[i].refptr != NULL);
1279 } else
1280 #endif /* DEBUG */
1281 continue;
1282 }
1283 arts[i].refptr->article = i;
1284 }
1285 }
1286
1287 /*
1288 * Do the right thing according to the threading strategy
1289 */
1290 switch (group->attribute->thread_articles) {
1291 case THREAD_NONE:
1292 break;
1293
1294 case THREAD_SUBJ:
1295 thread_by_subject();
1296 break;
1297
1298 case THREAD_REFS:
1299 thread_by_reference();
1300 break;
1301
1302 case THREAD_BOTH:
1303 thread_by_reference();
1304 collate_subjects();
1305 break;
1306
1307 case THREAD_MULTI:
1308 thread_by_multipart();
1309 break;
1310
1311 case THREAD_PERC:
1312 thread_by_percentage((unsigned) (100 - group->attribute->thread_perc));
1313 break;
1314
1315 default: /* not reached */
1316 break;
1317 }
1318
1319 /*
1320 * Rebuild base[]
1321 */
1322 find_base(group);
1323 }
1324
1325
1326 static t_compfunc
1327 eval_sort_arts_func(
1328 unsigned int sort_art_type)
1329 {
1330 switch (sort_art_type) {
1331 case SORT_ARTICLES_BY_NOTHING: /* don't sort at all */
1332 return artnum_comp;
1333
1334 case SORT_ARTICLES_BY_SUBJ_DESCEND:
1335 return subj_comp_desc;
1336
1337 case SORT_ARTICLES_BY_SUBJ_ASCEND:
1338 return subj_comp_asc;
1339
1340 case SORT_ARTICLES_BY_FROM_DESCEND:
1341 return from_comp_desc;
1342
1343 case SORT_ARTICLES_BY_FROM_ASCEND:
1344 return from_comp_asc;
1345
1346 case SORT_ARTICLES_BY_DATE_DESCEND:
1347 return date_comp_desc;
1348
1349 case SORT_ARTICLES_BY_DATE_ASCEND:
1350 return date_comp_asc;
1351
1352 case SORT_ARTICLES_BY_SCORE_DESCEND:
1353 return score_comp_desc;
1354
1355 case SORT_ARTICLES_BY_SCORE_ASCEND:
1356 return score_comp_asc;
1357
1358 case SORT_ARTICLES_BY_LINES_DESCEND:
1359 return lines_comp_desc;
1360
1361 case SORT_ARTICLES_BY_LINES_ASCEND:
1362 return lines_comp_asc;
1363
1364 default:
1365 break;
1366 }
1367 return NULL;
1368 }
1369
1370
1371 void
1372 sort_arts(
1373 unsigned int sort_art_type)
1374 {
1375 t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1376
1377 if (comp_func)
1378 SortBy(comp_func);
1379 }
1380
1381
1382 static void
1383 sort_base(
1384 unsigned int sort_threads_type)
1385 {
1386 switch (sort_threads_type) {
1387 case SORT_THREADS_BY_SCORE_DESCEND:
1388 case SORT_THREADS_BY_SCORE_ASCEND:
1389 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1390 break;
1391
1392 case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
1393 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
1394 break;
1395
1396 case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
1397 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
1398 break;
1399 }
1400 }
1401
1402
1403 /*
1404 * This is called to get header info for articles not already found in the
1405 * overview files.
1406 */
1407 static t_bool
1408 parse_headers(
1409 FILE *fp,
1410 struct t_article *h)
1411 {
1412 char art_from_addr[HEADER_LEN];
1413 char art_full_name[HEADER_LEN];
1414 char *s, *hdr, *ptr;
1415 t_bool got_from, got_lines;
1416
1417 got_from = got_lines = FALSE;
1418
1419 while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1420 /*
1421 * Look for the end of information which tin wants to get.
1422 * Applies when reading local spool and via NNTP.
1423 */
1424
1425 /*
1426 * End of headers ?
1427 */
1428 if (ptr[0] == '\0')
1429 break;
1430
1431 unfold_header(ptr);
1432 switch (my_toupper((unsigned char) *ptr)) {
1433 case 'D': /* Date: mandatory */
1434 if (!h->date) {
1435 if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1436 h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1437 }
1438 break;
1439
1440 case 'F': /* From: mandatory */
1441 if (!got_from) {
1442 if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1443 h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1444 h->from = hash_str(buffer_to_ascii(art_from_addr));
1445 if (*art_full_name)
1446 h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
1447 got_from = TRUE;
1448 }
1449 }
1450 break;
1451
1452 case 'L': /* Lines: optional */
1453 if (!got_lines) {
1454 if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1455 h->line_count = atoi(hdr);
1456 got_lines = TRUE;
1457 }
1458 }
1459 break;
1460
1461 case 'M': /* Message-ID: mandatory */
1462 if (!h->msgid) {
1463 if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1464 h->msgid = my_strdup(hdr);
1465 }
1466 break;
1467
1468 /* for Path:-filter when reading from local spool */
1469 case 'P': /* Path: */
1470 if (!h->path) {
1471 if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1472 h->path = my_strdup(hdr);
1473 }
1474 break;
1475
1476 case 'R': /* References: optional */
1477 if (!h->refs) {
1478 if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1479 h->refs = my_strdup(hdr);
1480 }
1481 break;
1482
1483 case 'S': /* Subject: mandatory */
1484 if (!h->subject) {
1485 if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
1486 #ifdef HAVE_UNICODE_NORMALIZATION
1487 if (IS_LOCAL_CHARSET("UTF-8"))
1488 s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1489 else
1490 #endif /* HAVE_UNICODE_NORMALIZATION */
1491 s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1492
1493 h->subject = hash_str(s);
1494 free(s);
1495 }
1496 }
1497 break;
1498
1499 case 'X': /* Xref: optional */
1500 if (!h->xref) {
1501 if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1502 h->xref = my_strdup(hdr);
1503 }
1504 break;
1505
1506 default:
1507 break;
1508 } /* switch */
1509
1510 } /* while */
1511
1512 #ifdef NNTP_ABLE
1513 if (ptr)
1514 drain_buffer(fp);
1515 #endif /* NNTP_ABLE */
1516
1517 if (tin_errno)
1518 return FALSE;
1519
1520 /*
1521 * The son of RFC 1036 states that the following hdrs are mandatory. It
1522 * also states that Subject, Newsgroups and Path are too. Ho hum.
1523 *
1524 * What about reading mail from local spool via ~/.tin/active.mail,
1525 * they might not have a Message-ID.
1526 */
1527 if (got_from && h->date && h->msgid) {
1528 if (!h->subject)
1529 h->subject = hash_str("");
1530
1531 #ifdef DEBUG
1532 if (debug & DEBUG_FILTER)
1533 debug_print_header(h);
1534 #endif /* DEBUG */
1535 return TRUE;
1536 }
1537
1538 return FALSE;
1539 }
1540
1541 #ifdef NNTP_ABLE
1542 /*
1543 * Loop over arts[] and find ranges without Path: header
1544 * If there are any try to optimize the ranges regarding traffic consumption
1545 * Start optimization if at least MIN_CNT ranges exist
1546 * If there are more than MAX_CNT ranges after optimization, fetch all in one
1547 * big range
1548 */
1549 #define MIN_CNT 10
1550 #define MAX_CNT 50
1551 static struct t_article_range *
1552 build_range_list(
1553 t_artnum min,
1554 t_artnum max,
1555 int *range_cnt)
1556 {
1557 int i, gap_cnt = 0;
1558 struct t_article_range *res = NULL, *gap_list, *curr, *from;
1559 t_artnum new_end;
1560
1561 new_end = T_ARTNUM_CONST(0);
1562 gap_list = my_malloc(sizeof(struct t_article_range));
1563 curr = gap_list;
1564 curr->start = min;
1565 curr->end = max;
1566 curr->cnt = T_ARTNUM_CONST(0);
1567 curr->next = NULL;
1568
1569 for_each_art(i) {
1570 if (arts[i].artnum < min)
1571 continue;
1572 if (arts[i].artnum > max)
1573 break;
1574 if (arts[i].path) {
1575 for (; i < top_art && arts[i].path; i++)
1576 ;
1577 /*
1578 * the current art has no path -> we use this one
1579 * if we reached top_art all arts have path
1580 * so we use max
1581 */
1582 curr->start = i == top_art ? max : arts[i--].artnum;
1583 } else {
1584 for (; i < top_art && !arts[i].path; i++)
1585 ;
1586 /* the current art has path -> we use the last one */
1587 new_end = curr->end = arts[--i].artnum;
1588 }
1589 if (new_end) {
1590 curr->cnt = curr->end - curr->start + 1;
1591 curr->next = my_malloc(sizeof(struct t_article_range));
1592 curr = curr->next;
1593 curr->start = new_end;
1594 curr->end = max;
1595 curr->cnt = T_ARTNUM_CONST(0);
1596 curr->next = NULL;
1597 new_end = T_ARTNUM_CONST(0);
1598 }
1599 }
1600
1601 curr = gap_list;
1602 while (curr && curr->cnt) {
1603 ++gap_cnt;
1604 # ifdef DEBUG
1605 if ((debug & DEBUG_NNTP) && verbose > 1)
1606 debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1607 # endif /* DEBUG */
1608 curr = curr->next;
1609 }
1610
1611 /*
1612 * Optimize only if there are at least MIN_CNT ranges
1613 */
1614 if (gap_cnt >= MIN_CNT) {
1615 res = my_malloc(sizeof(struct t_article_range));
1616 res->start = T_ARTNUM_CONST(0);
1617 res->end = T_ARTNUM_CONST(0);
1618 res->cnt = T_ARTNUM_CONST(0);
1619 res->next = NULL;
1620
1621 from = gap_list;
1622 curr = res;
1623 while (from) {
1624 curr->start = from->start;
1625 curr->end = from->end;
1626 curr->cnt = from->cnt;
1627 if ((from = from->next)) {
1628 /*
1629 * If the next range is grater then the gap between the current
1630 * one and the next one we build a new range including the
1631 * current one, the next one and the gap between
1632 */
1633 while (from && from->cnt >= from->start - curr->end - 1) {
1634 curr->end = from->end;
1635 from = from->next;
1636 }
1637 curr->cnt = curr->end - curr->start + 1;
1638 curr->next = my_malloc(sizeof(struct t_article_range));
1639 curr = curr->next;
1640 curr->start = T_ARTNUM_CONST(0);
1641 curr->end = T_ARTNUM_CONST(0);
1642 curr->cnt = T_ARTNUM_CONST(0);
1643 curr->next = NULL;
1644 }
1645 }
1646 }
1647
1648 /*
1649 * If there are less then MIN_CNT ranges
1650 * no res is build -> return the original list
1651 */
1652 if (res) {
1653 while (gap_list) {
1654 curr = gap_list;
1655 gap_list = curr->next;
1656 free(curr);
1657 }
1658 } else
1659 res = gap_list;
1660
1661 curr = res;
1662 gap_cnt = 0;
1663 while (curr && curr->cnt) {
1664 ++gap_cnt;
1665 # ifdef DEBUG
1666 if ((debug & DEBUG_NNTP) && verbose > 1)
1667 debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1668 # endif /* DEBUG */
1669 curr = curr->next;
1670 }
1671
1672 if (gap_cnt >= MAX_CNT) {
1673 curr = res;
1674 while (curr->next && curr->next->cnt) {
1675 res->end = curr->next->end;
1676 curr->next->cnt = 0;
1677 curr = curr->next;
1678 }
1679 res->cnt = res->end - res->start + 1;
1680 gap_cnt = 1;
1681 # ifdef DEBUG
1682 if ((debug & DEBUG_NNTP) && verbose > 1)
1683 debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1684 # endif /* DEBUG */
1685 }
1686 *range_cnt = gap_cnt;
1687
1688 return res;
1689 }
1690
1691
1692 /*
1693 * Fetch the Path header in case we want to filter on that in the given group
1694 *
1695 * Try [X]HDR first, then XPAT
1696 */
1697 static t_bool
1698 get_path_header(
1699 int cur,
1700 int cnt,
1701 struct t_group *group,
1702 t_artnum min,
1703 t_artnum max)
1704 {
1705 FILE *fp = NULL;
1706 char *prep_msg;
1707 char *buf, *ptr;
1708 char cmd[NNTP_STRLEN];
1709 t_artnum artnum, i;
1710 t_bool found = FALSE;
1711 static t_bool supported = TRUE; /* assume HDR || XPAT works */
1712
1713 if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1714 return FALSE;
1715
1716 # ifdef DEBUG
1717 if ((debug & DEBUG_NNTP) && verbose > 1)
1718 debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1719 # endif /* DEBUG */
1720
1721 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
1722 int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1723 switch (j) {
1724 case 215:
1725 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1726 # ifdef DEBUG
1727 if (debug & DEBUG_NNTP)
1728 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1729 # endif /* DEBUG */
1730 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
1731 strcat(nntp_caps.headers_range, ptr);
1732 strcat(nntp_caps.headers_range, "\n");
1733 }
1734 break;
1735
1736 default:
1737 break;
1738 }
1739 }
1740
1741 /* does HDR return Path? */
1742 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1743 do {
1744 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1745 found = TRUE;
1746 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1747 }
1748
1749 if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1750 if (min == max)
1751 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1752 else
1753 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1754 fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1755 if (!nntp_caps.hdr && fp)
1756 nntp_caps.hdr = TRUE;
1757 } else if (nntp_caps.xpat) {
1758 if (min == max)
1759 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1760 else
1761 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1762 fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1763 }
1764
1765 if (fp) {
1766 int j = 0;
1767
1768 prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1769 while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1770 # ifdef DEBUG
1771 if ((debug & DEBUG_NNTP) && verbose)
1772 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1773 # endif /* DEBUG */
1774 if ((ptr = tin_strtok(buf, " ")) == NULL)
1775 continue;
1776 artnum = atoartnum(ptr);
1777 if ((ptr = tin_strtok(NULL, " ")) == NULL)
1778 continue;
1779 for (i = j; i < top_art; i++) {
1780 if (arts[i].artnum == artnum) {
1781 FreeIfNeeded(arts[i].path);
1782 arts[i].path = my_strdup(ptr);
1783 j = (int) i;
1784 break;
1785 }
1786 }
1787 if (++artnum % MODULO_COUNT_NUM == 0)
1788 show_progress(prep_msg, artnum - min, max - min);
1789 }
1790 free(prep_msg);
1791 return supported;
1792 }
1793
1794 /* !fp */
1795 supported = FALSE;
1796 if (nntp_caps.xpat)
1797 nntp_caps.xpat = FALSE;
1798 /* as nntp_caps.hdr may work with other headers we don't disable it */
1799
1800 # ifdef DEBUG
1801 if ((debug & DEBUG_NNTP) && verbose > 1)
1802 debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
1803 # endif /* DEBUG */
1804 return supported;
1805 }
1806 #endif /* NNTP_ABLE */
1807
1808
1809 /*
1810 * Read in an overview index file. Fields are separated by TAB.
1811 * return the number of expired articles encountered or -1 if the user aborted
1812 * the read
1813 * 'top' is set to the highest artnum read
1814 * If 'local' is set then always open local overview cache in preference to
1815 * using NNTP XOVER
1816 *
1817 * Format (mandatory as far as line count [RFC2980]):
1818 * 1. article number (ie. 183) [mandatory]
1819 * 2. Subject: line (ie. Which newsreader?) [mandatory]
1820 * 3. From: line (ie. iain@ecrc.de) [mandatory]
1821 * 4. Date: line (rfc822 format) [mandatory]
1822 * 5. MessageID: (ie. <123@example.net>) [mandatory]
1823 * 6. References: (ie. <message-id> ....) [optional]
1824 * 7. Byte count (Skipped - not used) [mandatory]
1825 * 8. Line count (ie. 23) [mandatory]
1826 * 9. Xref: line (ie. alt.test:389) [optional]
1827 */
1828 static int
1829 read_overview(
1830 struct t_group *group,
1831 t_artnum min,
1832 t_artnum max,
1833 t_artnum *top,
1834 t_bool local,
1835 t_bool *rebuild_cache)
1836 {
1837 FILE *fp;
1838 char *ptr;
1839 char *q;
1840 char *buf;
1841 char *group_msg;
1842 char art_full_name[HEADER_LEN];
1843 char art_from_addr[HEADER_LEN];
1844 unsigned int count;
1845 int expired = 0;
1846 t_artnum artnum;
1847 t_bool path_found = FALSE, path_in_ofmt = FALSE;
1848 struct t_article *art;
1849 size_t over_fields = 1;
1850
1851 /*
1852 * open the overview file (whether it be local or via nntp)
1853 */
1854 if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1855 return expired;
1856
1857 if (group->xmax > max)
1858 group->xmax = max;
1859
1860 group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1861
1862 /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1863 if (ofmt) {
1864 for (; ofmt[over_fields].name; over_fields++) {
1865 if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1866 path_in_ofmt = TRUE;
1867 }
1868 }
1869
1870 if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1871 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1872 ofmt[0].type = OVER_T_INT;
1873 ofmt[0].name = my_strdup("Artnum:");
1874 ofmt[1].type = OVER_T_STRING;
1875 ofmt[1].name = my_strdup("Subject:");
1876 ofmt[2].type = OVER_T_STRING;
1877 ofmt[2].name = my_strdup("From:");
1878 ofmt[3].type = OVER_T_STRING;
1879 ofmt[3].name = my_strdup("Date:");
1880 ofmt[4].type = OVER_T_STRING;
1881 ofmt[4].name = my_strdup("Message-ID:");
1882 ofmt[5].type = OVER_T_STRING;
1883 ofmt[5].name = my_strdup("References:");
1884 ofmt[6].type = OVER_T_INT;
1885 ofmt[6].name = my_strdup("Bytes:");
1886 ofmt[7].type = OVER_T_INT;
1887 ofmt[7].name = my_strdup("Lines:");
1888 ofmt[8].type = OVER_T_ERROR;
1889 ofmt[8].name = NULL;
1890 over_fields = 7;
1891 }
1892
1893 while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1894 #ifdef DEBUG
1895 if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1896 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1897 #endif /* DEBUG */
1898
1899 if (need_resize) {
1900 handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
1901 need_resize = cNo;
1902 }
1903
1904 /*
1905 * Read artnum
1906 */
1907 if ((ptr = tin_strtok(buf, "\t")) == NULL)
1908 continue;
1909
1910 /*
1911 * read the article number, guaranteed to be the first field
1912 */
1913 artnum = atoartnum(ptr);
1914
1915 /*
1916 * artnum field invalid/corrupt or is 1st line of local cached overview
1917 * (group name)
1918 */
1919 if (artnum <= 0)
1920 continue;
1921
1922 /*
1923 * skip artnums below the given minimum (getart_limit)
1924 */
1925 if (artnum < min)
1926 continue;
1927
1928 /*
1929 * Check to make sure article in nov file has not expired in group
1930 */
1931 if (artnum < group->xmin) {
1932 expired++;
1933 continue;
1934 }
1935
1936 /*
1937 * artnum in overview data higher than groups high mark
1938 *
1939 * TODO: - warn user about broken overviews?
1940 * - try to parse the Xref:-line to get the correct artnum
1941 * - see also parse_unread_arts()
1942 */
1943 if (artnum > group->xmax)
1944 continue;
1945
1946 if (top_art >= max_art)
1947 expand_art();
1948
1949 art = &arts[top_art];
1950 set_article(art);
1951 art->artnum = *top = artnum;
1952
1953 /*
1954 * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
1955 * to check for additions like we do with xref_supported
1956 */
1957 for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
1958 /* skip unexpected tailing fields */
1959 if (count > over_fields) {
1960 #ifdef DEBUG
1961 if ((debug & DEBUG_NNTP) && verbose > 1)
1962 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
1963 #endif /* DEBUG */
1964
1965 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
1966 if (count == over_fields + 1) {
1967 if (!strncasecmp(ptr, "Xref: ", 6)) {
1968 #ifdef DEBUG
1969 if ((debug & DEBUG_NNTP) && verbose > 1)
1970 debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
1971 #endif /* DEBUG */
1972 over_fields++;
1973 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1974 ofmt[over_fields].type = OVER_T_FSTRING;
1975 ofmt[over_fields].name = my_strdup("Xref:");
1976 ofmt[over_fields + 1].type = OVER_T_ERROR;
1977 ofmt[over_fields + 1].name = NULL;
1978 xref_supported = TRUE;
1979 } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
1980 #ifdef DEBUG
1981 if ((debug & DEBUG_NNTP) && verbose > 1)
1982 debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
1983 #endif /* DEBUG */
1984 over_fields++;
1985 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1986 ofmt[over_fields].type = OVER_T_FSTRING;
1987 ofmt[over_fields].name = my_strdup("Path:");
1988 ofmt[over_fields + 1].type = OVER_T_ERROR;
1989 ofmt[over_fields + 1].name = NULL;
1990 xref_supported = TRUE;
1991 } else
1992 continue;
1993 } else
1994 continue;
1995 }
1996
1997 /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
1998 if (expensive_over_parse) { /* strange order */
1999 /* mandatory fields */
2000 if (ofmt[count].type == OVER_T_STRING) {
2001 if (!strcasecmp(ofmt[count].name, "Subject:")) {
2002 if (*ptr) {
2003 #ifdef HAVE_UNICODE_NORMALIZATION
2004 if (IS_LOCAL_CHARSET("UTF-8"))
2005 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2006 else
2007 #endif /* HAVE_UNICODE_NORMALIZATION */
2008 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2009
2010 art->subject = hash_str(q);
2011 free(q);
2012 } else {
2013 art->subject = hash_str("");
2014 #ifdef DEBUG
2015 if ((debug & DEBUG_NNTP) && verbose > 1)
2016 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2017 #endif /* DEBUG */
2018 }
2019 continue;
2020 }
2021
2022 if (!strcasecmp(ofmt[count].name, "From:")) {
2023 if (*ptr) {
2024 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2025 art->from = hash_str(buffer_to_ascii(art_from_addr));
2026 if (*art_full_name)
2027 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2028 } else {
2029 art->from = hash_str("");
2030 #ifdef DEBUG
2031 if ((debug & DEBUG_NNTP) && verbose > 1)
2032 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2033 #endif /* DEBUG */
2034 }
2035 continue;
2036 }
2037
2038 if (!strcasecmp(ofmt[count].name, "Date:")) {
2039 art->date = parsedate(ptr, (TIMEINFO *) 0);
2040 #ifdef DEBUG
2041 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2042 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2043 #endif /* DEBUG */
2044 continue;
2045 }
2046
2047 if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2048 if (*ptr) {
2049 FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2050 art->msgid = my_strdup(ptr);
2051 } else {
2052 art->msgid = NULL;
2053 #ifdef DEBUG
2054 if ((debug & DEBUG_NNTP) && verbose > 1)
2055 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2056 #endif /* DEBUG */
2057 }
2058 continue;
2059 }
2060
2061 if (!strcasecmp(ofmt[count].name, "References:")) {
2062 if (*ptr) {
2063 FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2064 art->refs = my_strdup(ptr);
2065 } else
2066 art->refs = NULL;
2067 continue;
2068 }
2069
2070 /*
2071 * non std. fields when doing
2072 * expensive overview parsing (very
2073 * rare, just happens if RFC 3977
2074 * 8.4.2 is violated) go here
2075 */
2076 /* for Path:-filter */
2077 if (!strcasecmp(ofmt[count].name, "Path:")) {
2078 if (!path_found)
2079 path_found = TRUE;
2080 if (*ptr) {
2081 FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2082 art->path = my_strdup(ptr);
2083 } else
2084 art->path = NULL;
2085 continue;
2086 }
2087 }
2088 /* metadata fields */
2089 if (ofmt[count].type == OVER_T_INT) {
2090 if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2091 if (*ptr) {
2092 #ifdef DEBUG
2093 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2094 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2095 #endif /* DEBUG */
2096 }
2097 continue;
2098 }
2099
2100 if (!strcasecmp(ofmt[count].name, "Lines:")) {
2101 if (*ptr) {
2102 if (isdigit((unsigned char) *ptr))
2103 art->line_count = atoi(ptr);
2104 else {
2105 art->line_count = 0;
2106 #ifdef DEBUG
2107 if ((debug & DEBUG_NNTP) && verbose > 1)
2108 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2109 #endif /* DEBUG */
2110 }
2111 } else
2112 art->line_count = 0;
2113 continue;
2114 }
2115 }
2116 } else { /* first 7 fields are in RFC 3977 order */
2117 switch (count) {
2118 case 1: /* Subject: */
2119 if (*ptr) {
2120 #ifdef HAVE_UNICODE_NORMALIZATION
2121 if (IS_LOCAL_CHARSET("UTF-8"))
2122 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2123 else
2124 #endif /* HAVE_UNICODE_NORMALIZATION */
2125 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2126
2127 art->subject = hash_str(q);
2128 free(q);
2129 } else {
2130 art->subject = hash_str("");
2131 #ifdef DEBUG
2132 if ((debug & DEBUG_NNTP) && verbose > 1)
2133 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2134 #endif /* DEBUG */
2135 }
2136 break;
2137
2138 case 2: /* From: */
2139 if (*ptr) {
2140 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2141 art->from = hash_str(buffer_to_ascii(art_from_addr));
2142 if (*art_full_name)
2143 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2144 } else {
2145 art->from = hash_str("");
2146 #ifdef DEBUG
2147 if ((debug & DEBUG_NNTP) && verbose > 1)
2148 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2149 #endif /* DEBUG */
2150 }
2151 break;
2152
2153 case 3: /* Date: */
2154 art->date = parsedate(ptr, (TIMEINFO *) 0);
2155 #ifdef DEBUG
2156 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2157 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2158 #endif /* DEBUG */
2159 break;
2160
2161 case 4: /* Message-ID: */
2162 if (*ptr)
2163 art->msgid = my_strdup(ptr);
2164 else {
2165 art->msgid = NULL;
2166 #ifdef DEBUG
2167 if ((debug & DEBUG_NNTP) && verbose > 1)
2168 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2169 #endif /* DEBUG */
2170 }
2171 break;
2172
2173 case 5: /* References: */
2174 if (*ptr)
2175 art->refs = my_strdup(ptr);
2176 else
2177 art->refs = NULL;
2178 break;
2179
2180 case 6: /* :bytes || Bytes: */
2181 if (*ptr) {
2182 #ifdef DEBUG
2183 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2184 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2185 #endif /* DEBUG */
2186 }
2187 break;
2188
2189 case 7: /* :lines || Lines: */
2190 if (*ptr) {
2191 if (isdigit((unsigned char) *ptr))
2192 art->line_count = atoi(ptr);
2193 else {
2194 art->line_count = 0;
2195 #ifdef DEBUG
2196 if ((debug & DEBUG_NNTP) && verbose > 1)
2197 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2198 #endif /* DEBUG */
2199 }
2200 } else
2201 art->line_count = 0;
2202 break;
2203
2204 default:
2205 break;
2206 }
2207 }
2208
2209 /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2210 if (ofmt[count].type == OVER_T_FSTRING) {
2211 if (*ptr) {
2212 if (!strcasecmp(ofmt[count].name, "Xref:")) {
2213 if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2214 FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2215 art->xref = my_strdup(q);
2216 }
2217 #ifdef DEBUG
2218 else {
2219 if ((debug & DEBUG_NNTP) && verbose > 1)
2220 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2221 }
2222 #endif /* DEBUG */
2223 continue;
2224 }
2225 /*
2226 * handling of addition overview fields
2227 * goes here
2228 */
2229 #ifdef DEBUG
2230 if ((debug & DEBUG_NNTP) && verbose > 1)
2231 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2232 #endif /* DEBUG */
2233 /* if we're lucky we've Path in NOV */
2234 /*
2235 * if reading locally cached overview data try
2236 * path regardless of the server OVERVIEW.FMT
2237 */
2238 if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2239 if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2240 if (!path_found)
2241 path_found = TRUE;
2242 FreeIfNeeded(art->path);
2243 art->path = my_strdup(q);
2244 #ifdef DEBUG
2245 if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2246 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2247 #endif /* DEBUG */
2248
2249 }
2250 continue;
2251 }
2252 }
2253 continue;
2254 }
2255 }
2256
2257 /*
2258 * RFC says Message-ID is mandatory in newsgroups (but not in
2259 * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2260 * threading in mailgroups
2261 */
2262 if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2263 continue;
2264
2265 /* we might lose accuracy here, but that shouldn't hurt */
2266 if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2267 show_progress(group_msg, artnum - min, max - min);
2268
2269 top_art++; /* Basically this statement commits the article */
2270 }
2271
2272 free(group_msg);
2273 TIN_FCLOSE(fp);
2274
2275 if (tin_errno)
2276 return -1;
2277
2278 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2279 if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
2280 char cbuf[HEADER_LEN];
2281 int i;
2282 static t_bool found;
2283 static t_bool first = TRUE;
2284
2285 if (first) {
2286 found = TRUE;
2287 /*
2288 * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2289 */
2290 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
2291 if (!*nntp_caps.headers_range) {
2292 i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2293
2294 found = FALSE;
2295 switch (i) {
2296 case 215:
2297 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2298 # ifdef DEBUG
2299 if (debug & DEBUG_NNTP)
2300 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2301 # endif /* DEBUG */
2302 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2303 found = TRUE;
2304 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
2305 strcat(nntp_caps.headers_range, ptr);
2306 strcat(nntp_caps.headers_range, "\n");
2307 }
2308 break;
2309
2310 default:
2311 break;
2312 }
2313 first = FALSE;
2314 } else {
2315 found = FALSE;
2316 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
2317 do {
2318 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2319 found = TRUE;
2320 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2321 }
2322 }
2323 }
2324 }
2325
2326 if (found) {
2327 snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2328 group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2329 if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2330 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2331 # ifdef DEBUG
2332 if ((debug & DEBUG_NNTP) && verbose)
2333 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2334 # endif /* DEBUG */
2335
2336 artnum = atoartnum(ptr);
2337 if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2338 continue;
2339 art = &arts[top_art];
2340 set_article(art);
2341 if (!art->xref && !strstr(ptr, "(none)")) {
2342 if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2343 continue;
2344 ptr = q;
2345 while (*ptr && isspace((int) *ptr))
2346 ptr++;
2347 q = strchr(ptr, '\n');
2348 if (q)
2349 *q = '\0';
2350 art->xref = my_strdup(ptr);
2351 }
2352 /* we might lose accuracy here, but that shouldn't hurt */
2353 if (artnum % (MODULO_COUNT_NUM * 20) == 0)
2354 show_progress(group_msg, artnum - min, max - min);
2355 }
2356 }
2357 free(group_msg);
2358 }
2359 }
2360 #endif /* NNTP_ABLE && XHDR_XREF */
2361
2362 if (local) {
2363 #ifdef NNTP_ABLE
2364 if (filter_on_path(group)) {
2365 struct t_article_range *ranges, *curr;
2366 t_bool supported = TRUE;
2367 int curr_range, range_cnt;
2368
2369 /*
2370 * Get the ranges without Path: header and try to fetch the
2371 * headers
2372 */
2373 if ((ranges = build_range_list(min, *top, &range_cnt))) {
2374 curr = ranges;
2375 curr_range = 1;
2376 while (curr && supported) {
2377 if (curr->cnt)
2378 supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2379 curr = curr->next;
2380 }
2381 if (!supported && path_in_ofmt) {
2382 /*
2383 * fetching Path: headers via [X]HDR or XPAT has failed
2384 * Path: is in the servers overview so let the next
2385 * read_overview() fetch them
2386 */
2387 free_art_array();
2388 free_msgids();
2389 top_art = 0;
2390 *top = T_ARTNUM_CONST(0);
2391 expired = 0;
2392 }
2393 *rebuild_cache = TRUE;
2394 while (ranges) {
2395 curr = ranges;
2396 ranges = curr->next;
2397 free(curr);
2398 }
2399 }
2400 }
2401 #endif /* NNTP_ABLE */
2402 } else
2403 if (!path_found && filter_on_path(group)) {
2404 #ifdef NNTP_ABLE
2405 if (!get_path_header(1, 1, group, min, *top))
2406 #endif /* NNTP_ABLE */
2407 wait_message(2, _(txt_cannot_filter_on_path));
2408 }
2409 return expired;
2410 }
2411
2412
2413 /*
2414 * Write an Nov/Xover index file. Fields are separated by '\t'.
2415 *
2416 * Format:
2417 * 1. article number (ie. 183) [mandatory]
2418 * 2. Subject: line (ie. Which newsreader?) [mandatory]
2419 * 3. From: line (ie. iain@ecrc.de) [mandatory]
2420 * 4. Date: line (rfc822 format) [mandatory]
2421 * 5. MessageID: (ie. <123@example.net>) [mandatory]
2422 * 6. References: (ie. <message-id> ....) [optional]
2423 * 7. Byte count (Skipped - not used) [mandatory]
2424 * 8. Line count (ie. 23) [mandatory]
2425 * 9. Xref: line (ie. alt.test:389) [optional]
2426 *
2427 * TODO: as we don't use the original data, we currently can't store
2428 * the data (from/subject) in the original charset (we don't store
2429 * that info). this has the advantage that we can avoid raw 8bit data
2430 * in our overviews, but the disadvantage that we might store the data
2431 * with a wrong charset and thus lose information. a similar problem
2432 * exists with the data for the from:-line, we don't store it in the
2433 * original format, whenever our from-parser (partially) fails we'll
2434 * lose information in our overviews (but those couldn't be handled
2435 * by tin anyway, so this is not a real problem).
2436 * long-term solution: store the original data in the overview
2437 * (tin has to handle raw 8bit data and other ugly stuff in the
2438 * overviews anyway and thus we preserver as much info as possible)
2439 * this would require some changes in read_overview() and
2440 * parse_headers(): don't do the decoding/unfolding there, but in a
2441 * second pass right after write_overview(), or two additional fields
2442 * which hold the raw data for from/subject. the latter has the
2443 * disadvantage that it costs (much) more memory.
2444 */
2445 static void
2446 write_overview(
2447 struct t_group *group)
2448 {
2449 FILE *fp;
2450 int i;
2451 struct t_article *article;
2452 #ifdef CHARSET_CONVERSION
2453 int c = -1;
2454 #endif /* CHARSET_CONVERSION */
2455
2456 /*
2457 * Can't write or caching is off or getart_limit is set
2458 */
2459 if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
2460 return;
2461
2462 if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2463 return;
2464
2465 if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
2466 SortBy(artnum_comp);
2467
2468 /*
2469 * Needed to preserve uniqueness in hashed private overview files
2470 */
2471 fprintf(fp, "%s\n", group->name);
2472
2473 #ifdef CHARSET_CONVERSION
2474 /* get undeclared_charset number if required */
2475 if (group->attribute->undeclared_charset) {
2476 for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2477 if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2478 c = i;
2479 break;
2480 }
2481 }
2482 }
2483 #endif /* CHARSET_CONVERSION */
2484
2485 if (verbose && batch_mode) /* -> lang.c */
2486 wait_message(0, _("Writing %s\n"), group->name);
2487
2488 for_each_art(i) {
2489 char *p;
2490 char *q, *ref;
2491
2492 article = &arts[i];
2493
2494 if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2495 ref = NULL;
2496
2497 if (!group->attribute->post_8bit_header) { /* write encoded data */
2498 /*
2499 * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2500 * here and in print_from() in the CHARSET_CONVERSION case.
2501 * note that this requires something like
2502 * buffer_to_network(article->subject, "UTF-8");
2503 * right before the rfc1522_encode() call.
2504 *
2505 * if we would cache the original undecoded data, we could
2506 * ignore stuff like this.
2507 */
2508 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
2509 /* as the subject might now be folded we have to unfold it */
2510 unfold_header(p);
2511 } else { /* raw data */
2512 p = my_strdup(article->subject);
2513 #ifdef CHARSET_CONVERSION
2514 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2515 buffer_to_network(p, c);
2516 #endif /* CHARSET_CONVERSION */
2517 }
2518
2519 /*
2520 * replace any '\t's with ' ' in the references-data
2521 *
2522 * TODO: nntpext-draft might come up with a new scheme:
2523 * For all fields, the value is processed by first
2524 * removing all US-ASCII CRLF pairs and then replacing
2525 * each remaining US-ASCII NUL, TAB, CR, or LF character
2526 * with a single US-ASCII space (for example, CR LF LF TAB
2527 * will become two spaces).
2528 */
2529 if (article->refs) {
2530 ref = q = my_strdup(article->refs);
2531 while (*q) {
2532 if (*q == '\t')
2533 *q = ' ';
2534 q++;
2535 }
2536 }
2537
2538 fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2539 article->artnum,
2540 p,
2541 #ifdef CHARSET_CONVERSION
2542 print_from(group, article, c),
2543 #else
2544 print_from(group, article, -1),
2545 #endif /* CHARSET_CONVERSION */
2546 print_date(article->date),
2547 BlankIfNull(article->msgid),
2548 BlankIfNull(ref),
2549 0, /* bytes */
2550 article->line_count);
2551
2552 if (article->xref)
2553 fprintf(fp, "\tXref: %s", article->xref);
2554
2555 if (article->path)
2556 fprintf(fp, "\tPath: %s", article->path);
2557
2558 fprintf(fp, "\n");
2559 free(p);
2560 if (article->refs) {
2561 FreeIfNeeded(ref);
2562 }
2563 }
2564 if (i % (MODULO_COUNT_NUM * 20) == 0) /* TODO: -> lang.c */
2565 show_progress(_("Writing overview cache..."), i, top_art);
2566 }
2567 #ifdef HAVE_FCHMOD
2568 fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2569 /*
2570 * TODO:
2571 * add code for !HAVE_FCHMOD && HAVE_CHMOD
2572 */
2573 #endif /* HAVE_FCHMOD */
2574 fclose(fp);
2575 }
2576
2577
2578 /*
2579 * A complex little function to determine the correct overview index file
2580 * according to 'mode' (read or write)
2581 * NULL is returned if the current setup dictates otherwise
2582 *
2583 * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2584 * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2585 *
2586 * Both of these are hashed
2587 *
2588 * GROUP_TYPE_NEWS index files are a little bit more complex
2589 *
2590 * When hashing the index filename will be in format number.number.
2591 * Hashing the groupname gets a number. See if that #.1 file exists;
2592 * if so, read first line. Is this the group we want? If no, try #.2.
2593 * Repeat until no such file or we find an existing file that matches
2594 * our group. Return pointer to path or NULL if not found.
2595 */
2596 static char *
2597 find_nov_file(
2598 struct t_group *group,
2599 int mode)
2600 {
2601 FILE *fp;
2602 const char *dir;
2603 char buf[PATH_LEN];
2604 int i;
2605 struct stat sb;
2606 unsigned long hash;
2607 static char nov_file[PATH_LEN];
2608 static t_bool once_only = FALSE; /* Trap things that are done only 1 time */
2609
2610 if (group == NULL || (mode != R_OK && mode != W_OK))
2611 return NULL;
2612
2613 switch (group->type) {
2614 case GROUP_TYPE_MAIL:
2615 dir = index_maildir;
2616 break;
2617
2618 case GROUP_TYPE_SAVE:
2619 dir = index_savedir;
2620 break;
2621
2622 case GROUP_TYPE_NEWS:
2623 /*
2624 * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2625 * about [X]OVER are handled in nntp_open()
2626 */
2627
2628 /*
2629 * When reading via NNTP, system wide overviews are irrelevant, of
2630 * course, and the private overview filename will be the same for
2631 * both reading and writing.
2632 *
2633 * When working locally, we only use a private cache for reading
2634 * if requested and when system wide overviews don't already exist.
2635 * When writing then only private overviews can be used since
2636 * updating system wide overviews is not safe wrt locking etc.
2637 *
2638 * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2639 *
2640 * INN >= 2.3.0 seems to use a new naming schemme with tradindexed
2641 * see untested gross hack below; buffindexed and ovdb are not
2642 * covered by the code at all.
2643 */
2644 #ifndef NNTP_ONLY
2645 if (!read_news_via_nntp) {
2646 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2647 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2648 # if 0 /* TODO: FIXME - ugly hack for inn >= 2.3.0 with ovmethod tradindexed */
2649 {
2650 char *gn = my_strdup(group->name);
2651 size_t t, j;
2652 t_bool w = FALSE;
2653
2654 for (t = 1, j = 1; t < strlen(group->name); t++) {
2655 if (!w) {
2656 if (group->name[t] == '.') {
2657 gn[j++] = '/';
2658 w = TRUE;
2659 }
2660 } else {
2661 if (group->name[t] != '.') { /* illegal .. in name? */
2662 gn[j++] = group->name[t];
2663 w = FALSE;
2664 }
2665 }
2666 }
2667 gn[j] = '\0';
2668
2669 joinpath(nov_file, sizeof(nov_file), novrootdir, gn);
2670 free(gn);
2671 snprintf(nov_file + strlen(nov_file), sizeof(nov_file) - strlen(nov_file), "/%s.DAT", group->name);
2672 }
2673 # endif /* 0 */
2674 if (access(nov_file, R_OK) == 0) {
2675 if (mode == R_OK)
2676 return nov_file; /* Use system wide overviews */
2677 else
2678 return NULL; /* Don't write cache in this case */
2679 }
2680 }
2681 #endif /* !NNTP_ONLY */
2682
2683 /*
2684 * We only get here when private overviews are going to be used
2685 * Go no further if they are explicitly turned off
2686 */
2687 if (!tinrc.cache_overview_files)
2688 return NULL;
2689
2690 /*
2691 * Append -<nntpserver> to private cache dir
2692 */
2693 if (!once_only && nntp_server) {
2694 size_t sp = sizeof(index_newsdir), ln = strlen(index_newsdir);
2695
2696 if (--sp - ln >= 2) {
2697 char *srv = my_strdup(nntp_server);
2698
2699 str_lwr(srv);
2700 strcat(index_newsdir, "-");
2701 my_strncpy(index_newsdir + ln + 1, srv, sp);
2702 free(srv);
2703 }
2704 once_only = TRUE;
2705 }
2706
2707 /*
2708 * Only try to set up the private cache when writing. If it
2709 * doesn't exist yet, then ergo we can't read from it.
2710 * The cache will be checked/created on every write; a previous
2711 * bug report complained that this was not the case
2712 */
2713 if (stat(index_newsdir, &sb) == -1) { /* Private cache doesn't exist */
2714 if (mode == R_OK)
2715 return NULL;
2716 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2717 return NULL;
2718 } else {
2719 if (!S_ISDIR(sb.st_mode))
2720 return NULL;
2721 }
2722
2723 /*
2724 * Update the newsgroups cache to point to the new location
2725 * now that we know it is valid
2726 */
2727 if (!once_only)
2728 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
2729
2730 dir = index_newsdir;
2731 break;
2732
2733 default: /* not reached */
2734 return NULL;
2735 }
2736
2737 /*
2738 * We only get here if writing to a private overview.
2739 * These always have hashed filenames.
2740 * Try <hash>.<seqno> and check the group name tagline until
2741 * matching index file is found. If not found return next unused
2742 * filename
2743 */
2744 hash = hash_groupname(group->name);
2745
2746 for (i = 1; ; i++) {
2747 char *ptr;
2748
2749 snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2750 joinpath(nov_file, sizeof(nov_file), dir, buf);
2751
2752 if ((fp = fopen(nov_file, "r")) == NULL)
2753 break;
2754
2755 /*
2756 * No group name header, so not a valid index file => overwrite it
2757 */
2758 if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2759 fclose(fp);
2760 break;
2761 }
2762 fclose(fp);
2763
2764 if ((ptr = strrchr(buf, '\n')) != NULL)
2765 *ptr = '\0';
2766
2767 if (strcmp(buf, group->name) == 0)
2768 break;
2769 }
2770
2771 return nov_file;
2772 }
2773
2774
2775 /*
2776 * Run the index file updater only for the groups we've loaded.
2777 */
2778 void
2779 do_update(
2780 t_bool catchup)
2781 {
2782 int i, j, k = 0;
2783 time_t beg_epoch = 0;
2784 struct t_article *art;
2785 struct t_group *group;
2786
2787 if (verbose)
2788 (void) time(&beg_epoch);
2789
2790 /*
2791 * loop through groups and update any required index files
2792 */
2793 for (i = 0; i < selmenu.max; i++) {
2794 group = &active[my_group[i]];
2795 /*
2796 * FIXME: workaround to get a valid CURR_GROUP
2797 * it also points to the currently processed group so that
2798 * the correct attributes are used
2799 * The correct fix is to get rid of CURR_GROUP
2800 */
2801 selmenu.curr = i;
2802
2803 if (group->bogus || !group->subscribed)
2804 continue;
2805
2806 if (!index_group(group)) {
2807 for_each_art(j) {
2808 art = &arts[j];
2809 FreeAndNull(art->refs);
2810 FreeAndNull(art->msgid);
2811 }
2812 continue;
2813 }
2814
2815 k++;
2816
2817 if (verbose) {
2818 my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2819 my_flush();
2820 }
2821
2822 if (catchup) {
2823 for_each_art(j)
2824 art_mark(group, &arts[j], ART_READ);
2825 }
2826 }
2827
2828 if (verbose) {
2829 wait_message(0, _(txt_catchup_update_info),
2830 (catchup ? _(txt_caughtup) : _(txt_updated)), k,
2831 PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2832 }
2833 }
2834
2835
2836 static int
2837 artnum_comp(
2838 t_comptype p1,
2839 t_comptype p2)
2840 {
2841 const struct t_article *s1 = (const struct t_article *) p1;
2842 const struct t_article *s2 = (const struct t_article *) p2;
2843
2844 /*
2845 * s1->artnum less than s2->artnum
2846 */
2847 if (s1->artnum < s2->artnum)
2848 return -1;
2849
2850 /*
2851 * s1->artnum greater than s2->artnum
2852 */
2853 if (s1->artnum > s2->artnum)
2854 return 1;
2855
2856 return 0;
2857 }
2858
2859
2860 /*
2861 * return result of strcmp (reversed for descending)
2862 */
2863 static int
2864 subj_comp_asc(
2865 t_comptype p1,
2866 t_comptype p2)
2867 {
2868 int retval;
2869 const struct t_article *s1 = (const struct t_article *) p1;
2870 const struct t_article *s2 = (const struct t_article *) p2;
2871
2872 if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2873 return retval;
2874
2875 return s1->date - s2->date > 0 ? 1 : -1;
2876 }
2877
2878
2879 static int
2880 subj_comp_desc(
2881 t_comptype p1,
2882 t_comptype p2)
2883 {
2884 int retval;
2885 const struct t_article *s1 = (const struct t_article *) p1;
2886 const struct t_article *s2 = (const struct t_article *) p2;
2887
2888 if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2889 return retval;
2890
2891 return s1->date - s2->date > 0 ? 1 : -1;
2892 }
2893
2894
2895 /*
2896 * return result of strcmp (reversed for descending)
2897 */
2898 static int
2899 from_comp_asc(
2900 t_comptype p1,
2901 t_comptype p2)
2902 {
2903 int retval;
2904 const struct t_article *s1 = (const struct t_article *) p1;
2905 const struct t_article *s2 = (const struct t_article *) p2;
2906
2907 if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2908 return retval;
2909
2910 return s1->date - s2->date > 0 ? 1 : -1;
2911 }
2912
2913
2914 static int
2915 from_comp_desc(
2916 t_comptype p1,
2917 t_comptype p2)
2918 {
2919 int retval;
2920 const struct t_article *s1 = (const struct t_article *) p1;
2921 const struct t_article *s2 = (const struct t_article *) p2;
2922
2923 if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2924 return retval;
2925
2926 return s1->date - s2->date > 0 ? 1 : -1;
2927 }
2928
2929
2930 /*
2931 * Works like strcmp() for comparing time_t type values
2932 * Return codes:
2933 * -1: If p1 is before p2
2934 * 0: If they are the same time
2935 * 1: If p1 is after p2
2936 * If the sort order is _not_ DATE_ASCEND then the sense of the above
2937 * is reversed.
2938 */
2939 static int
2940 date_comp_asc(
2941 t_comptype p1,
2942 t_comptype p2)
2943 {
2944 const struct t_article *s1 = (const struct t_article *) p1;
2945 const struct t_article *s2 = (const struct t_article *) p2;
2946
2947 /*
2948 * s1->date less than s2->date
2949 */
2950 if (s1->date < s2->date)
2951 return -1;
2952
2953 /*
2954 * s1->date greater than s2->date
2955 */
2956 if (s1->date > s2->date)
2957 return 1;
2958
2959 return 0;
2960 }
2961
2962
2963 static int
2964 date_comp_desc(
2965 t_comptype p1,
2966 t_comptype p2)
2967 {
2968 const struct t_article *s1 = (const struct t_article *) p1;
2969 const struct t_article *s2 = (const struct t_article *) p2;
2970
2971 /*
2972 * s2->date less than s1->date
2973 */
2974 if (s2->date < s1->date)
2975 return -1;
2976
2977 /*
2978 * s2->date greater than s1->date
2979 */
2980 if (s2->date > s1->date)
2981 return 1;
2982
2983 return 0;
2984 }
2985
2986
2987 /*
2988 * Same again, but for art[].score
2989 */
2990 static int
2991 score_comp_asc(
2992 t_comptype p1,
2993 t_comptype p2)
2994 {
2995 const struct t_article *s1 = (const struct t_article *) p1;
2996 const struct t_article *s2 = (const struct t_article *) p2;
2997
2998 if (s1->score < s2->score)
2999 return -1;
3000
3001 if (s1->score > s2->score)
3002 return 1;
3003
3004 return s1->date - s2->date > 0 ? 1 : -1;
3005 }
3006
3007
3008 static int
3009 score_comp_desc(
3010 t_comptype p1,
3011 t_comptype p2)
3012 {
3013 const struct t_article *s1 = (const struct t_article *) p1;
3014 const struct t_article *s2 = (const struct t_article *) p2;
3015
3016 if (s2->score < s1->score)
3017 return -1;
3018
3019 if (s2->score > s1->score)
3020 return 1;
3021
3022 return s1->date - s2->date > 0 ? 1 : -1;
3023 }
3024
3025
3026 /*
3027 * Same again, but for art[].line_count
3028 */
3029 static int
3030 lines_comp_asc(
3031 t_comptype p1,
3032 t_comptype p2)
3033 {
3034 const struct t_article *s1 = (const struct t_article *) p1;
3035 const struct t_article *s2 = (const struct t_article *) p2;
3036
3037 if (s1->line_count < s2->line_count)
3038 return -1;
3039
3040 if (s1->line_count > s2->line_count)
3041 return 1;
3042
3043 return s1->date - s2->date > 0 ? 1 : -1;
3044 }
3045
3046
3047 static int
3048 lines_comp_desc(
3049 t_comptype p1,
3050 t_comptype p2)
3051 {
3052 const struct t_article *s1 = (const struct t_article *) p1;
3053 const struct t_article *s2 = (const struct t_article *) p2;
3054
3055 if (s2->line_count < s1->line_count)
3056 return -1;
3057
3058 if (s2->line_count > s1->line_count)
3059 return 1;
3060
3061 return s1->date - s2->date > 0 ? 1 : -1;
3062 }
3063
3064
3065 /*
3066 * Compares the total score of two threads. Used for sorting base[].
3067 */
3068 static int
3069 score_comp_base(
3070 t_comptype p1,
3071 t_comptype p2)
3072 {
3073 int a = get_score_of_thread((int) *(const long *) p1);
3074 int b = get_score_of_thread((int) *(const long *) p2);
3075
3076 /* If scores are equal, compare using the article sort order.
3077 * This determines the order in a group of equally scored threads.
3078 */
3079 if (a == b) {
3080 const struct t_article *s1 = &arts[*(const long *) p1];
3081 const struct t_article *s2 = &arts[*(const long *) p2];
3082 t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3083
3084 if (comp_func)
3085 return (*comp_func)(s1, s2);
3086 return 0;
3087 }
3088
3089 if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3090 return a > b ? 1 : -1;
3091 return a < b ? 1 : -1;
3092 }
3093
3094
3095 /*
3096 * Compare the date of the last posted article of two threads.
3097 * Used for sorting base[].
3098 */
3099 static int
3100 last_date_comp_base_desc(
3101 t_comptype p1,
3102 t_comptype p2)
3103 {
3104 time_t s1_last = get_last_posting_date(*(const long *) p1);
3105 time_t s2_last = get_last_posting_date(*(const long *) p2);
3106
3107 if (s2_last < s1_last)
3108 return -1;
3109
3110 if (s2_last > s1_last)
3111 return 1;
3112
3113 return 0;
3114 }
3115
3116
3117 static int
3118 last_date_comp_base_asc(
3119 t_comptype p1,
3120 t_comptype p2)
3121 {
3122 time_t s1_last = get_last_posting_date(*(const long *) p1);
3123 time_t s2_last = get_last_posting_date(*(const long *) p2);
3124
3125 if (s2_last > s1_last)
3126 return -1;
3127
3128 if (s2_last < s1_last)
3129 return 1;
3130
3131 return 0;
3132 }
3133
3134
3135 static time_t
3136 get_last_posting_date(
3137 long n)
3138 {
3139 long i;
3140 time_t last = (time_t) 0;
3141
3142 for (i = n; i >= 0; i = arts[i].thread) {
3143 if (arts[i].date > last)
3144 last = arts[i].date;
3145 }
3146
3147 return last;
3148 }
3149
3150
3151 void
3152 set_article(
3153 struct t_article *art)
3154 {
3155 art->subject = NULL;
3156 art->from = NULL;
3157 art->name = NULL;
3158 art->date = (time_t) 0;
3159 art->xref = NULL;
3160 art->msgid = NULL;
3161 art->refs = NULL;
3162 art->refptr = NULL;
3163 art->line_count = -1;
3164 art->tagged = 0;
3165 art->thread = ART_EXPIRED;
3166 art->prev = ART_NORMAL;
3167 art->score = 0;
3168 art->status = ART_UNREAD;
3169 art->killed = ART_NOTKILLED;
3170 art->zombie = FALSE;
3171 art->delete_it = FALSE;
3172 art->selected = FALSE;
3173 art->inrange = FALSE;
3174 art->matched = FALSE;
3175 art->keep_in_base = FALSE;
3176 art->multipart_subj = FALSE;
3177 }
3178
3179
3180 /*
3181 * Do a binary chop to see if 'art' (an article number) exists in arts[]
3182 * Naturally arts[] must be sorted on artnum
3183 * Return index into arts[] or -1
3184 */
3185 static int
3186 valid_artnum(
3187 t_artnum art)
3188 {
3189 int prev, range;
3190 int dctop = top_art;
3191 int cur = 1;
3192
3193 while ((dctop >>= 1))
3194 cur <<= 1;
3195
3196 range = cur >> 1;
3197 cur--;
3198
3199 forever {
3200 if (arts[cur].artnum == art)
3201 return cur;
3202
3203 prev = cur;
3204 cur += ((arts[cur].artnum < art) ? range : -range);
3205 if (prev == cur)
3206 break;
3207
3208 if (cur >= top_art)
3209 cur = top_art - 1;
3210
3211 range >>= 1;
3212 }
3213 return -1;
3214 }
3215
3216
3217 /*
3218 * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3219 * Needed if arts[] is not sorted on artnum
3220 * Return index into arts[] or -1
3221 */
3222 int
3223 find_artnum(
3224 t_artnum art)
3225 {
3226 int i;
3227
3228 for_each_art(i) {
3229 if (arts[i].artnum == art)
3230 return i;
3231 }
3232 return -1;
3233 }
3234
3235
3236 /*----------------------------- Overview handling -----------------------*/
3237 /* TODO: use
3238 * setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3239 * my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3240 * instead?
3241 */
3242 static char *
3243 print_date(
3244 time_t secs)
3245 {
3246 static char date[25];
3247 struct tm *tm;
3248 static const char *const months_a[] = {
3249 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3250 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3251 };
3252
3253 if ((tm = gmtime(&secs)) != NULL)
3254 snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3255 tm->tm_mday,
3256 months_a[tm->tm_mon],
3257 tm->tm_year + 1900,
3258 tm->tm_hour, tm->tm_min, tm->tm_sec);
3259 else
3260 snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3261
3262 return date;
3263 }
3264
3265
3266 static char *
3267 print_from(
3268 struct t_group *group,
3269 struct t_article *article,
3270 int charset)
3271 {
3272 char *p, *q;
3273 static char from[PATH_LEN];
3274
3275 *from = '\0';
3276
3277 if (article->name != NULL) {
3278 q = my_strdup(article->name);
3279 #ifdef CHARSET_CONVERSION
3280 if (charset != -1)
3281 buffer_to_network(q, charset);
3282 #endif /* CHARSET_CONVERSION */
3283 p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
3284 unfold_header(p);
3285 if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3286 snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3287 else
3288 snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3289
3290 free(p);
3291 free(q);
3292 } else
3293 snprintf(from, sizeof(from), "<%s>", article->from);
3294
3295 return from;
3296 }
3297
3298
3299 /*
3300 * Open a group news overview file
3301 * Use NNTP XOVER where possible unless 'local' is set
3302 */
3303 static FILE *
3304 open_xover_fp(
3305 struct t_group *group,
3306 const char *mode,
3307 t_artnum min,
3308 t_artnum max,
3309 t_bool local)
3310 {
3311 #ifdef NNTP_ABLE
3312 if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3313 char line[NNTP_STRLEN];
3314
3315 if (!max)
3316 return NULL;
3317 if (min == max)
3318 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3319 else
3320 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3321 return (nntp_command(line, OK_XOVER, NULL, 0));
3322 }
3323 #endif /* NNTP_ABLE */
3324 {
3325 FILE *fp;
3326 char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3327
3328 if (nov_file != NULL) {
3329 if ((fp = fopen(nov_file, mode)) != NULL)
3330 return fp;
3331
3332 if (*mode != 'r')
3333 error_message(2, _(txt_cannot_open), nov_file);
3334 }
3335 }
3336 return NULL;
3337 }
3338
3339
3340 #ifdef USE_HEAPSORT
3341 int
3342 tin_sort(
3343 void *sbase,
3344 size_t nel,
3345 size_t width,
3346 t_compfunc compar)
3347 {
3348 int rc;
3349
3350 switch (tinrc.sort_function) {
3351 case 0:
3352 qsort(sbase, nel, width, compar);
3353 rc = 0;
3354 break;
3355
3356 case 1:
3357 rc = heapsort(sbase, nel, width, compar);
3358 break;
3359
3360 default:
3361 rc = -1;
3362 break;
3363 }
3364 return rc;
3365 }
3366 #endif /* USE_HEAPSORT */