"Fossies" - the Fresh Open Source Software Archive 
Member "tin-2.4.4/src/art.c" (20 Nov 2019, 89949 Bytes) of package /linux/misc/tin-2.4.4.tar.xz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "art.c" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
2.4.3_vs_2.4.4.
1 /*
2 * Project : tin - a Usenet reader
3 * Module : art.c
4 * Author : I.Lea & R.Skrenta
5 * Created : 1991-04-01
6 * Updated : 2019-06-12
7 * Notes :
8 *
9 * Copyright (c) 1991-2020 Iain Lea <iain@bricbrac.de>, Rich Skrenta <skrenta@pbm.com>
10 * All rights reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * 1. Redistributions of source code must retain the above copyright notice,
17 * this list of conditions and the following disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 *
23 * 3. Neither the name of the copyright holder nor the names of its
24 * contributors may be used to endorse or promote products derived from
25 * this software without specific prior written permission.
26 *
27 * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
31 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40
41 #ifndef TIN_H
42 # include "tin.h"
43 #endif /* !TIN_H */
44 #ifndef TCURSES_H
45 # include "tcurses.h"
46 #endif /* !TCURSES_H */
47 #ifndef NEWSRC_H
48 # include "newsrc.h"
49 #endif /* !NEWSRC_H */
50
51 #ifndef STPWATCH_H
52 # include "stpwatch.h"
53 #endif /* !STPWATCH_H */
54
55 /*
56 * TODO: fixup to remove CURR_GROUP dependency in all sort funcs
57 */
58 #define SortBy(func) tin_sort(arts, (size_t) top_art, sizeof(struct t_article), func);
59
60 int top_art = 0; /* # of articles in arts[] */
61
62 /*
63 * Local prototypes
64 */
65 static FILE *open_art_header(char *groupname, t_artnum art, t_artnum *next);
66 static FILE *open_xover_fp(struct t_group *group, const char *mode, t_artnum min, t_artnum max, t_bool local);
67 static char *find_nov_file(struct t_group *group, int mode);
68 static char *print_date(time_t secs);
69 static char *print_from(struct t_group *group, struct t_article *article, int charset);
70 static int artnum_comp(t_comptype p1, t_comptype p2);
71 static int base_comp(t_comptype p1, t_comptype p2);
72 static int date_comp_asc(t_comptype p1, t_comptype p2);
73 static int date_comp_desc(t_comptype p1, t_comptype p2);
74 static int from_comp_asc(t_comptype p1, t_comptype p2);
75 static int from_comp_desc(t_comptype p1, t_comptype p2);
76 static int global_get_multiparts(int aindex, MultiPartInfo **malloc_and_setme_info);
77 static int global_look_for_multipart_info(int aindex, MultiPartInfo *setme, char start, char stop, int *offset);
78 static int last_date_comp_base_asc(t_comptype p1, t_comptype p2);
79 static int last_date_comp_base_desc(t_comptype p1, t_comptype p2);
80 static int lines_comp_asc(t_comptype p1, t_comptype p2);
81 static int lines_comp_desc(t_comptype p1, t_comptype p2);
82 static int read_art_headers(struct t_group *group, int total, t_artnum top);
83 static int read_overview(struct t_group *group, t_artnum min, t_artnum max, t_artnum *top, t_bool local, t_bool *rebuild_cache);
84 static int score_comp_asc(t_comptype p1, t_comptype p2);
85 static int score_comp_desc(t_comptype p1, t_comptype p2);
86 static int score_comp_base(t_comptype p1, t_comptype p2);
87 static int subj_comp_asc(t_comptype p1, t_comptype p2);
88 static int subj_comp_desc(t_comptype p1, t_comptype p2);
89 static int valid_artnum(t_artnum art);
90 static t_artnum find_first_unread(struct t_group *group);
91 static t_artnum setup_hard_base(struct t_group *group);
92 static t_bool parse_headers(FILE *fp, struct t_article *h);
93 static t_compfunc eval_sort_arts_func(unsigned int sort_art_type);
94 static time_t get_last_posting_date(long n);
95 static void sort_base(unsigned int sort_threads_type);
96 static void thread_by_multipart(void);
97 static void thread_by_percentage(struct t_group *group);
98 static void thread_by_subject(void);
99 static void write_overview(struct t_group *group);
100 #ifdef NNTP_ABLE
101 static struct t_article_range *build_range_list(t_artnum min, t_artnum max, int *range_cnt);
102 static t_bool get_path_header(int cur, int cnt, struct t_group *group, t_artnum min, t_artnum max);
103 #endif /* NNTP_ABLE */
104
105
106 /*
107 * Display a suitable 'entering group' message if screen needs redrawing
108 * Allow for the non-printing %s, and the %-age counter
109 */
110 void
111 show_art_msg(
112 const char *group)
113 {
114 wait_message(0, _(txt_group), cCOLS - (strwidth(_(txt_group)) > cCOLS ? cCOLS : strwidth(_(txt_group)) + 2 - 3), group);
115 }
116
117
118 /*
119 * Construct the pointers to the first (base) article in each thread.
120 * If we are showing only unread, then point to the first unread. I have
121 * no idea why this should be so, it causes problems elsewhere [which_response]
122 */
123 void
124 find_base(
125 struct t_group *group)
126 {
127 int i, j;
128
129 grpmenu.max = 0;
130
131 #ifdef DEBUG
132 debug_print_arts();
133 #endif /* DEBUG */
134
135 for_each_art(i) {
136 /*
137 * .prev will be set on each article that is after the first article in
138 * the thread. Invalid articles which have been expired will have
139 * .thread set to ART_EXPIRED
140 */
141 if (arts[i].prev >= 0 || arts[i].thread == ART_EXPIRED || (arts[i].killed && tinrc.kill_level == KILL_NOTHREAD))
142 continue;
143
144 if (grpmenu.max >= max_base)
145 expand_base();
146
147 if (group->attribute->show_only_unread_arts) {
148 if (arts[i].status != ART_READ || arts[i].keep_in_base)
149 base[grpmenu.max++] = i;
150 else {
151 /* Find 1st unread art in thread */
152 for (j = i; j >= 0; j = arts[j].thread) {
153 if (arts[j].status != ART_READ || arts[j].keep_in_base) {
154 base[grpmenu.max++] = i;
155 break;
156 }
157 }
158 }
159 } else
160 base[grpmenu.max++] = i;
161 }
162 /* sort base[] */
163 if (group->attribute->sort_threads_type > SORT_THREADS_BY_NOTHING)
164 sort_base(group->attribute->sort_threads_type);
165 }
166
167
168 /*
169 * Longword comparison routine for the tin_sort()
170 */
171 static int
172 base_comp(
173 t_comptype p1,
174 t_comptype p2)
175 {
176 const t_artnum *a = (const t_artnum *) p1;
177 const t_artnum *b = (const t_artnum *) p2;
178
179 if (*a < *b)
180 return -1;
181
182 if (*a > *b)
183 return 1;
184
185 return 0;
186 }
187
188
189 /*
190 * via NNTP:
191 * Issue a LISTGROUP command
192 * Read the article numbers existing in the group into base[]
193 * If the LISTGROUP failed, issue a GROUP command. Use the results to
194 * create a less accurate version of base[]
195 * This data will already be sorted
196 *
197 * on local spool:
198 * Read the spool dir to populate base[] as above. Sort it.
199 *
200 * Grow the arts[] and bitmaps as needed.
201 * NB: the output will be sorted on artnum
202 *
203 * grpmenu.max is one past top.
204 * Returns total number of articles in group, or -1 on error
205 */
206 static t_artnum
207 setup_hard_base(
208 struct t_group *group)
209 {
210 t_artnum total = 0;
211
212 grpmenu.max = 0;
213
214 /*
215 * If reading with NNTP, issue a LISTGROUP
216 */
217 if (read_news_via_nntp && !read_saved_news && group->type == GROUP_TYPE_NEWS) {
218 #ifdef NNTP_ABLE
219 char buf[NNTP_STRLEN];
220 char line[NNTP_STRLEN];
221 int getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
222 FILE *fp;
223 t_artnum last, start, count = 0, j = 0;
224 static t_bool skip_listgroup = FALSE;
225
226 /*
227 * Some nntp servers are broken and need an extra GROUP command
228 * (reported by reorx@irc.pl). This affects (old?) versions of
229 * nntpcache, leafnode and SurgeNews. Usually this should not be
230 * needed.
231 *
232 * For getart_limit recheck lowwatermark as at least giganews gives
233 * very different results for LIST ACTIVE (3 year retention for all)
234 * and GROUP (based on the clients contract).
235 * Calculate range and prepare base[] not to lose unread arts.
236 */
237 if (nntp_caps.broken_listgroup || (!skip_listgroup && getart_limit && nntp_caps.type == CAPABILITIES && nntp_caps.reader)) {
238 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
239 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
240 return -1;
241
242 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start) != 2)
243 return -1;
244
245 if (getart_limit > 0) {
246 j = group->xmax - getart_limit;
247 count = MAX(find_first_unread(group), start);
248 }
249 if (getart_limit < 0) {
250 j = getart_limit + find_first_unread(group);
251 count = group->xmin;
252 }
253 if (j < group->xmin)
254 j = group->xmin;
255
256 for (; count < j; count++) {
257 if (grpmenu.max >= max_base)
258 expand_base();
259 base[grpmenu.max++] = count;
260 }
261 }
262
263 /*
264 * See if LISTGROUP works
265 */
266 if (!skip_listgroup && getart_limit != 0) { /* try to avoid traffic */
267 if (nntp_caps.type == CAPABILITIES && nntp_caps.reader) {
268 /* RFC 3977 allows ranges in LISTGROUP */
269 if (getart_limit > 0)
270 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT"", group->name, j, group->xmax);
271 else /* getart_limit < 0; fetch till newest art */
272 snprintf(buf, sizeof(buf), "LISTGROUP %s %"T_ARTNUM_PFMT"-", group->name, j);
273
274 } else /* for RFC 977 just use GROUP */
275 skip_listgroup = TRUE;
276
277 } else /* get all article numbers */
278 snprintf(buf, sizeof(buf), "LISTGROUP %s", group->name);
279
280 if (!skip_listgroup) {
281 if ((fp = nntp_command(buf, OK_GROUP, NULL, 0)) != NULL) {
282 char *ptr;
283
284 # ifdef DEBUG
285 if ((debug & DEBUG_NNTP) && verbose > 1)
286 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
287 # endif /* DEBUG */
288
289 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
290 if (grpmenu.max >= max_base)
291 expand_base();
292 base[grpmenu.max++] = atoartnum(ptr);
293 total++;
294 }
295
296 if (tin_errno)
297 return -1;
298 } else
299 skip_listgroup = TRUE;
300 }
301
302 if (skip_listgroup) { /* LISTGROUP was skipped or failed */
303 /*
304 * Handle the obscure case that the user aborted before the LISTGROUP
305 * had a chance to respond
306 */
307 if (tin_errno)
308 return -1;
309
310 snprintf(buf, sizeof(buf), "GROUP %s", group->name);
311 if (nntp_command(buf, OK_GROUP, line, sizeof(line)) == NULL)
312 return -1;
313
314 if (sscanf(line, "%"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT" %"T_ARTNUM_SFMT, &count, &start, &last) != 3)
315 return -1;
316
317 # ifdef DEBUG
318 if ((debug & DEBUG_NNTP) && verbose > 1)
319 debug_print_file("NNTP", "setup_hard_base(%s)", buf);
320 # endif /* DEBUG */
321 total = count;
322 grpmenu.max = 0;
323 if (getart_limit > 0) {
324 if ((j = find_first_unread(group)) > start) {
325 if (group->xmax > getart_limit) {
326 start = MIN(j, group->xmax - getart_limit);
327 total = getart_limit;
328 } else
329 start = j;
330 }
331 }
332 if (getart_limit < 0) {
333 if ((j = (getart_limit + find_first_unread(group))) > start)
334 start = j;
335 }
336 while (start <= last) {
337 if (grpmenu.max >= max_base)
338 expand_base();
339 base[grpmenu.max++] = start++;
340 }
341 }
342 #endif /* NNTP_ABLE */
343 /*
344 * Reading off local spool, read the directory files
345 */
346 } else {
347 DIR *d;
348 DIR_BUF *e;
349 char group_path[PATH_LEN];
350 t_artnum art;
351
352 make_base_group_path(group->spooldir, group->name, group_path, sizeof(group_path));
353
354 if ((d = opendir(group_path)) != NULL) {
355 while ((e = readdir(d)) != NULL) {
356 art = atoartnum(e->d_name);
357 if (art >= 1) {
358 total++;
359 if (grpmenu.max >= max_base)
360 expand_base();
361 base[grpmenu.max++] = art;
362 }
363 }
364 CLOSEDIR(d);
365 tin_sort((char *) base, (size_t) grpmenu.max, sizeof(t_artnum), base_comp);
366 } else {
367 perror_message(_(txt_cannot_open), group_path);
368 #if 0
369 if (access(group_path, R_OK) != 0)
370 error_message(2, _(txt_not_exist));
371 #endif /* 0 */
372 return -1;
373 }
374 }
375
376 if (grpmenu.max) {
377 if (base[grpmenu.max - 1] > group->xmax)
378 group->xmax = base[grpmenu.max - 1];
379 expand_bitmap(group, base[0]);
380 }
381
382 return total;
383 }
384
385
386 /*
387 * Main group indexing routine.
388 *
389 * Will read any existing index, create or incrementally update
390 * the index by looking at the articles in the spool directory,
391 * and attempt to write a new index if necessary.
392 *
393 * Returns FALSE if the user aborted the indexing, otherwise TRUE
394 */
395 t_bool
396 index_group(
397 struct t_group *group)
398 {
399 int i;
400 int changed; /* Count of articles whose overview has changed */
401 int getart_limit;
402 int respnum;
403 int total;
404 t_artnum last_read_article;
405 t_artnum min, new_min, max;
406 t_bool caching_xover;
407 t_bool filtered;
408 t_bool path_in_nov = FALSE;
409 t_bool rebuild_cache = FALSE;
410
411 if (group == NULL)
412 return TRUE;
413
414 if (!batch_mode)
415 show_art_msg(group->name);
416
417 signal_context = cArt; /* Set this only once curr_group is valid */
418
419 hash_reclaim();
420 free_art_array();
421 free_msgids();
422
423 BegStopWatch("setup_hard_base()");
424
425 /*
426 * Get list of valid article numbers
427 */
428 if (setup_hard_base(group) < 0)
429 return FALSE;
430
431 EndStopWatch();
432 PrintStopWatch();
433
434 #ifdef DEBUG
435 if (debug & DEBUG_NEWSRC) {
436 debug_print_comment("Before read_overview");
437 debug_print_bitmap(group, NULL);
438 }
439 #endif /* DEBUG */
440
441 min = grpmenu.max ? base[0] : group->xmin;
442 max = grpmenu.max ? base[grpmenu.max - 1] : min - 1;
443
444 getart_limit = (cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit;
445
446 if (getart_limit > 0) {
447 if (grpmenu.max && (grpmenu.max > getart_limit))
448 min = base[grpmenu.max - getart_limit];
449 else
450 getart_limit = 0;
451 } else if (getart_limit < 0) {
452 t_artnum first_unread = find_first_unread(group);
453
454 if (min - first_unread < getart_limit)
455 min = first_unread + getart_limit;
456 else
457 getart_limit = 0;
458 }
459
460 /*
461 * Quit now if no articles
462 */
463 if (max < 0)
464 return FALSE;
465
466 top_art = 0;
467 last_read_article = T_ARTNUM_CONST(0);
468
469 /*
470 * Read in the existing overview data for min..max
471 * This read has local=TRUE set if locally caching XOVER records to ensure
472 * we pull in any private overview caches in preference to using OVER
473 *
474 * When reading local spool, this will pull in the system wide overview
475 * cache (if found) otherwise the private overview cache will be read
476 */
477 caching_xover = (tinrc.cache_overview_files && nntp_caps.over_cmd && group->type == GROUP_TYPE_NEWS);
478 if ((changed = read_overview(group, min, max, &last_read_article, caching_xover, &rebuild_cache)) == -1)
479 return FALSE; /* user aborted indexing */
480
481 /*
482 * Fill in the range last_read_article...max using XOVER
483 * Only do this if the previous read_overview() was against private cache
484 */
485 if ((last_read_article < max) && caching_xover) {
486 new_min = (last_read_article >= min) ? last_read_article + 1 : min;
487
488 if ((i = read_overview(group, new_min, max, &last_read_article, FALSE, &rebuild_cache)) == -1)
489 return FALSE; /* user aborted indexing */
490 else
491 changed += i;
492 } else
493 caching_xover = FALSE;
494
495 /*
496 * Mark as UNTHREADED all articles that have been verified as valid
497 * Get num of new arts to index so the user will have an idea of index time
498 */
499 for (i = 0, total = 0; i < grpmenu.max; i++) {
500 if ((respnum = valid_artnum(base[i])) >= 0) {
501 arts[respnum].thread = ART_UNTHREADED;
502 continue;
503 }
504 if (base[i] <= last_read_article) /* It is vital this test be done last */
505 continue;
506 total++;
507 }
508
509 /*
510 * Add any articles to arts[] that are new or were killed
511 */
512 if (total > 0) {
513 new_min = (getart_limit != 0 && last_read_article < min) ? min - 1 : last_read_article;
514
515 if ((i = read_art_headers(group, total, new_min)) == -1)
516 return FALSE; /* user aborted indexing */
517 else
518 changed += i;
519 }
520
521 #ifdef DEBUG
522 if (debug & DEBUG_NEWSRC) {
523 debug_print_comment("Before parse_unread_arts()");
524 debug_print_bitmap(group, NULL);
525 }
526 #endif /* DEBUG */
527 /*
528 * Do this before calling art_mark(,, ART_READ) if you want
529 * the unread count to be correct.
530 */
531 min = getart_limit > 0 ? min : T_ARTNUM_CONST(0);
532 parse_unread_arts(group, min);
533 #ifdef DEBUG
534 if (debug & DEBUG_NEWSRC) {
535 debug_print_comment("After parse_unread_arts()");
536 debug_print_bitmap(group, NULL);
537 }
538 #endif /* DEBUG */
539
540 /*
541 * Stat all articles to see if any have expired
542 */
543 for_each_art(i) {
544 if (arts[i].thread == ART_EXPIRED) {
545 changed++;
546 #ifdef DEBUG
547 if (debug & DEBUG_NEWSRC)
548 debug_print_comment("art.c: index_group() purging...");
549 #endif /* DEBUG */
550 art_mark(group, &arts[i], ART_READ);
551 if (group->attribute->show_only_unread_arts)
552 arts[i].keep_in_base = FALSE;
553 }
554 if (!path_in_nov && arts[i].path && *arts[i].path != '\0')
555 path_in_nov = TRUE;
556 }
557
558 /*
559 * Only rewrite the index if it has changed
560 * TODO review the exact logic behind "|| caching_xover"
561 */
562 if (changed || caching_xover || rebuild_cache)
563 write_overview(group);
564
565 /*
566 * Create the reference tree. The msgid and ref ptrs will
567 * be free()d now that the NovFile has been written.
568 */
569 build_references(group);
570
571 /*
572 * Needs access to the reference tree
573 */
574 filtered = filter_articles(group);
575
576 BegStopWatch("make_threads()");
577
578 /*
579 * Thread the group
580 */
581 make_threads(group, FALSE);
582
583 EndStopWatch();
584 PrintStopWatch();
585
586 if ((changed > 0 || filtered) && !batch_mode)
587 clear_message();
588
589 return TRUE;
590 }
591
592
593 /*
594 * Returns number of first unread article
595 */
596 static t_artnum
597 find_first_unread(
598 struct t_group *group)
599 {
600 unsigned char *p;
601 unsigned char *end = group->newsrc.xbitmap;
602 t_artnum first = group->newsrc.xmin; /* initial value */
603
604 if ((p = group->newsrc.xbitmap)) {
605 end += group->newsrc.xbitlen / NBITS;
606 for (; *p == '\0' && p < end; p++, first += NBITS)
607 ;
608 }
609 return first;
610 }
611
612
613 /*
614 * Open an article for reading just the header
615 * 'next' is used/updated with the next article number
616 * to optimise the number of 'HEAD' commands issued on
617 * groups with holes.
618 */
619 static FILE *
620 open_art_header(
621 char *groupname,
622 t_artnum art,
623 t_artnum *next)
624 {
625 char buf[NNTP_STRLEN];
626 #ifdef NNTP_ABLE
627 FILE *fp;
628 int i;
629
630 if (read_news_via_nntp && CURR_GROUP.type == GROUP_TYPE_NEWS) {
631 /*
632 * Don't bother requesting if we have not got there yet.
633 * This is a big win if the group has got holes in it (ie. if 000's
634 * of articles have expired between active files min & max values).
635 */
636 if (art < *next)
637 return NULL;
638
639 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
640 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
641 return fp;
642
643 /*
644 * TODO:
645 * shall we stop on 5xx?, i.e JamNNTPd/2 1.3 responds with
646 * "503 Access denied" instead of 480 but NEXT still works,
647 * so tin loops over all articles without getting useful data
648 */
649
650 /*
651 * HEAD failed, try to find NEXT
652 * Should return "223 artno message-id more text...."
653 */
654 i = new_nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf));
655 switch (i) {
656 case OK_NOTEXT:
657 *next = atoartnum(buf); /* Set next art number */
658 break;
659
660 # ifndef BROKEN_LISTGROUP
661 /*
662 * might happen if LISTGROUP doesn't select group, but
663 * we are not -DBROKEN_LISTGROUP
664 */
665 case ERR_NCING:
666 nntp_caps.broken_listgroup = TRUE;
667 snprintf(buf, sizeof(buf), "GROUP %s", groupname);
668 if (nntp_command(buf, OK_GROUP, NULL, 0) == NULL)
669 return NULL;
670 snprintf(buf, sizeof(buf), "HEAD %"T_ARTNUM_PFMT, art);
671 if ((fp = nntp_command(buf, OK_HEAD, NULL, 0)) != NULL)
672 return fp;
673 if (nntp_command("NEXT", OK_NOTEXT, buf, sizeof(buf)))
674 *next = atoartnum(buf);
675 break;
676 # endif /* !BROKEN_LISTGROUP */
677
678 default:
679 /*
680 * TODO: abort loop over all arts on ERR_NONEXT
681 */
682 # ifndef BROKEN_LISTGROUP
683 /*
684 * to avoid out of sync responses
685 * (listgroup seems to work, but didn't select new group,
686 * so xover seems to work but returns old data)
687 * we set listgroup_broken = TRUE; once we saw a
688 * ERR_NOARTIG / ERR_NONEXT or the like - even if
689 * ERR_NOARTIG may occur on servers where listgroup
690 * isn't broken...
691 */
692 nntp_caps.broken_listgroup = TRUE;
693 # endif /* !BROKEN_LISTGROUP */
694 break;
695 }
696
697 return NULL;
698 }
699 #endif /* NNTP_ABLE */
700
701 snprintf(buf, sizeof(buf), "%"T_ARTNUM_PFMT, art);
702 return (fopen(buf, "r"));
703 }
704
705
706 /*
707 * Called after XOVER/local/private overview databases have been loaded
708 * Read and parse in headers for any arts not already found (usually
709 * new articles that have not been indexed yet)
710 * Any new articles that are added have ->thread set to ART_UNTHREADED
711 * 'top' is the current highest artnum read
712 *
713 * Return values are:
714 * >0 Number of additional articles read in
715 * 0 No additional (new) articles were found
716 * -1 user aborted during read
717 */
718 static int
719 read_art_headers(
720 struct t_group *group,
721 int total,
722 t_artnum top)
723 {
724 FILE *fp;
725 char dir[PATH_LEN];
726 char group_msg[LEN];
727 int i;
728 int modified = 0;
729 t_artnum art;
730 t_artnum head_next = -1; /* Reset the next article number index (for when HEAD fails) */
731 t_bool res;
732
733 /*
734 * Change to groups spooldir to optimize fopen()'s on local articles
735 * NB open_art_header() requires this
736 */
737 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS) {
738 char buf[PATH_LEN];
739
740 get_cwd(dir);
741 make_base_group_path(group->spooldir, group->name, buf, sizeof(buf));
742 if (chdir(buf) != 0) {
743 #ifdef DEBUG
744 if (debug & DEBUG_MISC)
745 fprintf(stderr, "read_art_headers(chdir(%s))", strerror(errno));
746 #endif /* DEBUG */
747 return -1;
748 }
749 }
750
751 snprintf(group_msg, sizeof(group_msg), _(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
752
753 for (i = 0; i < grpmenu.max; i++) { /* for each article number */
754 art = base[i];
755
756 /*
757 * Skip articles that are below the low water mark or are
758 * already present
759 */
760 if (valid_artnum(art) >= 0)
761 continue;
762 if (art <= top)
763 continue;
764
765 /*
766 * Try and open the article
767 */
768 if ((fp = open_art_header(group->name, art, &head_next)) == NULL)
769 continue;
770
771 /*
772 * Add article to arts[]
773 */
774 if (top_art >= max_art)
775 expand_art();
776
777 set_article(&arts[top_art]);
778 arts[top_art].artnum = art;
779 arts[top_art].thread = ART_UNTHREADED;
780
781 res = parse_headers(fp, &arts[top_art]);
782
783 TIN_FCLOSE(fp);
784 if (tin_errno) {
785 modified = -1;
786 break;
787 }
788
789 if (!res) {
790 #ifdef DEBUG
791 if (debug & DEBUG_FILTER) { /* we currently have no "local spool" debug level */
792 char buf[PATH_LEN];
793
794 snprintf(buf, sizeof(buf), "FAILED parse_headers(%"T_ARTNUM_PFMT")", art);
795 debug_print_file("ARTS", "read_art_headers() %s", buf);
796 }
797 #endif /* DEBUG */
798 arts[top_art].artnum = T_ARTNUM_CONST(0);
799 arts[top_art].date = (time_t) 0;
800 FreeAndNull(arts[top_art].xref);
801 FreeAndNull(arts[top_art].path);
802 FreeAndNull(arts[top_art].refs);
803 FreeAndNull(arts[top_art].msgid);
804 if (arts[top_art].archive) {
805 FreeAndNull(arts[top_art].archive->partnum);
806 FreeAndNull(arts[top_art].archive);
807 }
808 arts[top_art].tagged = 0;
809 arts[top_art].thread = ART_EXPIRED;
810 arts[top_art].prev = ART_NORMAL;
811 arts[top_art].status = ART_UNREAD;
812 arts[top_art].killed = ART_NOTKILLED;
813 arts[top_art].selected = FALSE;
814 continue;
815 }
816
817 top = arts[top_art].artnum; /* used if arts are killed */
818 top_art++;
819
820 if (++modified % MODULO_COUNT_NUM == 0)
821 show_progress(group_msg, modified, total);
822 }
823
824 /*
825 * Change back to previous dir before indexing started
826 */
827 if (!read_news_via_nntp || group->type != GROUP_TYPE_NEWS)
828 chdir(dir);
829
830 return modified;
831 }
832
833
834 /*
835 * The algorithm is elegant, using the fact that identical Subject lines
836 * are hashed to the same node in table[] (see hashstr.c)
837 *
838 * Mark i as being in j's thread list if
839 * . The article is _not_ being ignored
840 * . The article is not already threaded
841 * . One of the following is true:
842 * 1) The subject lines are the same
843 * 2) Both are part of the same archive (name's match and arch bit set)
844 * IMHO the tests for archive name are redundant and have been for years
845 */
846 static void
847 thread_by_subject(
848 void)
849 {
850 int i, j;
851 struct t_hashnode *h;
852
853 for_each_art(i) {
854 if (IGNORE_ART_THREAD(i))
855 continue;
856
857 /*
858 * Get the contents of the magic marker in the hashnode
859 */
860 h = (struct t_hashnode *) (arts[i].subject - sizeof(int) - sizeof(void *)); /* FIXME: cast increases required alignment of target type */
861
862 j = h->aptr;
863
864 if (j != -1 && j < i) {
865 #if 1
866 if (arts[i].prev == ART_NORMAL && (arts[i].subject == arts[j].subject))
867 #else
868 /* see also refs.c:collate_subjects() */
869 if (arts[i].prev == ART_NORMAL && ((arts[i].subject == arts[j].subject) || (arts[i].archive && arts[j].archive && (arts[i].archive->name == arts[j].archive->name))))
870 #endif /* 1 */
871 {
872 arts[j].thread = i;
873 arts[i].prev = j;
874 }
875 }
876
877 /*
878 * Update the magic marker with the highest numbered mesg in
879 * arts[] that has been used in this thread so far
880 */
881 h->aptr = i;
882 }
883
884 #if 0
885 fprintf(stderr, "Subj dump\n");
886 fprintf(stderr, "%3s %3s %3s %3s : %3s %3s\n", "#", "Par", "Sib", "Chd", "In", "Thd");
887 for_each_art(i) {
888 fprintf(stderr, "%3d %3d %3d %3d : %3d %3d : %.50s %s\n", i,
889 (arts[i].refptr->parent) ? arts[i].refptr->parent->article : -2,
890 (arts[i].refptr->sibling) ? arts[i].refptr->sibling->article : -2,
891 (arts[i].refptr->child) ? arts[i].refptr->child->article : -2,
892 arts[i].prev, arts[i].thread, arts[i].refptr->txt, arts[i].subject);
893 }
894 #endif /* 0 */
895 }
896
897
898 /*
899 * This Threading algorithm threads articles into 'buckets' where each bucket
900 * contains all the articles which match the root article's subject line to
901 * the configured percentage. Eg, if the root article had the subject "asdf"
902 * and the match percentage was configured to be 75% then any article would
903 * match if its subject was no different in more than a single character.
904 */
905 static void
906 thread_by_percentage(
907 struct t_group *group)
908 {
909 int i, j, k;
910 int root_num = 0; /* The index number of the root we are currently working on. */
911 unsigned int unmatched; /* This is the number of characters that don't match between the two strings */
912 unsigned int percentage = 100 - group->attribute->thread_perc;
913 size_t slen;
914
915 /* First we need to sort art[] to simplify and speed up the matching. */
916 SortBy(subj_comp_asc);
917
918 /*
919 * Now we put all the articles which match enough into the thread. If
920 * an article doesn't match enough we create a new thread and then add
921 * to that and so on.
922 */
923 base[0] = 0;
924 arts[0].prev = ART_NORMAL;
925 for_each_art(i) {
926 if (i == 0)
927 continue;
928
929 /* Check each character to see if it matched enough */
930 k = 0;
931 unmatched = 0;
932 for (j = 0; arts[base[root_num]].subject[j] != '\0' && arts[i].subject[k] != '\0'; j++, k++) {
933 if (arts[base[root_num]].subject[j] != arts[i].subject[k])
934 unmatched++;
935 }
936
937 /*
938 * By getting here we have a number of unmatched characters
939 * between the two strings. We also have the length of the
940 * strings available to us easily.
941 * All we need to do is see if the match is good enough, but
942 * we count differences in the length of the strings against
943 * them matching.
944 */
945 if (!(slen = strlen(arts[base[root_num]].subject)))
946 slen++;
947 unmatched += slen - strlen(arts[i].subject);
948 if (unmatched * 100 / slen > percentage) {
949 /*
950 * If there is less greater than percentage% different start a
951 * new thread.
952 */
953 base[++root_num] = i;
954 arts[i].prev = ART_NORMAL;
955 continue;
956 } else {
957 /*
958 * The subject lines match enough to consider them part of a single
959 * thread, so add the current article to the thread.
960 */
961 if (arts[base[root_num]].thread < 0)
962 arts[base[root_num]].thread = i;
963 arts[i].prev = i - 1;
964 arts[i - 1].thread = i;
965 continue;
966 }
967 }
968 }
969
970
971 /*
972 * This was brought over from tags.c, however this version doesn't
973 * operate on base_index
974 *
975 * Parses a subject header of the type "multipart message subject (01/42)"
976 * into a MultiPartInfo struct, or fails if the message subject isn't in the
977 * right form.
978 *
979 * @return nonzero on success
980 */
981 int
982 global_get_multipart_info(
983 int aindex,
984 MultiPartInfo *setme)
985 {
986 int i, j, offi, offj;
987 MultiPartInfo setmei, setmej;
988
989 i = global_look_for_multipart_info(aindex, &setmei, '[', ']', &offi);
990 j = global_look_for_multipart_info(aindex, &setmej, '(', ')', &offj);
991
992 /* Ok i hits first */
993 if (offi > offj) {
994 *setme = setmei;
995 return i;
996 }
997
998 /* Its j or they are both the same (which must be zero!) so we don't care */
999 *setme = setmej;
1000 return j;
1001 }
1002
1003
1004 /*
1005 * Again this was taken from tags.c, but works on global indices into arts
1006 * rather than on base_index.
1007 */
1008 static int
1009 global_look_for_multipart_info(
1010 int aindex,
1011 MultiPartInfo* setme,
1012 char start,
1013 char stop,
1014 int *offset)
1015 {
1016 char *subj;
1017 char *pch;
1018 MultiPartInfo tmp;
1019
1020 *offset = 0;
1021
1022 /* entry assertions */
1023 assert(0 <= aindex && aindex < top_art && "invalid index");
1024 assert(setme != NULL && "setme must not be NULL");
1025
1026 /* parse the message */
1027 subj = arts[aindex].subject;
1028 pch = strrchr(subj, start);
1029 if (!pch || !isdigit((int) pch[1]))
1030 return 0;
1031
1032 tmp.base_index = aindex; /* This could be confusing because we are actually storing the index into arts, not the base_index. */
1033 tmp.subject_compare_len = pch - subj;
1034 tmp.part_number = (int) strtol(pch + 1, &pch, 10);
1035 if (*pch != '/' && *pch != '|')
1036 return 0;
1037
1038 if (!isdigit((int) pch[1]))
1039 return 0;
1040
1041 tmp.total = (int) strtol(pch + 1, &pch, 10);
1042 if (*pch != stop)
1043 return 0;
1044
1045 tmp.subject = subj;
1046 *setme = tmp;
1047 *offset = pch - subj;
1048 return 1;
1049 }
1050
1051
1052 /*
1053 * Taken from tags.c but changed to use indices into arts[] instead of
1054 * base_index. Changed so that even when we don't have all the parts we
1055 * return a valid array.
1056 *
1057 * Tries to find all the parts to the multipart message pointed to by
1058 * base_index.
1059 *
1060 * @return on success, the number of parts found. On failure, zero if not a
1061 * multipart or the negative value of the first missing part.
1062 * @param base_index index pointing to one of the messages in a multipart
1063 * message.
1064 * @param malloc_and_setme_info on success, set to a malloced array the
1065 * parts found.
1066 */
1067 static int
1068 global_get_multiparts(
1069 int aindex,
1070 MultiPartInfo **malloc_and_setme_info)
1071 {
1072 int i;
1073 int part_index;
1074 MultiPartInfo tmp, tmp2;
1075 MultiPartInfo *info = NULL;
1076
1077 /* entry assertions */
1078 assert(0 <= aindex && aindex < top_art && "Invalid index");
1079 assert(malloc_and_setme_info != NULL && "malloc_and_setme_info must not be NULL");
1080
1081 /* make sure this is a multipart message... */
1082 if (!global_get_multipart_info(aindex, &tmp) || tmp.total < 1)
1083 return 0;
1084
1085 /* make a temporary buffer to hold the multipart info... */
1086 info = my_malloc(sizeof(MultiPartInfo) * tmp.total);
1087
1088 /* zero out part-number for the repost check below */
1089 for (i = 0; i < tmp.total; ++i) {
1090 info[i].total = tmp.total; /* Added this for thread_by_multipart */
1091 info[i].part_number = -1;
1092 }
1093
1094 /* try to find all the multiparts... */
1095 for_each_art(i) {
1096 if (strncmp(arts[i].subject, tmp.subject, tmp.subject_compare_len))
1097 continue;
1098
1099 if (!global_get_multipart_info(i, &tmp2))
1100 continue;
1101
1102 /* 'test (1/5)' is not the same as 'test (1/15)' */
1103 if (tmp.total != tmp2.total)
1104 continue;
1105
1106 part_index = tmp2.part_number - 1;
1107
1108 /*
1109 * skip "blah (00/102)" or "blah (103/102)" subjects
1110 */
1111 if (part_index < 0 || part_index >= tmp.total)
1112 continue;
1113
1114 /* repost check: do we already have this part? */
1115 if (info[part_index].part_number != -1) {
1116 assert(info[part_index].part_number == tmp2.part_number && "bookkeeping error");
1117 continue;
1118 }
1119
1120 /* we have a match, hooray! */
1121 info[part_index] = tmp2;
1122 }
1123
1124 /* see if we got them all. */
1125 for (i = 0; i < tmp.total; ++i) {
1126 if (info[i].part_number != i + 1) {
1127 *malloc_and_setme_info = info;
1128 return -(i + 1); /* missing part #(i+1) */
1129 }
1130 }
1131
1132 /* looks like a success .. */
1133 *malloc_and_setme_info = info;
1134 return tmp.total;
1135 }
1136
1137
1138 /*
1139 * The algorithm uses the tag multipart searches to thread articles together.
1140 */
1141 static void
1142 thread_by_multipart(
1143 void)
1144 {
1145 int i, j, threadNum;
1146 MultiPartInfo *minfo = NULL;
1147
1148 for_each_art(i) {
1149 if (IGNORE_ART_THREAD(i) || arts[i].prev >= 0 || !global_get_multiparts(i, &minfo)) {
1150 FreeAndNull(minfo);
1151 continue;
1152 }
1153
1154 threadNum = -1;
1155 for (j = minfo[0].total - 1; j >= 0; j--) {
1156 if (minfo[j].part_number != -1) {
1157 if (threadNum != -1) {
1158 arts[minfo[j].base_index].thread = threadNum;
1159 arts[threadNum].prev = minfo[j].base_index;
1160 }
1161 threadNum = minfo[j].base_index;
1162 }
1163 }
1164 FreeAndNull(minfo);
1165 }
1166 }
1167
1168
1169 /*
1170 * Go through the articles in arts[] and create threads. There are
1171 * 5 strategies currently defined :
1172 *
1173 * THREAD_NONE No threading
1174 * THREAD_SUBJ Threads are created using like Subject lines
1175 * THREAD_REFS Threads are created using the References headers
1176 * THREAD_BOTH Threads created using References and then Subject
1177 * THREAD_MULTI Threads created using Subject to search for Multiparts
1178 * THREAD_PERC Threads based upon a char for char match of greater than x%
1179 *
1180 * .thread and .prev are used to hold the threading information, see tin.h for
1181 * more information
1182 * Only process valid (unexpired) articles we haven't visited yet
1183 * (ie arts[].thread == ART_UNTHREADED)
1184 *
1185 * The rethread parameter is used to force the deletion of existing threading
1186 * information before threading which happens anyway expect when using
1187 * THREAD_NONE (I don't immediately see how this is useful)
1188 */
1189 /* TODO: rewrite that user can easily combine different 'threading'
1190 * methods, i.e:
1191 * - thread_by_multipart() + collate_subjects()
1192 */
1193 void
1194 make_threads(
1195 struct t_group *group,
1196 t_bool rethread)
1197 {
1198 if (!cmd_line && !batch_mode)
1199 info_message((group->attribute->thread_articles == THREAD_NONE ? _(txt_unthreading_arts) : _(txt_threading_arts)));
1200
1201 #ifdef DEBUG
1202 if (debug & DEBUG_MISC)
1203 error_message(2, "rethread=[%d] thread_articles=[%d] attr_thread_articles=[%d]",
1204 rethread, tinrc.thread_articles, group->attribute->thread_articles);
1205 #endif /* DEBUG */
1206
1207 /*
1208 * Sort all the articles using the preferred method
1209 * When find_base() is called, the bases are created ordered
1210 * on arts[] and so the base messages under all threading systems
1211 * will be sorted in this way.
1212 */
1213 sort_arts(group->attribute->sort_article_type);
1214
1215 /*
1216 * Reset all the ptrs to articles following the above sort
1217 */
1218 clear_art_ptrs();
1219
1220 /*
1221 * The threading pointers need to be reset if re-threading
1222 * If using ref threading, revector the links back to the articles
1223 */
1224 if (rethread || group->attribute->thread_articles) {
1225 int i;
1226
1227 for_each_art(i) {
1228 if (arts[i].thread >= 0)
1229 arts[i].thread = ART_UNTHREADED;
1230
1231 arts[i].prev = ART_NORMAL;
1232
1233 /* Should never happen if tree is built properly */
1234 if (arts[i].refptr == NULL) {
1235 #ifdef DEBUG
1236 if (debug & DEBUG_REFS) {
1237 my_fprintf(stderr, "\nError : art->refptr is NULL\n");
1238 my_fprintf(stderr, "Artnum : %"T_ARTNUM_PFMT"\n", arts[i].artnum);
1239 my_fprintf(stderr, "Subject: %s\n", arts[i].subject);
1240 my_fprintf(stderr, "From : %s\n", arts[i].from);
1241 assert(arts[i].refptr != NULL);
1242 } else
1243 #endif /* DEBUG */
1244 continue;
1245 }
1246 arts[i].refptr->article = i;
1247 }
1248 }
1249
1250 /*
1251 * Do the right thing according to the threading strategy
1252 */
1253 switch (group->attribute->thread_articles) {
1254 case THREAD_NONE:
1255 break;
1256
1257 case THREAD_SUBJ:
1258 thread_by_subject();
1259 break;
1260
1261 case THREAD_REFS:
1262 thread_by_reference();
1263 break;
1264
1265 case THREAD_BOTH:
1266 thread_by_reference();
1267 collate_subjects();
1268 break;
1269
1270 case THREAD_MULTI:
1271 thread_by_multipart();
1272 break;
1273
1274 case THREAD_PERC:
1275 thread_by_percentage(group);
1276 break;
1277
1278 default: /* not reached */
1279 break;
1280 }
1281
1282 /*
1283 * Rebuild base[]
1284 */
1285 find_base(group);
1286 }
1287
1288
1289 static t_compfunc
1290 eval_sort_arts_func(
1291 unsigned int sort_art_type)
1292 {
1293 switch (sort_art_type) {
1294 case SORT_ARTICLES_BY_NOTHING: /* don't sort at all */
1295 return artnum_comp;
1296
1297 case SORT_ARTICLES_BY_SUBJ_DESCEND:
1298 return subj_comp_desc;
1299
1300 case SORT_ARTICLES_BY_SUBJ_ASCEND:
1301 return subj_comp_asc;
1302
1303 case SORT_ARTICLES_BY_FROM_DESCEND:
1304 return from_comp_desc;
1305
1306 case SORT_ARTICLES_BY_FROM_ASCEND:
1307 return from_comp_asc;
1308
1309 case SORT_ARTICLES_BY_DATE_DESCEND:
1310 return date_comp_desc;
1311
1312 case SORT_ARTICLES_BY_DATE_ASCEND:
1313 return date_comp_asc;
1314
1315 case SORT_ARTICLES_BY_SCORE_DESCEND:
1316 return score_comp_desc;
1317
1318 case SORT_ARTICLES_BY_SCORE_ASCEND:
1319 return score_comp_asc;
1320
1321 case SORT_ARTICLES_BY_LINES_DESCEND:
1322 return lines_comp_desc;
1323
1324 case SORT_ARTICLES_BY_LINES_ASCEND:
1325 return lines_comp_asc;
1326
1327 default:
1328 break;
1329 }
1330 return NULL;
1331 }
1332
1333
1334 void
1335 sort_arts(
1336 unsigned int sort_art_type)
1337 {
1338 t_compfunc comp_func = eval_sort_arts_func(sort_art_type);
1339
1340 if (comp_func)
1341 SortBy(comp_func);
1342 }
1343
1344
1345 static void
1346 sort_base(
1347 unsigned int sort_threads_type)
1348 {
1349 switch (sort_threads_type) {
1350 case SORT_THREADS_BY_SCORE_DESCEND:
1351 case SORT_THREADS_BY_SCORE_ASCEND:
1352 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), score_comp_base);
1353 break;
1354
1355 case SORT_THREADS_BY_LAST_POSTING_DATE_DESCEND:
1356 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_desc);
1357 break;
1358
1359 case SORT_THREADS_BY_LAST_POSTING_DATE_ASCEND:
1360 tin_sort(base, (size_t) grpmenu.max, sizeof(t_artnum), last_date_comp_base_asc);
1361 break;
1362 }
1363 }
1364
1365
1366 /*
1367 * This is called to get header info for articles not already found in the
1368 * overview files.
1369 * Code reads (max_lineno) lines of article to catch headers like Archive-name:
1370 * which are not normally included in XOVER or even the normal block of headers.
1371 * How this is supposed to be useful when 99% of the time we'll have overview
1372 * data I don't know...
1373 * TODO: move Archive-name: parsing to article body parsing, remove the
1374 * TODO: max_lineno nonsense and parse just the hdrs. Only parse if
1375 * TODO: currgrp->auto_save is set, otherwise it is redundant info
1376 */
1377 static t_bool
1378 parse_headers(
1379 FILE *fp,
1380 struct t_article *h)
1381 {
1382 char art_from_addr[HEADER_LEN];
1383 char art_full_name[HEADER_LEN];
1384 char *s, *hdr, *ptr;
1385 unsigned int lineno = 0;
1386 unsigned int max_lineno = 25;
1387 t_bool got_from, got_lines, got_received;
1388
1389 got_from = got_lines = got_received = FALSE;
1390
1391 while ((ptr = tin_fgets(fp, TRUE)) != NULL) {
1392 /*
1393 * Look for the end of information which tin wants to get.
1394 * Applies when reading local spool and via NNTP.
1395 */
1396
1397 /*
1398 * as Archive-name: is placed in the body it's safe to exit
1399 * the loop if it was found. Don't mix up with Archive: from
1400 * RFC 5536.
1401 */
1402 if (lineno++ > max_lineno || h->archive)
1403 break;
1404
1405 unfold_header(ptr);
1406 switch (my_toupper((unsigned char) *ptr)) {
1407 case 'A': /* Archive-name: optional */
1408 /*
1409 * Archive-name: {name}/{part|patch}{number}
1410 * eg, acorn/faq/part01
1411 */
1412 if ((hdr = parse_header(ptr + 1, "rchive-name", FALSE, FALSE, FALSE))) {
1413 if ((s = strrchr(hdr, '/')) != NULL) {
1414 struct t_archive *archptr = my_malloc(sizeof(struct t_archive));
1415
1416 if (STRNCASECMPEQ(s + 1, "part", 4)) {
1417 archptr->partnum = my_strdup(s + 5);
1418 archptr->ispart = TRUE;
1419 } else if (STRNCASECMPEQ(s + 1, "patch", 5)) {
1420 archptr->partnum = my_strdup(s + 6);
1421 archptr->ispart = FALSE;
1422 } else { /* part or patch must be present */
1423 free(archptr);
1424 continue;
1425 }
1426 strtok(archptr->partnum, "\n");
1427 *s = '\0';
1428 archptr->name = hash_str(hdr);
1429 h->archive = archptr;
1430 }
1431 }
1432 break;
1433
1434 case 'D': /* Date: mandatory */
1435 if (!h->date) {
1436 if ((hdr = parse_header(ptr + 1, "ate", FALSE, FALSE, FALSE)))
1437 h->date = parsedate(hdr, (struct _TIMEINFO *) 0);
1438 }
1439 break;
1440
1441 case 'F': /* From: mandatory */
1442 if (!got_from) {
1443 if ((hdr = parse_header(ptr + 1, "rom", FALSE, FALSE, FALSE))) {
1444 h->gnksa_code = parse_from(hdr, art_from_addr, art_full_name);
1445 h->from = hash_str(buffer_to_ascii(art_from_addr));
1446 if (*art_full_name)
1447 h->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
1448 got_from = TRUE;
1449 }
1450 }
1451 break;
1452
1453 case 'L': /* Lines: optional */
1454 if (!got_lines) {
1455 if ((hdr = parse_header(ptr + 1, "ines", FALSE, FALSE, FALSE))) {
1456 h->line_count = atoi(hdr);
1457 got_lines = TRUE;
1458 }
1459 }
1460 break;
1461
1462 case 'M': /* Message-ID: mandatory */
1463 if (!h->msgid) {
1464 if ((hdr = parse_header(ptr + 1, "essage-ID", FALSE, FALSE, FALSE)))
1465 h->msgid = my_strdup(hdr);
1466 }
1467 break;
1468
1469 /* for Path:-filter when reading from local spool */
1470 case 'P': /* Path: */
1471 if (!h->path) {
1472 if ((hdr = parse_header(ptr + 1, "ath", FALSE, FALSE, FALSE)))
1473 h->path = my_strdup(hdr);
1474 }
1475 break;
1476
1477 case 'R': /* References: optional */
1478 if (!h->refs) {
1479 if ((hdr = parse_header(ptr + 1, "eferences", FALSE, FALSE, FALSE)))
1480 h->refs = my_strdup(hdr);
1481 }
1482
1483 /* Received: If found it's probably a mail article */
1484 if (!got_received) {
1485 if (parse_header(ptr + 1, "eceived", FALSE, FALSE, FALSE)) {
1486 max_lineno <<= 1; /* double the max number of line to read for mails */
1487 got_received = TRUE;
1488 }
1489 }
1490 break;
1491
1492 case 'S': /* Subject: mandatory */
1493 if (!h->subject) {
1494 if ((hdr = parse_header(ptr + 1, "ubject", FALSE, FALSE, FALSE))) {
1495 #ifdef HAVE_UNICODE_NORMALIZATION
1496 if (IS_LOCAL_CHARSET("UTF-8"))
1497 s = normalize(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1498 else
1499 #endif /* HAVE_UNICODE_NORMALIZATION */
1500 s = my_strdup(eat_re(convert_to_printable(rfc1522_decode(hdr), FALSE), FALSE));
1501
1502 h->subject = hash_str(s);
1503 free(s);
1504 }
1505 }
1506 break;
1507
1508 case 'X': /* Xref: optional */
1509 if (!h->xref) {
1510 if ((hdr = parse_header(ptr + 1, "ref", FALSE, FALSE, FALSE)))
1511 h->xref = my_strdup(hdr);
1512 }
1513 break;
1514
1515 default:
1516 break;
1517 } /* switch */
1518
1519 } /* while */
1520
1521 #ifdef NNTP_ABLE
1522 if (ptr)
1523 drain_buffer(fp);
1524 #endif /* NNTP_ABLE */
1525
1526 if (tin_errno)
1527 return FALSE;
1528
1529 /*
1530 * The son of RFC 1036 states that the following hdrs are mandatory. It
1531 * also states that Subject, Newsgroups and Path are too. Ho hum.
1532 *
1533 * What about reading mail from local spool via ~/.tin/active.mail,
1534 * they might not have a Message-ID but got_received is very likely to
1535 * be true.
1536 */
1537 if (got_from && h->date && h->msgid) {
1538 if (!h->subject)
1539 h->subject = hash_str("<No subject>");
1540
1541 #ifdef DEBUG
1542 debug_print_header(h);
1543 #endif /* DEBUG */
1544 return TRUE;
1545 }
1546
1547 return FALSE;
1548 }
1549
1550 #ifdef NNTP_ABLE
1551 /*
1552 * Loop over arts[] and find ranges without Path: header
1553 * If there are any try to optimize the ranges regarding traffic consumption
1554 * Start optimization if at least MIN_CNT ranges exist
1555 * If there are more than MAX_CNT ranges after optimization, fetch all in one
1556 * big range
1557 */
1558 #define MIN_CNT 10
1559 #define MAX_CNT 50
1560 static struct t_article_range *
1561 build_range_list(
1562 t_artnum min,
1563 t_artnum max,
1564 int *range_cnt)
1565 {
1566 int i, gap_cnt = 0;
1567 struct t_article_range *res = NULL, *gap_list, *curr, *from;
1568 t_artnum new_end;
1569
1570 new_end = T_ARTNUM_CONST(0);
1571 gap_list = my_malloc(sizeof(struct t_article_range));
1572 curr = gap_list;
1573 curr->start = min;
1574 curr->end = max;
1575 curr->cnt = T_ARTNUM_CONST(0);
1576 curr->next = NULL;
1577
1578 for_each_art(i) {
1579 if (arts[i].artnum < min)
1580 continue;
1581 if (arts[i].artnum > max)
1582 break;
1583 if (arts[i].path) {
1584 for (; i < top_art && arts[i].path; i++)
1585 ;
1586 /*
1587 * the current art has no path -> we use this one
1588 * if we reached top_art all arts have path
1589 * so we use max
1590 */
1591 curr->start = i == top_art ? max : arts[i--].artnum;
1592 } else {
1593 for (; i < top_art && !arts[i].path; i++)
1594 ;
1595 /* the current art has path -> we use the last one */
1596 new_end = curr->end = arts[--i].artnum;
1597 }
1598 if (new_end) {
1599 curr->cnt = curr->end - curr->start + 1;
1600 curr->next = my_malloc(sizeof(struct t_article_range));
1601 curr = curr->next;
1602 curr->start = new_end;
1603 curr->end = max;
1604 curr->cnt = T_ARTNUM_CONST(0);
1605 curr->next = NULL;
1606 new_end = T_ARTNUM_CONST(0);
1607 }
1608 }
1609
1610 curr = gap_list;
1611 while (curr && curr->cnt) {
1612 ++gap_cnt;
1613 # ifdef DEBUG
1614 if ((debug & DEBUG_NNTP) && verbose > 1)
1615 debug_print_file("NNTP", "range #%d without path in overview cache: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1616 # endif /* DEBUG */
1617 curr = curr->next;
1618 }
1619
1620 /*
1621 * Optimize only if there are at least MIN_CNT ranges
1622 */
1623 if (gap_cnt >= MIN_CNT) {
1624 res = my_malloc(sizeof(struct t_article_range));
1625 res->start = T_ARTNUM_CONST(0);
1626 res->end = T_ARTNUM_CONST(0);
1627 res->cnt = T_ARTNUM_CONST(0);
1628 res->next = NULL;
1629
1630 from = gap_list;
1631 curr = res;
1632 while (from) {
1633 curr->start = from->start;
1634 curr->end = from->end;
1635 curr->cnt = from->cnt;
1636 if ((from = from->next)) {
1637 /*
1638 * If the next range is grater then the gap between the current
1639 * one and the next one we build a new range including the
1640 * current one, the next one and the gap between
1641 */
1642 while (from && from->cnt >= from->start - curr->end - 1) {
1643 curr->end = from->end;
1644 from = from->next;
1645 }
1646 curr->cnt = curr->end - curr->start + 1;
1647 curr->next = my_malloc(sizeof(struct t_article_range));
1648 curr = curr->next;
1649 curr->start = T_ARTNUM_CONST(0);
1650 curr->end = T_ARTNUM_CONST(0);
1651 curr->cnt = T_ARTNUM_CONST(0);
1652 curr->next = NULL;
1653 }
1654 }
1655 }
1656
1657 /*
1658 * If there are less then MIN_CNT ranges
1659 * no res is build -> return the original list
1660 */
1661 if (res) {
1662 while (gap_list) {
1663 curr = gap_list;
1664 gap_list = curr->next;
1665 free(curr);
1666 }
1667 } else
1668 res = gap_list;
1669
1670 curr = res;
1671 gap_cnt = 0;
1672 while (curr && curr->cnt) {
1673 ++gap_cnt;
1674 # ifdef DEBUG
1675 if ((debug & DEBUG_NNTP) && verbose > 1)
1676 debug_print_file("NNTP", "optimized range #%d: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", gap_cnt, curr->start, curr->end, curr->cnt);
1677 # endif /* DEBUG */
1678 curr = curr->next;
1679 }
1680
1681 if (gap_cnt >= MAX_CNT) {
1682 curr = res;
1683 while (curr->next && curr->next->cnt) {
1684 res->end = curr->next->end;
1685 curr->next->cnt = 0;
1686 curr = curr->next;
1687 }
1688 res->cnt = res->end - res->start + 1;
1689 gap_cnt = 1;
1690 # ifdef DEBUG
1691 if ((debug & DEBUG_NNTP) && verbose > 1) {
1692 debug_print_file("NNTP", "more then %d ranges after optimization, fetch all at once instead: start: %"T_ARTNUM_PFMT" end: %"T_ARTNUM_PFMT" cnt: %"T_ARTNUM_PFMT"", MAX_CNT, res->start, res->end, res->cnt);
1693 }
1694 # endif /* DEBUG */
1695 }
1696 *range_cnt = gap_cnt;
1697
1698 return res;
1699 }
1700
1701
1702 /*
1703 * Fetch the Path header in case we want to filter on that in the given group
1704 *
1705 * Try [X]HDR first, then XPAT
1706 */
1707 static t_bool
1708 get_path_header(
1709 int cur,
1710 int cnt,
1711 struct t_group *group,
1712 t_artnum min,
1713 t_artnum max)
1714 {
1715 FILE *fp = NULL;
1716 char *prep_msg;
1717 char *buf, *ptr;
1718 char cmd[NNTP_STRLEN];
1719 t_artnum artnum, i;
1720 t_bool found = FALSE;
1721 static t_bool supported = TRUE; /* assume HDR || XPAT works */
1722
1723 if (!read_news_via_nntp || !supported || group->type != GROUP_TYPE_NEWS)
1724 return FALSE;
1725
1726 # ifdef DEBUG
1727 if ((debug & DEBUG_NNTP) && verbose > 1)
1728 debug_print_file("NNTP", "%s: Filtering on Path header requested.", group->name);
1729 # endif /* DEBUG */
1730
1731 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers && !*nntp_caps.headers_range && nntp_caps.hdr_cmd[0] != 'X') {
1732 int j = new_nntp_command("LIST HEADERS RANGE", 215, cmd, sizeof(cmd));
1733 switch (j) {
1734 case 215:
1735 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
1736 # ifdef DEBUG
1737 if (debug & DEBUG_NNTP)
1738 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
1739 # endif /* DEBUG */
1740 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
1741 strcat(nntp_caps.headers_range, ptr);
1742 strcat(nntp_caps.headers_range, "\n");
1743 }
1744 break;
1745
1746 default:
1747 break;
1748 }
1749 }
1750
1751 /* does HDR return Path? */
1752 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n")) != NULL) {
1753 do {
1754 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Path", 4))
1755 found = TRUE;
1756 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
1757 }
1758
1759 if ((nntp_caps.hdr || nntp_caps.hdr_cmd) && (!(nntp_caps.type == CAPABILITIES) || found)) {
1760 if (min == max)
1761 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min);
1762 else
1763 snprintf(cmd, sizeof(cmd), "%s Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, max);
1764 fp = nntp_command(cmd, nntp_caps.hdr_cmd[0] == 'X' ? OK_XHDR : OK_HDR, NULL, 0);
1765 if (!nntp_caps.hdr && fp)
1766 nntp_caps.hdr = TRUE;
1767 } else if (nntp_caps.xpat) {
1768 if (min == max)
1769 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT" *", min);
1770 else
1771 snprintf(cmd, sizeof(cmd), "XPAT Path %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT" *", min, max);
1772 fp = nntp_command(cmd, OK_XPAT, NULL, 0);
1773 }
1774
1775 if (fp) {
1776 prep_msg = fmt_string(_(txt_prep_for_filter_on_path), cur, cnt);
1777 while ((buf = tin_fgets(fp, FALSE)) != NULL && buf[0] != '.') {
1778 # ifdef DEBUG
1779 if ((debug & DEBUG_NNTP) && verbose)
1780 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1781 # endif /* DEBUG */
1782 if ((ptr = tin_strtok(buf, " ")) == NULL)
1783 continue;
1784 artnum = atoartnum(ptr);
1785 if ((ptr = tin_strtok(NULL, " ")) == NULL)
1786 continue;
1787 for_each_art(i) {
1788 if (arts[i].artnum == artnum) {
1789 FreeIfNeeded(arts[i].path);
1790 arts[i].path = my_strdup(ptr);
1791 }
1792 }
1793 if (++artnum % MODULO_COUNT_NUM == 0)
1794 show_progress(prep_msg, artnum - min, max - min);
1795 }
1796 free(prep_msg);
1797 return supported;
1798 }
1799
1800 /* !fp */
1801 supported = FALSE;
1802 if (nntp_caps.xpat)
1803 nntp_caps.xpat = FALSE;
1804 /* as nntp_caps.hdr may work with other headers we don't disable it*/
1805
1806 # ifdef DEBUG
1807 if ((debug & DEBUG_NNTP) && verbose > 1)
1808 debug_print_file("NNTP", "%s: Neither \"[X]HDR Path\" nor \"XPAT Path\" are supported.", group->name);
1809 # endif /* DEBUG */
1810 return supported;
1811 }
1812 #endif /* NNTP_ABLE */
1813
1814
1815 /*
1816 * Read in an overview index file. Fields are separated by TAB.
1817 * return the number of expired articles encountered or -1 if the user aborted
1818 * the read
1819 * 'top' is set to the highest artnum read
1820 * If 'local' is set then always open local overview cache in preference to
1821 * using NNTP XOVER
1822 *
1823 * Format (mandatory as far as line count [RFC2980]):
1824 * 1. article number (ie. 183) [mandatory]
1825 * 2. Subject: line (ie. Which newsreader?) [mandatory]
1826 * 3. From: line (ie. iain@ecrc.de) [mandatory]
1827 * 4. Date: line (rfc822 format) [mandatory]
1828 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
1829 * 6. References: (ie. <message-id> ....) [optional]
1830 * 7. Byte count (Skipped - not used) [mandatory]
1831 * 8. Line count (ie. 23) [mandatory]
1832 * 9. Xref: line (ie. alt.test:389) [optional]
1833 */
1834 static int
1835 read_overview(
1836 struct t_group *group,
1837 t_artnum min,
1838 t_artnum max,
1839 t_artnum *top,
1840 t_bool local,
1841 t_bool *rebuild_cache)
1842 {
1843 FILE *fp;
1844 char *ptr;
1845 char *q;
1846 char *buf;
1847 char *group_msg;
1848 char art_full_name[HEADER_LEN];
1849 char art_from_addr[HEADER_LEN];
1850 unsigned int count;
1851 int expired = 0;
1852 t_artnum artnum;
1853 t_bool path_found = FALSE, path_in_ofmt = FALSE;
1854 struct t_article *art;
1855 size_t over_fields = 1;
1856
1857 /*
1858 * open the overview file (whether it be local or via nntp)
1859 */
1860 if ((fp = open_xover_fp(group, "r", min, max, local)) == NULL)
1861 return expired;
1862
1863 if (group->xmax > max)
1864 group->xmax = max;
1865
1866 group_msg = fmt_string(_(txt_group), cCOLS - MIN(cCOLS - 1, strwidth(_(txt_group))) + 2 - 3, group->name);
1867
1868 /* get the number of fields per over-record as announced by LIST OVERVIEW.FMT */
1869 if (ofmt) {
1870 for (; ofmt[over_fields].name; over_fields++) {
1871 if (local && !path_in_ofmt && !strcasecmp(ofmt[over_fields].name, "Path:"))
1872 path_in_ofmt = TRUE;
1873 }
1874 }
1875
1876 if (!--over_fields) { /* e.g. nntp_caps.type == CAPABILITIES && !nntp_caps.list_overview_fmt -> assume defaults */
1877 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (8 + 1));
1878 ofmt[0].type = OVER_T_INT;
1879 ofmt[0].name = my_strdup("Artnum:");
1880 ofmt[1].type = OVER_T_STRING;
1881 ofmt[1].name = my_strdup("Subject:");
1882 ofmt[2].type = OVER_T_STRING;
1883 ofmt[2].name = my_strdup("From:");
1884 ofmt[3].type = OVER_T_STRING;
1885 ofmt[3].name = my_strdup("Date:");
1886 ofmt[4].type = OVER_T_STRING;
1887 ofmt[4].name = my_strdup("Message-ID:");
1888 ofmt[5].type = OVER_T_STRING;
1889 ofmt[5].name = my_strdup("References:");
1890 ofmt[6].type = OVER_T_INT;
1891 ofmt[6].name = my_strdup("Bytes:");
1892 ofmt[7].type = OVER_T_INT;
1893 ofmt[7].name = my_strdup("Lines:");
1894 ofmt[8].type = OVER_T_ERROR;
1895 ofmt[8].name = NULL;
1896 over_fields = 7;
1897 }
1898
1899 while ((buf = tin_fgets(fp, FALSE)) != NULL) {
1900 #ifdef DEBUG
1901 if ((debug & DEBUG_NNTP) && fp == FAKE_NNTP_FP && verbose)
1902 debug_print_file("NNTP", "<<<%s%s", logtime(), buf);
1903 #endif /* DEBUG */
1904
1905 if (need_resize) {
1906 handle_resize((need_resize == cRedraw) ? TRUE : FALSE);
1907 need_resize = cNo;
1908 }
1909
1910 /*
1911 * Read artnum
1912 */
1913 if ((ptr = tin_strtok(buf, "\t")) == NULL)
1914 continue;
1915
1916 /*
1917 * read the article number, guaranteed to be the first field
1918 */
1919 artnum = atoartnum(ptr);
1920
1921 /*
1922 * artnum field invalid/corrupt or is 1st line of local cached overview
1923 * (group name)
1924 */
1925 if (artnum <= 0)
1926 continue;
1927
1928 /*
1929 * skip artnums below the given minimum (getart_limit)
1930 */
1931 if (artnum < min)
1932 continue;
1933
1934 /*
1935 * Check to make sure article in nov file has not expired in group
1936 */
1937 if (artnum < group->xmin) {
1938 expired++;
1939 continue;
1940 }
1941
1942 /*
1943 * artnum in overview data higher than groups high mark
1944 *
1945 * TODO: - warn user about broken overviews?
1946 * - try to parse the Xref:-line to get the correct artnum
1947 * - see also parse_unread_arts()
1948 */
1949 if (artnum > group->xmax)
1950 continue;
1951
1952 if (top_art >= max_art)
1953 expand_art();
1954
1955 art = &arts[top_art];
1956 set_article(art);
1957 art->artnum = *top = artnum;
1958
1959 /*
1960 * Note: Fields after line count are not mandatory, use "LIST OVERVIEW.FMT"
1961 * to check for additions like we do with xref_supported
1962 */
1963 for (count = 1; (ptr = tin_strtok(NULL, "\t")) != NULL; count++) {
1964 /* skip unexpected tailing fields */
1965 if (count > over_fields) {
1966 #ifdef DEBUG
1967 if ((debug & DEBUG_NNTP) && verbose > 1)
1968 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") Unexpected overview-field %d of %d: %s", nntp_caps.over_cmd, artnum, count, over_fields, ptr);
1969 #endif /* DEBUG */
1970
1971 /* "common error" Xref:full in overview-data but not in OVERVIEW.FMT */
1972 if (count == over_fields + 1) {
1973 if (!strncasecmp(ptr, "Xref: ", 6)) {
1974 #ifdef DEBUG
1975 if ((debug & DEBUG_NNTP) && verbose > 1)
1976 debug_print_file("NNTP", "%s: found unexpected Xref: on semi std. position", nntp_caps.over_cmd);
1977 #endif /* DEBUG */
1978 over_fields++;
1979 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1980 ofmt[over_fields].type = OVER_T_FSTRING;
1981 ofmt[over_fields].name = my_strdup("Xref:");
1982 ofmt[over_fields + 1].type = OVER_T_ERROR;
1983 ofmt[over_fields + 1].name = NULL;
1984 xref_supported = TRUE;
1985 } else if (local && !strncasecmp(ptr, "Path: ", 6)) {
1986 #ifdef DEBUG
1987 if ((debug & DEBUG_NNTP) && verbose > 1)
1988 debug_print_file("NNTP", "%s: found Path:", nntp_caps.over_cmd);
1989 #endif /* DEBUG */
1990 over_fields++;
1991 ofmt = my_realloc(ofmt, sizeof(struct t_overview_fmt) * (over_fields + 2)); /* + 2 = artnum and end-marker */
1992 ofmt[over_fields].type = OVER_T_FSTRING;
1993 ofmt[over_fields].name = my_strdup("Path:");
1994 ofmt[over_fields + 1].type = OVER_T_ERROR;
1995 ofmt[over_fields + 1].name = NULL;
1996 xref_supported = TRUE;
1997 } else
1998 continue;
1999 } else
2000 continue;
2001 }
2002
2003 /* for duplicated headers this is last match counts, INN >= 2.5.3 does first match counts */
2004 if (expensive_over_parse) { /* strange order */
2005 /* mandatory fields */
2006 if (ofmt[count].type == OVER_T_STRING) {
2007 if (!strcasecmp(ofmt[count].name, "Subject:")) {
2008 if (*ptr) {
2009 #ifdef HAVE_UNICODE_NORMALIZATION
2010 if (IS_LOCAL_CHARSET("UTF-8"))
2011 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2012 else
2013 #endif /* HAVE_UNICODE_NORMALIZATION */
2014 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2015
2016 art->subject = hash_str(q);
2017 free(q);
2018 } else {
2019 art->subject = hash_str("");
2020 #ifdef DEBUG
2021 if ((debug & DEBUG_NNTP) && verbose > 1)
2022 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2023 #endif /* DEBUG */
2024 }
2025 continue;
2026 }
2027
2028 if (!strcasecmp(ofmt[count].name, "From:")) {
2029 if (*ptr) {
2030 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2031 art->from = hash_str(buffer_to_ascii(art_from_addr));
2032 if (*art_full_name)
2033 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2034 } else {
2035 art->from = hash_str("");
2036 #ifdef DEBUG
2037 if ((debug & DEBUG_NNTP) && verbose > 1)
2038 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2039 #endif /* DEBUG */
2040 }
2041 continue;
2042 }
2043
2044 if (!strcasecmp(ofmt[count].name, "Date:")) {
2045 art->date = parsedate(ptr, (TIMEINFO *) 0);
2046 #ifdef DEBUG
2047 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2048 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2049 #endif /* DEBUG */
2050 continue;
2051 }
2052
2053 if (!strcasecmp(ofmt[count].name, "Message-ID:")) {
2054 if (*ptr) {
2055 FreeIfNeeded(art->msgid); /* if field is listed more than once in overview.fmt */
2056 art->msgid = my_strdup(ptr);
2057 } else {
2058 art->msgid = NULL;
2059 #ifdef DEBUG
2060 if ((debug & DEBUG_NNTP) && verbose > 1)
2061 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2062 #endif /* DEBUG */
2063 }
2064 continue;
2065 }
2066
2067 if (!strcasecmp(ofmt[count].name, "References:")) {
2068 if (*ptr) {
2069 FreeIfNeeded(art->refs); /* if field is listed more than once in overview.fmt */
2070 art->refs = my_strdup(ptr);
2071 } else
2072 art->refs = NULL;
2073 continue;
2074 }
2075
2076 /*
2077 * non std. fields when doing
2078 * expensive overview parsing (very
2079 * rare, just happens if RFC 3977
2080 * 8.4.2 is violated) go here
2081 */
2082 /* for Path:-filter */
2083 if (!strcasecmp(ofmt[count].name, "Path:")) {
2084 if (!path_found)
2085 path_found = TRUE;
2086 if (*ptr) {
2087 FreeIfNeeded(art->path); /* if field is listed more than once in overview.fmt */
2088 art->path = my_strdup(ptr);
2089 } else
2090 art->path = NULL;
2091 continue;
2092 }
2093 }
2094 /* metadata fields */
2095 if (ofmt[count].type == OVER_T_INT) {
2096 if (!strcasecmp(ofmt[count].name, "Bytes:")) {
2097 if (*ptr) {
2098 #ifdef DEBUG
2099 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2100 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2101 #endif /* DEBUG */
2102 }
2103 continue;
2104 }
2105
2106 if (!strcasecmp(ofmt[count].name, "Lines:")) {
2107 if (*ptr) {
2108 if (isdigit((unsigned char) *ptr))
2109 art->line_count = atoi(ptr);
2110 else {
2111 art->line_count = 0;
2112 #ifdef DEBUG
2113 if ((debug & DEBUG_NNTP) && verbose > 1)
2114 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2115 #endif /* DEBUG */
2116 }
2117 } else
2118 art->line_count = 0;
2119 continue;
2120 }
2121 }
2122 } else { /* first 7 fields are in RFC 3977 order */
2123 switch (count) {
2124 case 1: /* Subject: */
2125 if (*ptr) {
2126 #ifdef HAVE_UNICODE_NORMALIZATION
2127 if (IS_LOCAL_CHARSET("UTF-8"))
2128 q = normalize(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2129 else
2130 #endif /* HAVE_UNICODE_NORMALIZATION */
2131 q = my_strdup(eat_re(eat_tab(convert_to_printable(rfc1522_decode(ptr), FALSE)), FALSE));
2132
2133 art->subject = hash_str(q);
2134 free(q);
2135 }
2136 else {
2137 art->subject = hash_str("");
2138 #ifdef DEBUG
2139 if ((debug & DEBUG_NNTP) && verbose > 1)
2140 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2141 #endif /* DEBUG */
2142 }
2143 break;
2144
2145 case 2: /* From: */
2146 if (*ptr) {
2147 art->gnksa_code = parse_from(ptr, art_from_addr, art_full_name);
2148 art->from = hash_str(buffer_to_ascii(art_from_addr));
2149 if (*art_full_name)
2150 art->name = hash_str(eat_tab(convert_to_printable(rfc1522_decode(art_full_name), FALSE)));
2151 } else {
2152 art->from = hash_str("");
2153 #ifdef DEBUG
2154 if ((debug & DEBUG_NNTP) && verbose > 1)
2155 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2156 #endif /* DEBUG */
2157 }
2158 break;
2159
2160 case 3: /* Date: */
2161 art->date = parsedate(ptr, (TIMEINFO *) 0);
2162 #ifdef DEBUG
2163 if ((debug & DEBUG_NNTP) && verbose > 1 && art->date == (time_t) -1)
2164 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2165 #endif /* DEBUG */
2166 break;
2167
2168 case 4: /* Message-ID: */
2169 if (*ptr)
2170 art->msgid = my_strdup(ptr);
2171 else {
2172 art->msgid = NULL;
2173 #ifdef DEBUG
2174 if ((debug & DEBUG_NNTP) && verbose > 1)
2175 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") empty overview-field %s", nntp_caps.over_cmd, artnum, ofmt[count].name);
2176 #endif /* DEBUG */
2177 }
2178 break;
2179
2180 case 5: /* References: */
2181 if (*ptr)
2182 art->refs = my_strdup(ptr);
2183 else
2184 art->refs = NULL;
2185 break;
2186
2187 case 6: /* :bytes || Bytes: */
2188 if (*ptr) {
2189 #ifdef DEBUG
2190 if ((debug & DEBUG_NNTP) && verbose > 1 && !isdigit((unsigned char) *ptr))
2191 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2192 #endif /* DEBUG */
2193 }
2194 break;
2195
2196 case 7: /* :lines || Lines: */
2197 if (*ptr) {
2198 if (isdigit((unsigned char) *ptr))
2199 art->line_count = atoi(ptr);
2200 else {
2201 art->line_count = 0;
2202 #ifdef DEBUG
2203 if ((debug & DEBUG_NNTP) && verbose > 1)
2204 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") overview field %d (%s) mismatch: %s", nntp_caps.over_cmd, artnum, count, ofmt[count].name, ptr);
2205 #endif /* DEBUG */
2206 }
2207 } else
2208 art->line_count = 0;
2209 break;
2210
2211 default:
2212 break;
2213 }
2214 }
2215
2216 /* optional fields; for duplicated headers: last match counts, INN >= 2.5.3 does first match counts */
2217 if (ofmt[count].type == OVER_T_FSTRING) {
2218 if (*ptr) {
2219 if (!strcasecmp(ofmt[count].name, "Xref:")) {
2220 if ((q = parse_header(ptr, "Xref", FALSE, FALSE, FALSE)) != NULL) {
2221 FreeIfNeeded(art->xref); /* if field is listed more than once in overview.fmt */
2222 art->xref = my_strdup(q);
2223 }
2224 #ifdef DEBUG
2225 else {
2226 if ((debug & DEBUG_NNTP) && verbose > 1)
2227 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") bogus overview-field %s %s", nntp_caps.over_cmd, artnum, ofmt[count].name, ptr);
2228 }
2229 #endif /* DEBUG */
2230 continue;
2231 }
2232 /*
2233 * handling of addition overview fields
2234 * goes here
2235 */
2236 #ifdef DEBUG
2237 if ((debug & DEBUG_NNTP) && verbose > 1)
2238 debug_print_file("NNTP", "%s(%"T_ARTNUM_PFMT") extra overview-field \"%s\" at position %d %s", nntp_caps.over_cmd, artnum, ofmt[count].name, count, ptr);
2239 #endif /* DEBUG */
2240 /* if we're lucky we've Path in NOV */
2241 /*
2242 * if reading locally cached overview data try
2243 * path regardless of the server OVERVIEW.FMT
2244 */
2245 if (local || !strcasecmp(ofmt[count].name, "Path:")) {
2246 if ((q = parse_header(ptr, "Path", FALSE, FALSE, FALSE)) != NULL) {
2247 if (!path_found)
2248 path_found = TRUE;
2249 FreeIfNeeded(art->path);
2250 art->path = my_strdup(q);
2251 #ifdef DEBUG
2252 if ((debug & DEBUG_NNTP) && verbose > 1 && strcasecmp(ofmt[count].name, "Path:"))
2253 debug_print_file("NNTP", "\tUsing as \"Path:\" not \"%s\"", ofmt[count].name);
2254 #endif /* DEBUG */
2255
2256 }
2257 continue;
2258 }
2259 }
2260 continue;
2261 }
2262 }
2263
2264 /*
2265 * RFC says Message-ID is mandatory in newsgroups (but not in
2266 * mailgroups etc..) NB. a NULL Message-ID would abort if we ever do
2267 * threading in mailgroups
2268 */
2269 if (!art->msgid && group->type == GROUP_TYPE_NEWS)
2270 continue;
2271
2272 /* we might lose accuracy here, but that shouldn't hurt */
2273 if (artnum % MODULO_COUNT_NUM == 0)
2274 show_progress(group_msg, artnum - min, max - min);
2275
2276 top_art++; /* Basically this statement commits the article */
2277 }
2278
2279 free(group_msg);
2280 TIN_FCLOSE(fp);
2281
2282 if (tin_errno)
2283 return -1;
2284
2285 #if defined(NNTP_ABLE) && defined(XHDR_XREF)
2286 if (read_news_via_nntp && !read_saved_news && !xref_supported && nntp_caps.hdr_cmd) {
2287 char cbuf[HEADER_LEN];
2288 int i;
2289 static t_bool found;
2290 static t_bool first = TRUE;
2291
2292 if (first) {
2293 found = TRUE;
2294 /*
2295 * TODO: if "LIST HEADERS RANGE" failed try "LIST HEADERS"?
2296 */
2297 if (nntp_caps.type == CAPABILITIES && nntp_caps.list_headers) {
2298 if (!*nntp_caps.headers_range) {
2299 i = new_nntp_command("LIST HEADERS RANGE", 215, cbuf, sizeof(cbuf));
2300
2301 found = FALSE;
2302 switch (i) {
2303 case 215:
2304 while ((ptr = tin_fgets(FAKE_NNTP_FP, FALSE)) != NULL) {
2305 # ifdef DEBUG
2306 if (debug & DEBUG_NNTP)
2307 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2308 # endif /* DEBUG */
2309 if (!found && ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4)))
2310 found = TRUE;
2311 nntp_caps.headers_range = my_realloc(nntp_caps.headers_range, strlen(nntp_caps.headers_range) + strlen(ptr) + 2);
2312 strcat(nntp_caps.headers_range, ptr);
2313 strcat(nntp_caps.headers_range, "\n");
2314 }
2315 break;
2316
2317 default:
2318 break;
2319 }
2320 first = FALSE;
2321 } else {
2322 found = FALSE;
2323 if (nntp_caps.headers_range && (ptr = strtok(nntp_caps.headers_range, "\n" )) != NULL) {
2324 do {
2325 if ((*ptr == ':' && *(ptr + 1) == '\0') || !strncasecmp(ptr, "Xref", 4))
2326 found = TRUE;
2327 } while (!found && *ptr && (ptr = strtok(NULL, "\n")) != NULL);
2328 }
2329 }
2330 }
2331 }
2332
2333 if (found) {
2334 snprintf(cbuf, sizeof(cbuf), "%s XREF %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.hdr_cmd, min, MAX(min, max));
2335 group_msg = fmt_string("%s XREF loop", nntp_caps.hdr_cmd); /* TODO: find a better message, move to lang.c */
2336 if ((fp = nntp_command(cbuf, nntp_caps.hdr ? OK_HDR : OK_HEAD, NULL, 0)) != NULL) { /* RFC 2980 (XHDR) uses 221; RFC 3977 (HDR) uses 225 */
2337 while ((ptr = tin_fgets(fp, FALSE)) != NULL) {
2338 # ifdef DEBUG
2339 if ((debug & DEBUG_NNTP) && verbose)
2340 debug_print_file("NNTP", "<<<%s%s", logtime(), ptr);
2341 # endif /* DEBUG */
2342
2343 artnum = atoartnum(ptr);
2344 if (artnum <= 0 || artnum < group->xmin || artnum > group->xmax)
2345 continue;
2346 art = &arts[top_art];
2347 set_article(art);
2348 if (!art->xref && !strstr(ptr, "(none)")) {
2349 if ((q = strchr(ptr, ' ')) == NULL) /* skip article number */
2350 continue;
2351 ptr = q;
2352 while (*ptr && isspace((int) *ptr))
2353 ptr++;
2354 q = strchr(ptr, '\n');
2355 if (q)
2356 *q = '\0';
2357 art->xref = my_strdup(ptr);
2358 }
2359 /* we might lose accuracy here, but that shouldn't hurt */
2360 if (artnum % MODULO_COUNT_NUM == 0)
2361 show_progress(group_msg, artnum - min, max - min);
2362 }
2363 }
2364 free(group_msg);
2365 }
2366 }
2367 #endif /* NNTP_ABLE && XHDR_XREF */
2368
2369 if (local) {
2370 #ifdef NNTP_ABLE
2371 if (filter_on_path(group)) {
2372 struct t_article_range *ranges, *curr;
2373 t_bool supported = TRUE;
2374 int curr_range, range_cnt;
2375
2376 /*
2377 * Get the ranges without Path: header and try to fetch the
2378 * headers
2379 */
2380 if ((ranges = build_range_list(min, *top, &range_cnt))) {
2381 curr = ranges;
2382 curr_range = 1;
2383 while (curr && supported) {
2384 if (curr->cnt)
2385 supported = get_path_header(curr_range++, range_cnt, group, curr->start, curr->end);
2386 curr = curr->next;
2387 }
2388 if (!supported && path_in_ofmt) {
2389 /*
2390 * fetching Path: headers via [X]HDR or XPAT has failed
2391 * Path: is in the servers overview so let the next
2392 * read_overview() fetch them
2393 */
2394 free_art_array();
2395 free_msgids();
2396 top_art = 0;
2397 *top = T_ARTNUM_CONST(0);
2398 expired = 0;
2399 }
2400 *rebuild_cache = TRUE;
2401 while (ranges) {
2402 curr = ranges;
2403 ranges = curr->next;
2404 free(curr);
2405 }
2406 }
2407 }
2408 #endif /* NNTP_ABLE */
2409 } else
2410 if (!path_found && filter_on_path(group)) {
2411 #ifdef NNTP_ABLE
2412 if (!get_path_header(1, 1, group, min, *top))
2413 #endif /* NNTP_ABLE */
2414 wait_message(2, _(txt_cannot_filter_on_path));
2415 }
2416 return expired;
2417 }
2418
2419
2420 /*
2421 * Write an Nov/Xover index file. Fields are separated by '\t'.
2422 *
2423 * Format:
2424 * 1. article number (ie. 183) [mandatory]
2425 * 2. Subject: line (ie. Which newsreader?) [mandatory]
2426 * 3. From: line (ie. iain@ecrc.de) [mandatory]
2427 * 4. Date: line (rfc822 format) [mandatory]
2428 * 5. MessageID: (ie. <123@ether.net>) [mandatory]
2429 * 6. References: (ie. <message-id> ....) [optional]
2430 * 7. Byte count (Skipped - not used) [mandatory]
2431 * 8. Line count (ie. 23) [mandatory]
2432 * 9. Xref: line (ie. alt.test:389) [optional]
2433 *
2434 * TODO: as we don't use the original data, we currently can't store
2435 * the data (from/subject) in the original charset (we don't store
2436 * that info). this has the advantage that we can avoid raw 8bit data
2437 * in our overviews, but the disadvantage that we might store the data
2438 * with a wrong charset and thus lose information. a similar problem
2439 * exists with the data for the from:-line, we don't store it in the
2440 * original format, whenever our from-parser (partially) fails we'll
2441 * lose information in our overviews (but those couldn't be handled
2442 * by tin anyway, so this is not a real problem).
2443 * long-term solution: store the original data in the overview
2444 * (tin has to handle raw 8bit data and other ugly stuff in the
2445 * overviews anyway and thus we preserver as much info as possible)
2446 * this would require some changes in read_overview() and
2447 * parse_headers(): don't do the decoding/unfolding there, but in a
2448 * second pass right after write_overview(), or two additional fields
2449 * which hold the raw data for from/subject. the latter has the
2450 * disadvantage that it costs (much) more memory.
2451 */
2452 static void
2453 write_overview(
2454 struct t_group *group)
2455 {
2456 FILE *fp;
2457 int i;
2458 struct t_article *article;
2459 #ifdef CHARSET_CONVERSION
2460 int c = -1;
2461 #endif /* CHARSET_CONVERSION */
2462
2463 /*
2464 * Can't write or caching is off or getart_limit is set
2465 */
2466 if (no_write || !tinrc.cache_overview_files || ((cmdline.args & CMDLINE_GETART_LIMIT) ? cmdline.getart_limit : tinrc.getart_limit) != 0)
2467 return;
2468
2469 if ((fp = open_xover_fp(group, "w", T_ARTNUM_CONST(0), T_ARTNUM_CONST(0), FALSE)) == NULL)
2470 return;
2471
2472 if (group->attribute->sort_article_type != SORT_ARTICLES_BY_NOTHING)
2473 SortBy(artnum_comp);
2474
2475 /*
2476 * Needed to preserve uniqueness in hashed private overview files
2477 */
2478 fprintf(fp, "%s\n", group->name);
2479
2480 #ifdef CHARSET_CONVERSION
2481 /* get undeclared_charset number if required */
2482 if (group->attribute->undeclared_charset) {
2483 for (i = 0; txt_mime_charsets[i] != NULL; i++) {
2484 if (!strcasecmp(group->attribute->undeclared_charset, txt_mime_charsets[i])) {
2485 c = i;
2486 break;
2487 }
2488 }
2489 }
2490 #endif /* CHARSET_CONVERSION */
2491
2492 for_each_art(i) {
2493 char *p;
2494 char *q, *ref;
2495
2496 article = &arts[i];
2497
2498 if (article->thread != ART_EXPIRED && article->artnum >= group->xmin) {
2499 ref = NULL;
2500
2501 if (!group->attribute->post_8bit_header) { /* write encoded data */
2502 /*
2503 * TODO: instead of tinrc.mm_local_charset we'd better use UTF-8
2504 * here and in print_from() in the CHARSET_CONVERSION case.
2505 * note that this requires something like
2506 * buffer_to_network(article->subject, "UTF-8");
2507 * right before the rfc1522_encode() call.
2508 *
2509 * if we would cache the original undecoded data, we could
2510 * ignore stuff like this.
2511 */
2512 p = rfc1522_encode(article->subject, tinrc.mm_local_charset, FALSE);
2513 /* as the subject might now be folded we have to unfold it */
2514 unfold_header(p);
2515 } else { /* raw data */
2516 p = my_strdup(article->subject);
2517 #ifdef CHARSET_CONVERSION
2518 if (group->attribute->undeclared_charset && c != -1) /* use undeclared_charset if set (otherwise local charset is used) */
2519 buffer_to_network(p, c);
2520 #endif /* CHARSET_CONVERSION */
2521 }
2522
2523 /*
2524 * replace any '\t's with ' ' in the references-data
2525 *
2526 * TODO: nntpext-draft might come up with a new scheme:
2527 * For all fields, the value is processed by first
2528 * removing all US-ASCII CRLF pairs and then replacing
2529 * each remaining US-ASCII NUL, TAB, CR, or LF character
2530 * with a single US-ASCII space (for example, CR LF LF TAB
2531 * will become two spaces).
2532 */
2533 if (article->refs) {
2534 ref = q = my_strdup(article->refs);
2535 while (*q) {
2536 if (*q == '\t')
2537 *q = ' ';
2538 q++;
2539 }
2540 }
2541
2542 fprintf(fp, "%"T_ARTNUM_PFMT"\t%s\t%s\t%s\t%s\t%s\t%d\t%d",
2543 article->artnum,
2544 p,
2545 #ifdef CHARSET_CONVERSION
2546 print_from(group, article, c),
2547 #else
2548 print_from(group, article, -1),
2549 #endif /* CHARSET_CONVERSION */
2550 print_date(article->date),
2551 BlankIfNull(article->msgid),
2552 BlankIfNull(ref),
2553 0, /* bytes */
2554 article->line_count);
2555
2556 if (article->xref)
2557 fprintf(fp, "\tXref: %s", article->xref);
2558
2559 if (article->path)
2560 fprintf(fp, "\tPath: %s", article->path);
2561
2562 fprintf(fp, "\n");
2563 free(p);
2564 if (article->refs) {
2565 FreeIfNeeded(ref);
2566 q = NULL;
2567 }
2568 }
2569 }
2570 fchmod(fileno(fp), (mode_t) (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH));
2571 fclose(fp);
2572 }
2573
2574
2575 /*
2576 * A complex little function to determine the correct overview index file
2577 * according to 'mode' (read or write)
2578 * NULL is returned if the current setup dictates otherwise
2579 *
2580 * GROUP_TYPE_MAIL index files are read/written in ~/.tin/.mail
2581 * GROUP_TYPE_SAVE index files are read/written in ~/.tin/.save
2582 *
2583 * Both of these are hashed
2584 *
2585 * GROUP_TYPE_NEWS index files are a little bit more complex
2586 *
2587 * When hashing the index filename will be in format number.number.
2588 * Hashing the groupname gets a number. See if that #.1 file exists;
2589 * if so, read first line. Is this the group we want? If no, try #.2.
2590 * Repeat until no such file or we find an existing file that matches
2591 * our group. Return pointer to path or NULL if not found.
2592 */
2593 static char *
2594 find_nov_file(
2595 struct t_group *group,
2596 int mode)
2597 {
2598 FILE *fp;
2599 const char *dir;
2600 char buf[PATH_LEN];
2601 int i;
2602 struct stat sb;
2603 unsigned long hash;
2604 static char nov_file[PATH_LEN];
2605 static t_bool once_only = FALSE; /* Trap things that are done only 1 time */
2606
2607 if (group == NULL || (mode != R_OK && mode != W_OK))
2608 return NULL;
2609
2610 switch (group->type) {
2611 case GROUP_TYPE_MAIL:
2612 dir = index_maildir;
2613 break;
2614
2615 case GROUP_TYPE_SAVE:
2616 dir = index_savedir;
2617 break;
2618
2619 case GROUP_TYPE_NEWS:
2620 /*
2621 * nntp.caps.over_cmd is not an issue here, any gripes and warnings
2622 * about [X]OVER are handled in nntp_open()
2623 */
2624
2625 /*
2626 * When reading via NNTP, system wide overviews are irrelevant, of
2627 * course, and the private overview filename will be the same for
2628 * both reading and writing.
2629 *
2630 * When working locally, we only use a private cache for reading
2631 * if requested and when system wide overviews don't already exist.
2632 * When writing then only private overviews can be used since
2633 * updating system wide overviews is not safe wrt locking etc.
2634 *
2635 * See if local overview file $SPOOLDIR/<groupname>/.overview exists
2636 */
2637 #ifndef NNTP_ONLY
2638 if (!read_news_via_nntp) {
2639 make_base_group_path(novrootdir, group->name, buf, sizeof(buf));
2640 joinpath(nov_file, sizeof(nov_file), buf, novfilename);
2641 if (access(nov_file, R_OK) == 0) {
2642 if (mode == R_OK)
2643 return nov_file; /* Use system wide overviews */
2644 else
2645 return NULL; /* Don't write cache in this case */
2646 }
2647 }
2648 #endif /* !NNTP_ONLY */
2649
2650 /*
2651 * We only get here when private overviews are going to be used
2652 * Go no further if they are explicitly turned off
2653 */
2654 if (!tinrc.cache_overview_files)
2655 return NULL;
2656
2657 /*
2658 * Append -<nntpserver> to private cache dir
2659 */
2660 if (!once_only && nntp_server) {
2661 size_t sp, ln = strlen(index_newsdir);
2662
2663 if ((sp = sizeof(index_newsdir) - ln - 1) >= 2) {
2664 char *srv = my_strdup(nntp_server);
2665
2666 strcat(index_newsdir, "-");
2667 sp--;
2668 ln++;
2669 str_lwr(srv);
2670 my_strncpy(index_newsdir + ln, srv, sp);
2671 free(srv);
2672 }
2673 once_only = TRUE;
2674 }
2675
2676 /*
2677 * Only try to set up the private cache when writing. If it
2678 * doesn't exist yet, then ergo we can't read from it.
2679 * The cache will be checked/created on every write; a previous
2680 * bug report complained that this was not the case
2681 */
2682 if (stat(index_newsdir, &sb) == -1) { /* Private cache doesn't exist */
2683 if (mode == R_OK)
2684 return NULL;
2685 if (my_mkdir(index_newsdir, (mode_t) S_IRWXU) != 0)
2686 return NULL;
2687 } else {
2688 if (!S_ISDIR(sb.st_mode))
2689 return NULL;
2690 }
2691
2692 /*
2693 * Update the newsgroups cache to point to the new location
2694 * now that we know it is valid
2695 */
2696 if (!once_only)
2697 joinpath(local_newsgroups_file, sizeof(local_newsgroups_file), index_newsdir, NEWSGROUPS_FILE);
2698
2699 dir = index_newsdir;
2700 break;
2701
2702 default: /* not reached */
2703 return NULL;
2704 }
2705
2706 /*
2707 * We only get here if writing to a private overview.
2708 * These always have hashed filenames.
2709 * Try <hash>.<seqno> and check the group name tagline until
2710 * matching index file is found. If not found return next unused
2711 * filename
2712 */
2713 hash = hash_groupname(group->name);
2714
2715 for (i = 1; ; i++) {
2716 char *ptr;
2717
2718 snprintf(buf, sizeof(buf), "%lu.%d", hash, i);
2719 joinpath(nov_file, sizeof(nov_file), dir, buf);
2720
2721 if ((fp = fopen(nov_file, "r")) == NULL)
2722 break;
2723
2724 /*
2725 * No group name header, so not a valid index file => overwrite it
2726 */
2727 if (fgets(buf, (int) sizeof(buf), fp) == NULL) {
2728 fclose(fp);
2729 break;
2730 }
2731 fclose(fp);
2732
2733 if ((ptr = strrchr(buf, '\n')) != NULL)
2734 *ptr = '\0';
2735
2736 if (strcmp(buf, group->name) == 0)
2737 break;
2738 }
2739
2740 return nov_file;
2741 }
2742
2743
2744 /*
2745 * Run the index file updater only for the groups we've loaded.
2746 */
2747 void
2748 do_update(
2749 t_bool catchup)
2750 {
2751 int i, j, k = 0;
2752 time_t beg_epoch = 0;
2753 struct t_article *art;
2754 struct t_group *group;
2755
2756 if (verbose)
2757 (void) time(&beg_epoch);
2758
2759 /*
2760 * loop through groups and update any required index files
2761 */
2762 for (i = 0; i < selmenu.max; i++) {
2763 group = &active[my_group[i]];
2764 /*
2765 * FIXME: workaround to get a valid CURR_GROUP
2766 * it also points to the currently processed group so that
2767 * the correct attributes are used
2768 * The correct fix is to get rid of CURR_GROUP
2769 */
2770 selmenu.curr = i;
2771
2772 if (group->bogus || !group->subscribed)
2773 continue;
2774
2775 if (!index_group(group)) {
2776 for_each_art(j) {
2777 art = &arts[j];
2778 FreeAndNull(art->refs);
2779 FreeAndNull(art->msgid);
2780 }
2781 continue;
2782 }
2783
2784 k++;
2785
2786 if (verbose) {
2787 my_printf("%s %s\n", (catchup ? _(txt_catchup) : _(txt_updating)), group->name);
2788 my_flush();
2789 }
2790
2791 if (catchup) {
2792 for_each_art(j)
2793 art_mark(group, &arts[j], ART_READ);
2794 }
2795 }
2796
2797 if (verbose) {
2798 wait_message(0, _(txt_catchup_update_info),
2799 (catchup ? _(txt_caughtup) : _(txt_updated)), k,
2800 PLURAL(selmenu.max, txt_group), (unsigned long int) (time(NULL) - beg_epoch));
2801 }
2802 }
2803
2804
2805 static int
2806 artnum_comp(
2807 t_comptype p1,
2808 t_comptype p2)
2809 {
2810 const struct t_article *s1 = (const struct t_article *) p1;
2811 const struct t_article *s2 = (const struct t_article *) p2;
2812
2813 /*
2814 * s1->artnum less than s2->artnum
2815 */
2816 if (s1->artnum < s2->artnum)
2817 return -1;
2818
2819 /*
2820 * s1->artnum greater than s2->artnum
2821 */
2822 if (s1->artnum > s2->artnum)
2823 return 1;
2824
2825 return 0;
2826 }
2827
2828
2829 /*
2830 * return result of strcmp (reversed for descending)
2831 */
2832 static int
2833 subj_comp_asc(
2834 t_comptype p1,
2835 t_comptype p2)
2836 {
2837 int retval;
2838 const struct t_article *s1 = (const struct t_article *) p1;
2839 const struct t_article *s2 = (const struct t_article *) p2;
2840
2841 if ((retval = strcasecmp(s1->subject, s2->subject))) /* != 0 */
2842 return retval;
2843
2844 return s1->date - s2->date > 0 ? 1 : -1;
2845 }
2846
2847
2848 static int
2849 subj_comp_desc(
2850 t_comptype p1,
2851 t_comptype p2)
2852 {
2853 int retval;
2854 const struct t_article *s1 = (const struct t_article *) p1;
2855 const struct t_article *s2 = (const struct t_article *) p2;
2856
2857 if ((retval = strcasecmp(s2->subject, s1->subject))) /* != 0 */
2858 return retval;
2859
2860 return s1->date - s2->date > 0 ? 1 : -1;
2861 }
2862
2863
2864 /*
2865 * return result of strcmp (reversed for descending)
2866 */
2867 static int
2868 from_comp_asc(
2869 t_comptype p1,
2870 t_comptype p2)
2871 {
2872 int retval;
2873 const struct t_article *s1 = (const struct t_article *) p1;
2874 const struct t_article *s2 = (const struct t_article *) p2;
2875
2876 if ((retval = strcasecmp(s1->from, s2->from))) /* != 0 */
2877 return retval;
2878
2879 return s1->date - s2->date > 0 ? 1 : -1;
2880 }
2881
2882
2883 static int
2884 from_comp_desc(
2885 t_comptype p1,
2886 t_comptype p2)
2887 {
2888 int retval;
2889 const struct t_article *s1 = (const struct t_article *) p1;
2890 const struct t_article *s2 = (const struct t_article *) p2;
2891
2892 if ((retval = strcasecmp(s2->from, s1->from))) /* != 0 */
2893 return retval;
2894
2895 return s1->date - s2->date > 0 ? 1 : -1;
2896 }
2897
2898
2899 /*
2900 * Works like strcmp() for comparing time_t type values
2901 * Return codes:
2902 * -1: If p1 is before p2
2903 * 0: If they are the same time
2904 * 1: If p1 is after p2
2905 * If the sort order is _not_ DATE_ASCEND then the sense of the above
2906 * is reversed.
2907 */
2908 static int
2909 date_comp_asc(
2910 t_comptype p1,
2911 t_comptype p2)
2912 {
2913 const struct t_article *s1 = (const struct t_article *) p1;
2914 const struct t_article *s2 = (const struct t_article *) p2;
2915
2916 /*
2917 * s1->date less than s2->date
2918 */
2919 if (s1->date < s2->date)
2920 return -1;
2921
2922 /*
2923 * s1->date greater than s2->date
2924 */
2925 if (s1->date > s2->date)
2926 return 1;
2927
2928 return 0;
2929 }
2930
2931
2932 static int
2933 date_comp_desc(
2934 t_comptype p1,
2935 t_comptype p2)
2936 {
2937 const struct t_article *s1 = (const struct t_article *) p1;
2938 const struct t_article *s2 = (const struct t_article *) p2;
2939
2940 /*
2941 * s2->date less than s1->date
2942 */
2943 if (s2->date < s1->date)
2944 return -1;
2945
2946 /*
2947 * s2->date greater than s1->date
2948 */
2949 if (s2->date > s1->date)
2950 return 1;
2951
2952 return 0;
2953 }
2954
2955
2956 /*
2957 * Same again, but for art[].score
2958 */
2959 static int
2960 score_comp_asc(
2961 t_comptype p1,
2962 t_comptype p2)
2963 {
2964 const struct t_article *s1 = (const struct t_article *) p1;
2965 const struct t_article *s2 = (const struct t_article *) p2;
2966
2967 if (s1->score < s2->score)
2968 return -1;
2969
2970 if (s1->score > s2->score)
2971 return 1;
2972
2973 return s1->date - s2->date > 0 ? 1 : -1;
2974 }
2975
2976
2977 static int
2978 score_comp_desc(
2979 t_comptype p1,
2980 t_comptype p2)
2981 {
2982 const struct t_article *s1 = (const struct t_article *) p1;
2983 const struct t_article *s2 = (const struct t_article *) p2;
2984
2985 if (s2->score < s1->score)
2986 return -1;
2987
2988 if (s2->score > s1->score)
2989 return 1;
2990
2991 return s1->date - s2->date > 0 ? 1 : -1;
2992 }
2993
2994
2995 /*
2996 * Same again, but for art[].line_count
2997 */
2998 static int
2999 lines_comp_asc(
3000 t_comptype p1,
3001 t_comptype p2)
3002 {
3003 const struct t_article *s1 = (const struct t_article *) p1;
3004 const struct t_article *s2 = (const struct t_article *) p2;
3005
3006 if (s1->line_count < s2->line_count)
3007 return -1;
3008
3009 if (s1->line_count > s2->line_count)
3010 return 1;
3011
3012 return s1->date - s2->date > 0 ? 1 : -1;
3013 }
3014
3015
3016 static int
3017 lines_comp_desc(
3018 t_comptype p1,
3019 t_comptype p2)
3020 {
3021 const struct t_article *s1 = (const struct t_article *) p1;
3022 const struct t_article *s2 = (const struct t_article *) p2;
3023
3024 if (s2->line_count < s1->line_count)
3025 return -1;
3026
3027 if (s2->line_count > s1->line_count)
3028 return 1;
3029
3030 return s1->date - s2->date > 0 ? 1 : -1;
3031 }
3032
3033
3034 /*
3035 * Compares the total score of two threads. Used for sorting base[].
3036 */
3037 static int
3038 score_comp_base(
3039 t_comptype p1,
3040 t_comptype p2)
3041 {
3042 int a = get_score_of_thread(*(const long *) p1);
3043 int b = get_score_of_thread(*(const long *) p2);
3044
3045 /* If scores are equal, compare using the article sort order.
3046 * This determines the order in a group of equally scored threads.
3047 */
3048 if (a == b) {
3049 const struct t_article *s1 = &arts[*(const long *) p1];
3050 const struct t_article *s2 = &arts[*(const long *) p2];
3051 t_compfunc comp_func = eval_sort_arts_func(CURR_GROUP.attribute->sort_article_type);
3052
3053 if (comp_func)
3054 return (*comp_func)(s1, s2);
3055 return 0;
3056 }
3057
3058 if (CURR_GROUP.attribute->sort_threads_type == SORT_THREADS_BY_SCORE_ASCEND)
3059 return a > b ? 1 : -1;
3060 return a < b ? 1 : -1;
3061 }
3062
3063
3064 /*
3065 * Compare the date of the last posted article of two threads.
3066 * Used for sorting base[].
3067 */
3068 static int
3069 last_date_comp_base_desc(
3070 t_comptype p1,
3071 t_comptype p2)
3072 {
3073 time_t s1_last = get_last_posting_date(*(const long *) p1);
3074 time_t s2_last = get_last_posting_date(*(const long *) p2);
3075
3076 if (s2_last < s1_last)
3077 return -1;
3078
3079 if (s2_last > s1_last)
3080 return 1;
3081
3082 return 0;
3083 }
3084
3085
3086 static int
3087 last_date_comp_base_asc(
3088 t_comptype p1,
3089 t_comptype p2)
3090 {
3091 time_t s1_last = get_last_posting_date(*(const long *) p1);
3092 time_t s2_last = get_last_posting_date(*(const long *) p2);
3093
3094 if (s2_last > s1_last)
3095 return -1;
3096
3097 if (s2_last < s1_last)
3098 return 1;
3099
3100 return 0;
3101 }
3102
3103
3104 static time_t
3105 get_last_posting_date(
3106 long n)
3107 {
3108 long i;
3109 time_t last = (time_t) 0;
3110
3111 for (i = n; i >= 0; i = arts[i].thread) {
3112 if (arts[i].date > last)
3113 last = arts[i].date;
3114 }
3115
3116 return last;
3117 }
3118
3119
3120 void
3121 set_article(
3122 struct t_article *art)
3123 {
3124 art->subject = NULL;
3125 art->from = NULL;
3126 art->name = NULL;
3127 art->date = (time_t) 0;
3128 art->xref = NULL;
3129 art->msgid = NULL;
3130 art->refs = NULL;
3131 art->refptr = NULL;
3132 art->line_count = -1;
3133 art->archive = NULL;
3134 art->tagged = 0;
3135 art->thread = ART_EXPIRED;
3136 art->prev = ART_NORMAL;
3137 art->score = 0;
3138 art->status = ART_UNREAD;
3139 art->killed = ART_NOTKILLED;
3140 art->zombie = FALSE;
3141 art->delete_it = FALSE;
3142 art->selected = FALSE;
3143 art->inrange = FALSE;
3144 art->matched = FALSE;
3145 art->keep_in_base = FALSE;
3146 }
3147
3148
3149 /*
3150 * Do a binary chop to see if 'art' (an article number) exists in arts[]
3151 * Naturally arts[] must be sorted on artnum
3152 * Return index into arts[] or -1
3153 */
3154 static int
3155 valid_artnum(
3156 t_artnum art)
3157 {
3158 int prev, range;
3159 int dctop = top_art;
3160 int cur = 1;
3161
3162 while ((dctop >>= 1))
3163 cur <<= 1;
3164
3165 range = cur >> 1;
3166 cur--;
3167
3168 forever {
3169 if (arts[cur].artnum == art)
3170 return cur;
3171
3172 prev = cur;
3173 cur += ((arts[cur].artnum < art) ? range : -range);
3174 if (prev == cur)
3175 break;
3176
3177 if (cur >= top_art)
3178 cur = top_art - 1;
3179
3180 range >>= 1;
3181 }
3182 return -1;
3183 }
3184
3185
3186 /*
3187 * Loop over arts[] to see if 'art' (an article number) exists in arts[]
3188 * Needed if arts[] is not sorted on artnum
3189 * Return index into arts[] or -1
3190 */
3191 int
3192 find_artnum(
3193 t_artnum art)
3194 {
3195 int i;
3196
3197 for_each_art(i) {
3198 if (arts[i].artnum == art)
3199 return i;
3200 }
3201 return -1;
3202 }
3203
3204
3205 /*----------------------------- Overview handling -----------------------*/
3206 /* TODO: use
3207 * setlocale(LC_ALL, "POSIX"); setlocale(LC_TIME, "POSIX");
3208 * my_strftime(date, sizeof(date) -1, "%d %b %Y %H:%M:%S GMT", gmtime(&secs));
3209 * instead?
3210 */
3211 static char *
3212 print_date(
3213 time_t secs)
3214 {
3215 static char date[25];
3216 struct tm *tm;
3217 static const char *const months_a[] = {
3218 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3219 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3220 };
3221
3222 if ((tm = gmtime(&secs)) != NULL)
3223 snprintf(date, sizeof(date), "%02d %.3s %04d %02d:%02d:%02d GMT",
3224 tm->tm_mday,
3225 months_a[tm->tm_mon],
3226 tm->tm_year + 1900,
3227 tm->tm_hour, tm->tm_min, tm->tm_sec);
3228 else
3229 snprintf(date, sizeof(date), "01 Jan 1970 00:00:00 UTC");
3230
3231 return date;
3232 }
3233
3234
3235 static char *
3236 print_from(
3237 struct t_group *group,
3238 struct t_article *article,
3239 int charset)
3240 {
3241 char *p, *q;
3242 static char from[PATH_LEN];
3243
3244 *from = '\0';
3245
3246 if (article->name != NULL) {
3247 q = my_strdup(article->name);
3248 #ifdef CHARSET_CONVERSION
3249 if (charset != -1) {
3250 buffer_to_network(q, charset);
3251 }
3252 #endif /* CHARSET_CONVERSION */
3253 p = rfc1522_encode(article->name, tinrc.mm_local_charset, FALSE);
3254 unfold_header(p);
3255 if (strpbrk(article->name, "\".:;<>@[]()\\") != NULL && article->name[0] != '"' && article->name[strlen(article->name)] != '"')
3256 snprintf(from, sizeof(from), "\"%s\" <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3257 else
3258 snprintf(from, sizeof(from), "%s <%s>", group->attribute->post_8bit_header ? q : p, article->from);
3259
3260 free(p);
3261 free(q);
3262 } else
3263 STRCPY(from, article->from);
3264
3265 return from;
3266 }
3267
3268
3269 /*
3270 * Open a group news overview file
3271 * Use NNTP XOVER where possible unless 'local' is set
3272 */
3273 static FILE *
3274 open_xover_fp(
3275 struct t_group *group,
3276 const char *mode,
3277 t_artnum min,
3278 t_artnum max,
3279 t_bool local)
3280 {
3281 #ifdef NNTP_ABLE
3282 if (!local && nntp_caps.over_cmd && *mode == 'r' && group->type == GROUP_TYPE_NEWS) {
3283 char line[NNTP_STRLEN];
3284
3285 if (!max)
3286 return NULL;
3287 if (min == max)
3288 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT, nntp_caps.over_cmd, min);
3289 else
3290 snprintf(line, sizeof(line), "%s %"T_ARTNUM_PFMT"-%"T_ARTNUM_PFMT, nntp_caps.over_cmd, min, MAX(min, max));
3291 return (nntp_command(line, OK_XOVER, NULL, 0));
3292 }
3293 #endif /* NNTP_ABLE */
3294 {
3295 FILE *fp;
3296 char *nov_file = find_nov_file(group, (*mode == 'r') ? R_OK : W_OK);
3297
3298 if (nov_file != NULL) {
3299 if ((fp = fopen(nov_file, mode)) != NULL)
3300 return fp;
3301
3302 if (*mode != 'r')
3303 error_message(2, _(txt_cannot_open), nov_file);
3304 }
3305 }
3306 return NULL;
3307 }
3308
3309
3310 #ifdef USE_HEAPSORT
3311 int
3312 tin_sort(
3313 void *sbase,
3314 size_t nel,
3315 size_t width,
3316 t_compfunc compar)
3317 {
3318 int rc;
3319
3320 switch (tinrc.sort_function) {
3321 case 0:
3322 qsort(sbase, nel, width, compar);
3323 rc = 0;
3324 break;
3325
3326 case 1:
3327 rc = heapsort(sbase, nel, width, compar);
3328 break;
3329
3330 default:
3331 rc = -1;
3332 break;
3333 }
3334 return rc;
3335 }
3336 #endif /* USE_HEAPSORT */