"Fossies" - the Fresh Open Source Software Archive 
"Fossies" - the Fresh Open Source Software Archive A hint: This file contains one or more very long lines, so maybe it is better readable using the pure text view mode that shows the contents as wrapped lines within the browser window.
1 2 /* 3 * zsync - client side rsync over http 4 * Copyright (C) 2004,2005,2007,2009 Colin Phipps <cph@moria.org.uk> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the Artistic License v2 (see the accompanying 8 * file COPYING for the full license terms), or, at your option, any later 9 * version of the same license. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * COPYING file for details. 15 */ 16 17 /* zsync command-line client program */ 18 19 #include "zsglobal.h" 20 21 #include <stdio.h> 22 #include <stdlib.h> 23 #include <string.h> 24 25 #include <unistd.h> 26 #include <fcntl.h> 27 #include <errno.h> 28 #include <ctype.h> 29 #include <sys/types.h> 30 #include <sys/stat.h> 31 #include <utime.h> 32 33 #ifdef WITH_DMALLOC 34 # include <dmalloc.h> 35 #endif 36 37 #include "libzsync/zsync.h" 38 39 #include "http.h" 40 #include "url.h" 41 #include "progress.h" 42 43 /* FILE* f = open_zcat_pipe(file_str) 44 * Returns a (popen) filehandle which when read returns the un-gzipped content 45 * of the given file. Or NULL on error; or the filehandle may fail to read. It 46 * is up to the caller to call pclose() on the handle and check the return 47 * value of that. 48 */ 49 FILE* open_zcat_pipe(const char* fname) 50 { 51 /* Get buffer to build command line */ 52 char *cmd = malloc(6 + strlen(fname) * 2); 53 if (!cmd) 54 return NULL; 55 56 strcpy(cmd, "zcat "); 57 { /* Add filename to commandline, escaping any characters that the shell 58 *might consider special. */ 59 int i, j; 60 61 for (i = 0, j = 5; fname[i]; i++) { 62 if (!isalnum(fname[i])) 63 cmd[j++] = '\\'; 64 cmd[j++] = fname[i]; 65 } 66 cmd[j] = 0; 67 } 68 69 if (!no_progress) 70 fprintf(stderr, "reading seed %s: ", cmd); 71 { /* Finally, open the subshell for reading, and return the handle */ 72 FILE* f = popen(cmd, "r"); 73 free(cmd); 74 return f; 75 } 76 } 77 78 /* read_seed_file(zsync, filename_str) 79 * Reads the given file (decompressing it if appropriate) and applies the rsync 80 * checksum algorithm to it, so any data that is contained in the target file 81 * is written to the in-progress target. So use this function to supply local 82 * source files which are believed to have data in common with the target. 83 */ 84 void read_seed_file(struct zsync_state *z, const char *fname) { 85 /* If we should decompress this file */ 86 if (zsync_hint_decompress(z) && strlen(fname) > 3 87 && !strcmp(fname + strlen(fname) - 3, ".gz")) { 88 /* Open for reading */ 89 FILE *f = open_zcat_pipe(fname); 90 if (!f) { 91 perror("popen"); 92 fprintf(stderr, "not using seed file %s\n", fname); 93 } 94 else { 95 96 /* Give the contents to libzsync to read and find any useful 97 * content */ 98 zsync_submit_source_file(z, f, !no_progress); 99 100 /* Close and check for errors */ 101 if (pclose(f) != 0) { 102 perror("close"); 103 } 104 } 105 } 106 else { 107 /* Simple uncompressed file - open it */ 108 FILE *f = fopen(fname, "r"); 109 if (!f) { 110 perror("open"); 111 fprintf(stderr, "not using seed file %s\n", fname); 112 } 113 else { 114 115 /* Give the contents to libzsync to read, to find any content that 116 * is part of the target file. */ 117 if (!no_progress) 118 fprintf(stderr, "reading seed file %s: ", fname); 119 zsync_submit_source_file(z, f, !no_progress); 120 121 /* And close */ 122 if (fclose(f) != 0) { 123 perror("close"); 124 } 125 } 126 } 127 128 { /* And print how far we've progressed towards the target file */ 129 long long done, total; 130 131 zsync_progress(z, &done, &total); 132 if (!no_progress) 133 fprintf(stderr, "\rRead %s. Target %02.1f%% complete. \n", 134 fname, (100.0f * done) / total); 135 } 136 } 137 138 long long http_down; 139 140 /* A ptrlist is a very simple structure for storing lists of pointers. This is 141 * the only function in its API. The structure (not actually a struct) consists 142 * of a (pointer to a) void*[] and an int giving the number of entries. 143 * 144 * ptrlist = append_ptrlist(&entries, ptrlist, new_entry) 145 * Like realloc(2), this returns the new location of the ptrlist array; the 146 * number of entries is passed by reference and updated in place. The new entry 147 * is appended to the list. 148 */ 149 static void **append_ptrlist(int *n, void **p, void *a) { 150 if (!a) 151 return p; 152 p = realloc(p, (*n + 1) * sizeof *p); 153 if (!p) { 154 fprintf(stderr, "out of memory\n"); 155 exit(1); 156 } 157 p[*n] = a; 158 (*n)++; 159 return p; 160 } 161 162 /* zs = read_zsync_control_file(location_str, filename) 163 * Reads a zsync control file from either a URL or filename specified in 164 * location_str. This is treated as a URL if no local file exists of that name 165 * and it starts with a URL scheme ; only http URLs are supported. 166 * Second parameter is a filename in which to locally save the content of the 167 * .zsync _if it is retrieved from a URL_; can be NULL in which case no local 168 * copy is made. 169 */ 170 struct zsync_state *read_zsync_control_file(const char *p, const char *fn) { 171 FILE *f; 172 struct zsync_state *zs; 173 char *lastpath = NULL; 174 175 /* Try opening as a local path */ 176 f = fopen(p, "r"); 177 if (!f) { 178 /* No such local file - if not a URL either, report error */ 179 if (!is_url_absolute(p)) { 180 perror(p); 181 exit(2); 182 } 183 184 /* Try URL fetch */ 185 f = http_get(p, &lastpath, fn); 186 if (!f) { 187 fprintf(stderr, "could not read control file from URL %s\n", p); 188 exit(3); 189 } 190 referer = lastpath; 191 } 192 193 /* Read the .zsync */ 194 if ((zs = zsync_begin(f)) == NULL) { 195 exit(1); 196 } 197 198 /* And close it */ 199 if (fclose(f) != 0) { 200 perror("fclose"); 201 exit(2); 202 } 203 return zs; 204 } 205 206 /* str = get_filename_prefix(path_str) 207 * Returns a (malloced) string of the alphanumeric leading segment of the 208 * filename in the given file path. 209 */ 210 static char *get_filename_prefix(const char *p) { 211 char *s = strdup(p); 212 char *t = strrchr(s, '/'); 213 char *u; 214 215 if (t) 216 *t++ = 0; 217 else 218 t = s; 219 u = t; 220 while (isalnum(*u)) { 221 u++; 222 } 223 *u = 0; 224 if (*t > 0) 225 t = strdup(t); 226 else 227 t = NULL; 228 free(s); 229 return t; 230 } 231 232 /* filename_str = get_filename(zs, source_filename_str) 233 * Returns a (malloced string with a) suitable filename for a zsync download, 234 * using the given zsync state and source filename strings as hints. */ 235 char *get_filename(const struct zsync_state *zs, const char *source_name) { 236 char *p = zsync_filename(zs); 237 char *filename = NULL; 238 239 if (p) { 240 if (strchr(p, '/')) { 241 fprintf(stderr, 242 "Rejected filename specified in %s, contained path component.\n", 243 source_name); 244 free(p); 245 } 246 else { 247 char *t = get_filename_prefix(source_name); 248 249 if (t && !memcmp(p, t, strlen(t))) 250 filename = p; 251 else 252 free(p); 253 254 if (t && !filename) { 255 fprintf(stderr, 256 "Rejected filename specified in %s - prefix %s differed from filename %s.\n", 257 source_name, t, p); 258 } 259 free(t); 260 } 261 } 262 if (!filename) { 263 filename = get_filename_prefix(source_name); 264 if (!filename) 265 filename = strdup("zsync-download"); 266 } 267 return filename; 268 } 269 270 /* prog = calc_zsync_progress(zs) 271 * Returns the progress ratio 0..1 (none...done) for the given zsync_state */ 272 static float calc_zsync_progress(const struct zsync_state *zs) { 273 long long zgot, ztot; 274 275 zsync_progress(zs, &zgot, &ztot); 276 return (100.0f * zgot / ztot); 277 } 278 279 /* fetch_remaining_blocks_http(zs, url, type) 280 * For the given zsync_state, using the given URL (which is a copy of the 281 * actual content of the target file is type == 0, or a compressed copy of it 282 * if type == 1), retrieve the parts of the target that are currently missing. 283 * Returns true if this URL was useful, false if we crashed and burned. 284 */ 285 #define BUFFERSIZE 8192 286 287 int fetch_remaining_blocks_http(struct zsync_state *z, const char *url, 288 int type) { 289 int ret = 0; 290 struct range_fetch *rf; 291 unsigned char *buf; 292 struct zsync_receiver *zr; 293 294 /* URL might be relative - we need an absolute URL to do a fetch */ 295 char *u = make_url_absolute(referer, url); 296 if (!u) { 297 fprintf(stderr, 298 "URL '%s' from the .zsync file is relative, but I don't know the referer URL (you probably downloaded the .zsync separately and gave it to me as a file). I need to know the referring URL (the URL of the .zsync) in order to locate the download. You can specify this with -u (or edit the URL line(s) in the .zsync file you have).\n", 299 url); 300 return -1; 301 } 302 303 /* Start a range fetch and a zsync receiver */ 304 rf = range_fetch_start(u); 305 if (!rf) { 306 free(u); 307 return -1; 308 } 309 zr = zsync_begin_receive(z, type); 310 if (!zr) { 311 range_fetch_end(rf); 312 free(u); 313 return -1; 314 } 315 316 if (!no_progress) 317 fprintf(stderr, "downloading from %s:", u); 318 319 /* Create a read buffer */ 320 buf = malloc(BUFFERSIZE); 321 if (!buf) { 322 zsync_end_receive(zr); 323 range_fetch_end(rf); 324 free(u); 325 return -1; 326 } 327 328 { /* Get a set of byte ranges that we need to complete the target */ 329 int nrange; 330 off_t *zbyterange = zsync_needed_byte_ranges(z, &nrange, type); 331 if (!zbyterange) 332 return 1; 333 if (nrange == 0) 334 return 0; 335 336 /* And give that to the range fetcher */ 337 range_fetch_addranges(rf, zbyterange, nrange); 338 free(zbyterange); 339 } 340 341 { 342 int len; 343 off_t zoffset; 344 struct progress p = { 0, 0, 0, 0 }; 345 346 /* Set up progress display to run during the fetch */ 347 if (!no_progress) { 348 fputc('\n', stderr); 349 do_progress(&p, calc_zsync_progress(z), range_fetch_bytes_down(rf)); 350 } 351 352 /* Loop while we're receiving data, until we're done or there is an error */ 353 while (!ret 354 && (len = get_range_block(rf, &zoffset, buf, BUFFERSIZE)) > 0) { 355 /* Pass received data to the zsync receiver, which writes it to the 356 * appropriate location in the target file */ 357 if (zsync_receive_data(zr, buf, zoffset, len) != 0) 358 ret = 1; 359 360 /* Maintain progress display */ 361 if (!no_progress) 362 do_progress(&p, calc_zsync_progress(z), 363 range_fetch_bytes_down(rf)); 364 365 // Needed in case next call returns len=0 and we need to signal where the EOF was. 366 zoffset += len; 367 } 368 369 /* If error, we need to flag that to our caller */ 370 if (len < 0) 371 ret = -1; 372 else /* Else, let the zsync receiver know that we're at EOF; there 373 *could be data in its buffer that it can use or needs to process */ 374 zsync_receive_data(zr, NULL, zoffset, 0); 375 376 if (!no_progress) 377 end_progress(&p, zsync_status(z) >= 2 ? 2 : len == 0 ? 1 : 0); 378 } 379 380 /* Clean up */ 381 free(buf); 382 http_down += range_fetch_bytes_down(rf); 383 zsync_end_receive(zr); 384 range_fetch_end(rf); 385 free(u); 386 return ret; 387 } 388 389 /* fetch_remaining_blocks(zs) 390 * Using the URLs in the supplied zsync state, downloads data to complete the 391 * target file. 392 */ 393 int fetch_remaining_blocks(struct zsync_state *zs) { 394 int n, utype; 395 const char *const *url = zsync_get_urls(zs, &n, &utype); 396 int *status; /* keep status for each URL - 0 means no error */ 397 int ok_urls = n; 398 399 if (!url) { 400 fprintf(stderr, "no URLs available from zsync?"); 401 return 1; 402 } 403 status = calloc(n, sizeof *status); 404 405 /* Keep going until we're done or have no useful URLs left */ 406 while (zsync_status(zs) < 2 && ok_urls) { 407 /* Still need data; pick a URL to use. */ 408 int try = rand() % n; 409 410 if (!status[try]) { 411 const char *tryurl = url[try]; 412 413 /* Try fetching data from this URL */ 414 int rc = fetch_remaining_blocks_http(zs, tryurl, utype); 415 if (rc != 0) { 416 fprintf(stderr, "failed to retrieve from %s\n", tryurl); 417 status[try] = 1; 418 ok_urls--; 419 } 420 } 421 } 422 free(status); 423 return 0; 424 } 425 426 static int set_mtime(char* filename, time_t mtime) { 427 struct stat s; 428 struct utimbuf u; 429 430 /* Get the access time, which I don't want to modify. */ 431 if (stat(filename, &s) != 0) { 432 perror("stat"); 433 return -1; 434 } 435 436 /* Set the modification time. */ 437 u.actime = s.st_atime; 438 u.modtime = mtime; 439 if (utime(filename, &u) != 0) { 440 perror("utime"); 441 return -1; 442 } 443 return 0; 444 } 445 446 /**************************************************************************** 447 * 448 * Main program */ 449 int main(int argc, char **argv) { 450 struct zsync_state *zs; 451 char *temp_file = NULL; 452 char **seedfiles = NULL; 453 int nseedfiles = 0; 454 char *filename = NULL; 455 long long local_used; 456 char *zfname = NULL; 457 time_t mtime; 458 459 srand(getpid()); 460 { /* Option parsing */ 461 int opt; 462 463 while ((opt = getopt(argc, argv, "A:k:o:i:Vsqu:")) != -1) { 464 switch (opt) { 465 case 'A': /* Authentication options for remote server */ 466 { /* Scan string as hostname=username:password */ 467 char *p = strdup(optarg); 468 char *q = strchr(p, '='); 469 char *r = q ? strchr(q, ':') : NULL; 470 471 if (!q || !r) { 472 fprintf(stderr, 473 "-A takes hostname=username:password\n"); 474 exit(1); 475 } 476 else { 477 *q++ = *r++ = 0; 478 add_auth(p, q, r); 479 } 480 } 481 break; 482 case 'k': 483 free(zfname); 484 zfname = strdup(optarg); 485 break; 486 case 'o': 487 free(filename); 488 filename = strdup(optarg); 489 break; 490 case 'i': 491 seedfiles = append_ptrlist(&nseedfiles, seedfiles, optarg); 492 break; 493 case 'V': 494 printf(PACKAGE " v" VERSION " (compiled " __DATE__ " " __TIME__ 495 ")\n" "By Colin Phipps <cph@moria.org.uk>\n" 496 "Published under the Artistic License v2, see the COPYING file for details.\n"); 497 exit(0); 498 case 's': 499 case 'q': 500 no_progress = 1; 501 break; 502 case 'u': 503 referer = strdup(optarg); 504 break; 505 } 506 } 507 } 508 509 /* Last and only non-option parameter must be the path/URL of the .zsync */ 510 if (optind == argc) { 511 fprintf(stderr, 512 "No .zsync file specified.\nUsage: zsync http://example.com/some/filename.zsync\n"); 513 exit(3); 514 } 515 else if (optind < argc - 1) { 516 fprintf(stderr, 517 "Usage: zsync http://example.com/some/filename.zsync\n"); 518 exit(3); 519 } 520 521 /* No progress display except on terminal */ 522 if (!isatty(0)) 523 no_progress = 1; 524 { /* Get proxy setting from the environment */ 525 char *pr = getenv("http_proxy"); 526 527 if (pr != NULL) 528 set_proxy_from_string(pr); 529 } 530 531 /* STEP 1: Read the zsync control file */ 532 if ((zs = read_zsync_control_file(argv[optind], zfname)) == NULL) 533 exit(1); 534 535 /* Get eventual filename for output, and filename to write to while working */ 536 if (!filename) 537 filename = get_filename(zs, argv[optind]); 538 temp_file = malloc(strlen(filename) + 6); 539 strcpy(temp_file, filename); 540 strcat(temp_file, ".part"); 541 542 { /* STEP 2: read available local data and fill in what we know in the 543 *target file */ 544 int i; 545 546 /* If the target file already exists, we're probably updating that file 547 * - so it's a seed file */ 548 if (!access(filename, R_OK)) { 549 seedfiles = append_ptrlist(&nseedfiles, seedfiles, filename); 550 } 551 /* If the .part file exists, it's probably an interrupted earlier 552 * effort; a normal HTTP client would 'resume' from where it got to, 553 * but zsync can't (because we don't know this data corresponds to the 554 * current version on the remote) and doesn't need to, because we can 555 * treat it like any other local source of data. Use it now. */ 556 if (!access(temp_file, R_OK)) { 557 seedfiles = append_ptrlist(&nseedfiles, seedfiles, temp_file); 558 } 559 560 /* Try any seed files supplied by the command line */ 561 for (i = 0; i < nseedfiles; i++) { 562 int dup = 0, j; 563 564 /* And stop reading seed files once the target is complete. */ 565 if (zsync_status(zs) >= 2) break; 566 567 /* Skip dups automatically, to save the person running the program 568 * having to worry about this stuff. */ 569 for (j = 0; j < i; j++) { 570 if (!strcmp(seedfiles[i],seedfiles[j])) dup = 1; 571 } 572 573 /* And now, if not a duplicate, read it */ 574 if (!dup) 575 read_seed_file(zs, seedfiles[i]); 576 } 577 /* Show how far that got us */ 578 zsync_progress(zs, &local_used, NULL); 579 580 /* People that don't understand zsync might use it wrongly and end up 581 * downloading everything. Although not essential, let's hint to them 582 * that they probably messed up. */ 583 if (!local_used) { 584 if (!no_progress) 585 fputs 586 ("No relevent local data found - I will be downloading the whole file. If that's not what you want, CTRL-C out. You should specify the local file is the old version of the file to download with -i (you might have to decompress it with gzip -d first). Or perhaps you just have no data that helps download the file\n", 587 stderr); 588 } 589 } 590 591 /* libzsync has been writing to a randomely-named temp file so far - 592 * because we didn't want to overwrite the .part from previous runs. Now 593 * we've read any previous .part, we can replace it with our new 594 * in-progress run (which should be a superset of the old .part - unless 595 * the content changed, in which case it still contains anything relevant 596 * from the old .part). */ 597 if (zsync_rename_file(zs, temp_file) != 0) { 598 perror("rename"); 599 exit(1); 600 } 601 602 /* STEP 3: fetch remaining blocks via the URLs from the .zsync */ 603 if (fetch_remaining_blocks(zs) != 0) { 604 fprintf(stderr, 605 "failed to retrieve all remaining blocks - no valid download URLs remain. Incomplete transfer left in %s.\n(If this is the download filename with .part appended, zsync will automatically pick this up and reuse the data it has already done if you retry in this dir.)\n", 606 temp_file); 607 exit(3); 608 } 609 610 { /* STEP 4: verify download */ 611 int r; 612 613 if (!no_progress) 614 printf("verifying download..."); 615 r = zsync_complete(zs); 616 switch (r) { 617 case -1: 618 fprintf(stderr, "Aborting, download available in %s\n", temp_file); 619 exit(2); 620 case 0: 621 if (!no_progress) 622 printf("no recognised checksum found\n"); 623 break; 624 case 1: 625 if (!no_progress) 626 printf("checksum matches OK\n"); 627 break; 628 } 629 } 630 631 free(temp_file); 632 633 /* Get any mtime that we is suggested to set for the file, and then shut 634 * down the zsync_state as we are done on the file transfer. Getting the 635 * current name of the file at the same time. */ 636 mtime = zsync_mtime(zs); 637 temp_file = zsync_end(zs); 638 639 /* STEP 5: Move completed .part file into place as the final target */ 640 if (filename) { 641 char *oldfile_backup = malloc(strlen(filename) + 8); 642 int ok = 1; 643 644 strcpy(oldfile_backup, filename); 645 strcat(oldfile_backup, ".zs-old"); 646 647 if (!access(filename, F_OK)) { 648 /* Backup the old file. */ 649 /* First, remove any previous backup. We don't care if this fails - 650 * the link below will catch any failure */ 651 unlink(oldfile_backup); 652 653 /* Try linking the filename to the backup file name, so we will 654 atomically replace the target file in the next step. 655 If that fails due to EPERM, it is probably a filesystem that 656 doesn't support hard-links - so try just renaming it to the 657 backup filename. */ 658 if (link(filename, oldfile_backup) != 0 659 && (errno != EPERM || rename(filename, oldfile_backup) != 0)) { 660 perror("linkname"); 661 fprintf(stderr, 662 "Unable to back up old file %s - completed download left in %s\n", 663 filename, temp_file); 664 ok = 0; /* Prevent overwrite of old file below */ 665 } 666 } 667 if (ok) { 668 /* Rename the file to the desired name */ 669 if (rename(temp_file, filename) == 0) { 670 /* final, final thing - set the mtime on the file if we have one */ 671 if (mtime != -1) set_mtime(filename, mtime); 672 } 673 else { 674 perror("rename"); 675 fprintf(stderr, 676 "Unable to back up old file %s - completed download left in %s\n", 677 filename, temp_file); 678 } 679 } 680 free(oldfile_backup); 681 free(filename); 682 } 683 else { 684 printf 685 ("No filename specified for download - completed download left in %s\n", 686 temp_file); 687 } 688 689 /* Final stats and cleanup */ 690 if (!no_progress) 691 printf("used %lld local, fetched %lld\n", local_used, http_down); 692 free(referer); 693 free(temp_file); 694 return 0; 695 }