"Fossies" - the Fresh Open Source Software Archive

Member "ncdc-1.22.1/src/dlfile.c" (26 Mar 2019, 21826 Bytes) of package /linux/privat/ncdc-1.22.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "dlfile.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.20_vs_1.21.

    1 /* ncdc - NCurses Direct Connect client
    2 
    3   Copyright (c) 2011-2019 Yoran Heling
    4 
    5   Permission is hereby granted, free of charge, to any person obtaining
    6   a copy of this software and associated documentation files (the
    7   "Software"), to deal in the Software without restriction, including
    8   without limitation the rights to use, copy, modify, merge, publish,
    9   distribute, sublicense, and/or sell copies of the Software, and to
   10   permit persons to whom the Software is furnished to do so, subject to
   11   the following conditions:
   12 
   13   The above copyright notice and this permission notice shall be included
   14   in all copies or substantial portions of the Software.
   15 
   16   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   17   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   18   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   19   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
   20   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
   21   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
   22   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
   23 
   24 */
   25 
   26 
   27 #include "ncdc.h"
   28 #include "dlfile.h"
   29 
   30 
   31 /* Terminology
   32  *
   33  *   chunk: Smallest "addressable" byte range within a file, see DLFILE_CHUNKSIZE
   34  *   block: Smallest "verifiable" byte range within a file (i.e. what a TTH
   35  *          leaf represents, see DL_MINBLOCKSIZE). Always a multiple of the
   36  *          chunk size.
   37  *  thread: A range of chunks that haven't been downloaded yet.
   38  * segment: A range of chunks that is requested for downloading in a single
   39  *          CGET/$ADCGET. Not necessarily aligned to or a multiple of the block
   40  *          size. Segments are allocated at the start of a thread.
   41  */
   42 
   43 
   44 #if INTERFACE
   45 
   46 /* Size of a chunk within the downloaded file. This determines the granularity
   47  * of the file data that is remembered across restarts, the size of the chunk
   48  * bitmap and the minimum download request.
   49  * Must be a power of two and less than or equal to DL_MINBLOCKSIZE */
   50 #define DLFILE_CHUNKSIZE (128*1024)
   51 
   52 
   53 /* For file lists (dl->islist), only len and chunk are used. The other fields
   54  * aren't used because no length of TTH info is known before downloading. */
   55 struct dlfile_thread_t {
   56   dl_t *dl;
   57   tth_ctx_t hash_tth;
   58   guint32 allocated; /* Number of remaining chunks allocated to this thread (including current) */
   59   guint32 avail;     /* Number of undownloaded chunks in and after this thread (including current & allocated) */
   60   guint32 chunk;     /* Current chunk number */
   61   guint32 len;       /* Number of bytes downloaded into this chunk */
   62   gboolean busy;     /* Whether this thread is being used */
   63   /* Fields for deferred error reporting */
   64   guint64 uid;
   65   char *err_msg, *uerr_msg;
   66   char err, uerr;
   67 };
   68 
   69 #endif
   70 
   71 
   72 static guint32 dlfile_chunks(guint64 size) {
   73   return (size+DLFILE_CHUNKSIZE-1)/DLFILE_CHUNKSIZE;
   74 }
   75 
   76 
   77 static gboolean dlfile_hasfreeblock(dlfile_thread_t *t) {
   78   guint32 chunksinblock = t->dl->hash_block / DLFILE_CHUNKSIZE;
   79   return t->avail - t->allocated > chunksinblock
   80     || (t->chunk + t->avail == dlfile_chunks(t->dl->size) && t->chunk + t->allocated <= (((dlfile_chunks(t->dl->size)-1)/chunksinblock)*chunksinblock));
   81 }
   82 
   83 
   84 /* Highly verbose debugging function. Prints out a list of threads for a particular dl item. */
   85 static void dlfile_threaddump(dl_t *dl, int n) {
   86 #if 0
   87   GSList *l;
   88   for(l=dl->threads; l; l=l->next) {
   89     dlfile_thread_t *ti = l->data;
   90     g_debug("THREAD DUMP#%p.%d: busy = %d, chunk = %u, allocated = %u, avail = %u", dl, n, ti->busy, ti->chunk, ti->allocated, ti->avail);
   91   }
   92 #endif
   93 }
   94 
   95 
   96 static void dlfile_fatal_load_error(dl_t *dl, const char *op, const char *err) {
   97   g_error("Unable to %s incoming file `%s', (%s; incoming file for `%s').\n"
   98     "Delete the incoming file or otherwise repair it before restarting ncdc.",
   99     op, dl->inc, err ? err : g_strerror(errno), dl->dest);
  100 }
  101 
  102 
  103 /* Must be called while the lock is held when dl->active_threads may be >0 */
  104 static gboolean dlfile_save_bitmap(dl_t *dl, int fd) {
  105   guint8 *buf = dl->bitmap;
  106   off_t off = dl->size;
  107   size_t left = bita_size(dlfile_chunks(dl->size));
  108   while(left > 0) {
  109     int r = pwrite(fd, buf, left, off);
  110     if(r < 0)
  111       return FALSE;
  112     left -= r;
  113     off += r;
  114     buf += r;
  115   }
  116   return TRUE;
  117 }
  118 
  119 
  120 static gboolean dlfile_save_bitmap_timeout(gpointer dat) {
  121   dl_t *dl = dat;
  122   g_static_mutex_lock(&dl->lock);
  123   dl->bitmap_src = 0;
  124   if(dl->incfd > 0 && !dlfile_save_bitmap(dl, dl->incfd)) {
  125     g_warning("Error writing bitmap for `%s': %s.", dl->dest, g_strerror(errno));
  126     dl_queue_seterr(dl, DLE_IO_INC, g_strerror(errno));
  127   }
  128   if(dl->incfd > 0 && !dl->active_threads) {
  129     close(dl->incfd);
  130     dl->incfd = 0;
  131   }
  132   g_static_mutex_unlock(&dl->lock);
  133   return FALSE;
  134 }
  135 
  136 
  137 /* Must be called while dl->lock is held. */
  138 static void dlfile_save_bitmap_defer(dl_t *dl) {
  139   if(!dl->bitmap_src)
  140     dl->bitmap_src = g_timeout_add_seconds(5, dlfile_save_bitmap_timeout, dl);
  141 }
  142 
  143 
  144 static void dlfile_load_canconvert(dl_t *dl) {
  145   static gboolean canconvert = FALSE;
  146   if(canconvert)
  147     return;
  148   printf(
  149     "I found a partially downloaded file without a bitmap. This probably\n"
  150     "means that you are upgrading from ncdc 1.17 or earlier, which did not\n"
  151     "yet support segmented downloading.\n\n"
  152     "To convert your partially downloaded files to the new format and to\n"
  153     "continue with starting up ncdc, press enter. To abort, hit Ctrl+C.\n\n"
  154     "Note: After this conversion, you should NOT downgrade ncdc. If you\n"
  155     " wish to do that, backup or delete your inc/ directory first.\n\n"
  156     "Note#2: If you get this message when you haven't upgraded ncdc, then\n"
  157     " your partially downloaded file is likely corrupt, and continuing\n"
  158     " this conversion will not help. In that case, the best you can do is\n"
  159     " delete the corrupted file and restart ncdc.\n\n"
  160     "The file that triggered this warning is:\n"
  161     "  %s\n"
  162     "Which is the incoming file for:\n"
  163     "  %s\n"
  164     "(But there are possibly more affected files)\n",
  165     dl->inc, dl->dest);
  166   getchar();
  167   canconvert = TRUE;
  168 }
  169 
  170 
  171 static void dlfile_load_nonbitmap(dl_t *dl, int fd, guint8 *bitmap) {
  172   struct stat st;
  173   if(fstat(fd, &st) < 0)
  174     dlfile_fatal_load_error(dl, "stat", NULL);
  175   if((guint64)st.st_size >= dl->size)
  176     dlfile_fatal_load_error(dl, "load", "File too large");
  177 
  178   dlfile_load_canconvert(dl);
  179 
  180   guint64 left = st.st_size;
  181   guint32 chunk = 0;
  182   while(left > DLFILE_CHUNKSIZE) {
  183     bita_set(bitmap, chunk);
  184     chunk++;
  185     left -= DLFILE_CHUNKSIZE;
  186   }
  187 }
  188 
  189 
  190 static gboolean dlfile_load_bitmap(dl_t *dl, int fd) {
  191   gboolean needsave = FALSE;
  192   guint32 chunks = dlfile_chunks(dl->size);
  193   guint8 *bitmap = bita_new(chunks);
  194   guint8 *dest = bitmap;
  195 
  196   off_t off = dl->size;
  197   size_t left = bita_size(chunks);
  198   while(left > 0) {
  199     int r = pread(fd, dest, left, off);
  200     if(r < 0)
  201       dlfile_fatal_load_error(dl, "read bitmap from", NULL);
  202     if(!r) {
  203       dlfile_load_nonbitmap(dl, fd, bitmap);
  204       needsave = TRUE;
  205       break;
  206     }
  207     left -= r;
  208     off += r;
  209     dest += r;
  210   }
  211 
  212   bita_free(dl->bitmap);
  213   dl->bitmap = bitmap;
  214   return needsave;
  215 }
  216 
  217 
  218 static dlfile_thread_t *dlfile_load_block(dl_t *dl, int fd, guint32 chunk, guint32 chunksinblock, guint32 *reset) {
  219   dlfile_thread_t *t = g_slice_new0(dlfile_thread_t);
  220   t->dl = dl;
  221   t->chunk = chunk;
  222   t->avail = chunksinblock;
  223   tth_init(&t->hash_tth);
  224 
  225   char *bufp = malloc(DLFILE_CHUNKSIZE);
  226 
  227   *reset = chunksinblock;
  228   while(bita_get(dl->bitmap, t->chunk)) {
  229     char *buf = bufp;
  230     off_t off = (guint64)t->chunk * DLFILE_CHUNKSIZE;
  231     size_t left = DLFILE_CHUNKSIZE;
  232     while(left > 0) {
  233       int r = pread(fd, buf, left, off);
  234       if(r <= 0)
  235         dlfile_fatal_load_error(dl, "read from", NULL);
  236       off -= r;
  237       left -= r;
  238       buf += r;
  239     }
  240     tth_update(&t->hash_tth, bufp, DLFILE_CHUNKSIZE);
  241     t->chunk++;
  242     t->avail--;
  243     dl->have += DLFILE_CHUNKSIZE;
  244     (*reset)--;
  245   }
  246 
  247   free(bufp);
  248   dl->threads = g_slist_prepend(dl->threads, t);
  249   return t;
  250 }
  251 
  252 
  253 /* Go over the bitmap and create a thread for each range of undownloaded
  254  * chunks. Threads are created in a TTHL-block-aligned fashion to ensure that
  255  * the downloading progress can continue from the threads while keeping the
  256  * integrity checks. */
  257 static gboolean dlfile_load_threads(dl_t *dl, int fd) {
  258   guint32 chunknum = dlfile_chunks(dl->size);
  259   guint32 chunksperblock = dl->hash_block / DLFILE_CHUNKSIZE;
  260   gboolean needsave = FALSE;
  261   dlfile_thread_t *t = NULL;
  262 
  263   guint32 i,j;
  264   for(i=0; i<chunknum; i+=chunksperblock) {
  265     guint32 reset = 0;
  266     guint32 chunksinblock = MIN(chunksperblock, dlfile_chunks(dl->size) - i);
  267 
  268     for(j=i; j<i+chunksinblock; j++)
  269       if(!bita_get(dl->bitmap, j))
  270         break;
  271     gboolean hasfullblock = j == i+chunksinblock;
  272 
  273     if(t && !bita_get(dl->bitmap, i)) {
  274       t->avail += chunksinblock;
  275       reset = chunksinblock;
  276     } else if(hasfullblock) {
  277       t = NULL;
  278       dl->have += dl->hash_block;
  279     } else
  280       t = dlfile_load_block(dl, fd, i, chunksinblock, &reset);
  281 
  282     for(j=i+(chunksinblock-reset); j<i+chunksinblock; j++)
  283       if(bita_get(dl->bitmap, j)) {
  284         bita_reset(dl->bitmap, j);
  285         needsave = TRUE;
  286       }
  287   }
  288   return needsave;
  289 }
  290 
  291 
  292 void dlfile_load(dl_t *dl) {
  293   /* If moving to the destination failed in a previous run, assume that the
  294    * incoming file is complete and all that's left to do is resume the
  295    * finalization (which the user has to initiate by clearing the error). This
  296    * needs to be handled as a special case because the incoming file will not
  297    * contain the bitmap anymore at this point, and the loading process below
  298    * will fail. */
  299   if(dl->prio == DLP_ERR && dl->error == DLE_IO_DEST) {
  300     g_message("Download for `%s' in IO_DEST error state, assuming `%s' contains the finished download.", dl->dest, dl->inc);
  301     dl->have = dl->size;
  302     return;
  303   }
  304 
  305   dl->have = 0;
  306   int fd = open(dl->inc, O_RDWR);
  307   if(fd < 0) {
  308     if(errno != ENOENT)
  309       dlfile_fatal_load_error(dl, "open", NULL);
  310     return;
  311   }
  312 
  313   /* If the above didn't fail, then we should already have TTHL data.
  314    * Otherwise, close and delete whatever we have. */
  315   if(!dl->hastthl) {
  316     g_warning("No TTHL data for `%s', deleting partially downloaded data.", dl->dest);
  317     close(fd);
  318     unlink(dl->inc);
  319     return;
  320   }
  321 
  322   gboolean needsave = dlfile_load_bitmap(dl, fd);
  323   if(dlfile_load_threads(dl, fd))
  324     needsave = TRUE;
  325 
  326   if(needsave && !dlfile_save_bitmap(dl, fd))
  327     dlfile_fatal_load_error(dl, "save bitmap to", NULL);
  328 
  329   dlfile_threaddump(dl, 0);
  330   close(fd);
  331 }
  332 
  333 
  334 /* Called from dl.c when a dl item is being deleted, either from
  335  * dlfile_finished() or when the item is removed from the UI. */
  336 void dlfile_rm(dl_t *dl) {
  337   g_return_if_fail(!dl->active_threads);
  338 
  339   if(dl->bitmap_src)
  340     g_source_remove(dl->bitmap_src);
  341 
  342   if(dl->incfd > 0)
  343     g_warn_if_fail(close(dl->incfd) == 0);
  344 
  345   if(dl->inc)
  346     unlink(dl->inc);
  347 
  348   GSList *l;
  349   for(l=dl->threads; l; l=l->next)
  350     g_slice_free(dlfile_thread_t, l->data);
  351   g_slist_free(dl->threads);
  352   g_free(dl->bitmap);
  353 }
  354 
  355 
  356 /* Create the inc file and initialize the necessary structs to prepare for
  357  * handling downloaded data. */
  358 static gboolean dlfile_open(dl_t *dl) {
  359   if(dl->incfd <= 0)
  360     dl->incfd = open(dl->inc, O_WRONLY|O_CREAT, 0666);
  361   if(dl->incfd < 0) {
  362     g_warning("Error opening %s: %s", dl->inc, g_strerror(errno));
  363     dl_queue_seterr(dl, DLE_IO_INC, g_strerror(errno));
  364     return FALSE;
  365   }
  366 
  367   /* Everything else has already been initialized if we have a thread or bitmap */
  368   if(dl->threads || dl->bitmap)
  369     return TRUE;
  370 
  371   if(!dl->islist) {
  372     dl->bitmap = bita_new(dlfile_chunks(dl->size));
  373     if(!dlfile_save_bitmap(dl, dl->incfd)) {
  374       g_warning("Error writing bitmap for `%s': %s.", dl->dest, g_strerror(errno));
  375       dl_queue_seterr(dl, DLE_IO_INC, g_strerror(errno));
  376       free(dl->bitmap);
  377       dl->bitmap = NULL;
  378       return FALSE;
  379     }
  380   }
  381 
  382   dlfile_thread_t *t = g_slice_new0(dlfile_thread_t);
  383   t->dl = dl;
  384   t->chunk = 0;
  385   t->allocated = 0;
  386   if(!dl->islist)
  387     t->avail = dlfile_chunks(dl->size);
  388   tth_init(&t->hash_tth);
  389   dl->threads = g_slist_prepend(dl->threads, t);
  390   return TRUE;
  391 }
  392 
  393 
  394 /* XXX: This function may block in the main thread for a while. Perhaps do it in a threadpool? */
  395 void dlfile_finished(dl_t *dl) {
  396   if(dl->incfd <= 0 && !dlfile_open(dl))
  397     return;
  398 
  399   /* Regular files: Remove bitmap from the file
  400    * File lists: Ensure that the file size is correct after we've downloaded a
  401    *   longer file list before that got interrupted. */
  402   if(ftruncate(dl->incfd, dl->size) < 0) {
  403     g_warning("Error truncating the incoming file for `%s': %s.", dl->dest, g_strerror(errno));
  404     dl_queue_seterr(dl, DLE_IO_INC, g_strerror(errno));
  405     return;
  406   }
  407   int r = close(dl->incfd);
  408   dl->incfd = 0;
  409   if(r < 0) {
  410     g_warning("Error closing the incoming file for `%s': %s.", dl->dest, g_strerror(errno));
  411     dl_queue_seterr(dl, DLE_IO_INC, g_strerror(errno));
  412     return;
  413   }
  414 
  415   char *fdest = g_filename_from_utf8(dl->dest, -1, NULL, NULL, NULL);
  416   if(!fdest)
  417     fdest = g_strdup(dl->dest);
  418 
  419   /* Create destination directory, if it does not exist yet. */
  420   char *parent = g_path_get_dirname(fdest);
  421   r = g_mkdir_with_parents(parent, 0777);
  422   g_free(parent);
  423   if(r < 0) {
  424     g_warning("Error creating directory for `%s': %s.", dl->dest, g_strerror(errno));
  425     dl_queue_seterr(dl, DLE_IO_DEST, g_strerror(errno));
  426     g_free(fdest);
  427     return;
  428   }
  429 
  430   /* Prevent overwiting other files by appending a prefix to the destination if
  431    * it already exists. It is assumed that fn + any dupe-prevention-extension
  432    * does not exceed NAME_MAX. (Not that checking against NAME_MAX is really
  433    * reliable - some filesystems have an even more strict limit) */
  434   int num = 1;
  435   char *dest = g_strdup(fdest);
  436   while(!dl->islist && g_file_test(dest, G_FILE_TEST_EXISTS)) {
  437     g_free(dest);
  438     dest = g_strdup_printf("%s.%d", fdest, num++);
  439   }
  440   g_free(fdest);
  441 
  442   GError *err = NULL;
  443   file_move(dl->inc, dest, dl->islist, &err);
  444   g_free(dest);
  445   if(err) {
  446     g_warning("Error moving file to destination `%s': %s.", dl->dest, err->message);
  447     dl_queue_seterr(dl, DLE_IO_DEST, err->message);
  448     g_error_free(err);
  449     return;
  450   }
  451 
  452   dl_finished(dl);
  453 }
  454 
  455 
  456 /* The 'speed' argument should be a pessimistic estimate of the peers' speed,
  457  * in bytes/s. I think this is best obtained from a 30 second average.
  458  * Returns the thread pointer. */
  459 dlfile_thread_t *dlfile_getchunk(dl_t *dl, guint64 uid, guint64 speed) {
  460   dlfile_thread_t *t = NULL;
  461   if(!dlfile_open(dl))
  462     return NULL;
  463 
  464   /* File lists should always be downloaded in a single GET request because
  465    * their contents may be modified between subsequent requests. */
  466   if(dl->islist) {
  467     t = dl->threads->data;
  468     t->chunk = 0;
  469     t->len = 0;
  470     t->uid = uid;
  471     t->busy = TRUE;
  472     dl->hassize = FALSE;
  473     dl->have = 0;
  474     dl->allbusy = TRUE;
  475     dl->active_threads++;
  476     return t;
  477   }
  478 
  479   /* Walk through the threads and look for:
  480    *      t = Thread with largest avail and with allocated = 0
  481    *   tsec = Thread with an unallocated block and largest avail-allocated
  482    */
  483   dlfile_thread_t *tsec = NULL;
  484   GSList *l;
  485 
  486   g_static_mutex_lock(&dl->lock);
  487   dlfile_threaddump(dl, 1);
  488   for(l=dl->threads; l; l=l->next) {
  489     dlfile_thread_t *ti = l->data;
  490     if(ti->avail && (!tsec || ti->avail-ti->allocated > tsec->avail-tsec->allocated) && dlfile_hasfreeblock(ti))
  491       tsec = ti;
  492     if(!ti->busy && (!t || ti->avail > t->avail))
  493       t = ti;
  494   }
  495 
  496   if(!t) {
  497     guint32 chunksinblock = dl->hash_block/DLFILE_CHUNKSIZE;
  498     guint32 chunk = ((tsec->chunk + tsec->allocated + (tsec->avail - tsec->allocated)/2) / chunksinblock) * chunksinblock;
  499     if(chunk < tsec->chunk + tsec->allocated) /* Only possible for the last block in the file */
  500       chunk += chunksinblock;
  501     t = g_slice_new0(dlfile_thread_t);
  502     t->dl = dl;
  503     t->chunk = chunk;
  504     t->avail = tsec->avail - (chunk - tsec->chunk);
  505     g_return_val_if_fail(t->avail > 0, NULL);
  506     tth_init(&t->hash_tth);
  507 
  508     tsec->avail -= t->avail;
  509     dl->threads = g_slist_prepend(dl->threads, t);
  510   }
  511 
  512   /* Number of chunks to request as one segment. The size of a segment is
  513    * chosen to approximate a download time of ~5 min. */
  514   guint32 minsegment = var_get_int64(0, VAR_download_segment);
  515   if(minsegment) {
  516     guint32 chunks = MIN(G_MAXUINT32, 1 + ((speed * 300) / DLFILE_CHUNKSIZE));
  517     chunks = MAX(chunks, (minsegment+DLFILE_CHUNKSIZE-1) / DLFILE_CHUNKSIZE);
  518     t->allocated = MIN(t->avail, chunks);
  519   } else
  520     t->allocated = t->avail;
  521   t->busy = TRUE;
  522   t->uid = uid;
  523   dl->active_threads++;
  524 
  525   /* Go through the list again to update dl->allbusy */
  526   for(l=dl->threads; l; l=l->next) {
  527     dlfile_thread_t *ti = l->data;
  528     if(ti->avail && (!ti->busy || dlfile_hasfreeblock(ti)))
  529       break;
  530   }
  531   dl->allbusy = !l;
  532 
  533   dlfile_threaddump(dl, 2);
  534   g_static_mutex_unlock(&dl->lock);
  535   g_debug("Allocating: allbusy = %d, chunk = %u, allocated = %u, avail = %u, chunksinblock = %u, chunksinfile = %u",
  536       dl->allbusy, t->chunk, t->allocated, t->avail, (guint32)dl->hash_block/DLFILE_CHUNKSIZE, dlfile_chunks(dl->size));
  537   return t;
  538 }
  539 
  540 
  541 static gboolean dlfile_recv_check(dlfile_thread_t *t, char *leaf) {
  542   guint32 num = (t->chunk-1)/(t->dl->hash_block / DLFILE_CHUNKSIZE);
  543   if(t->dl->size < t->dl->hash_block ? memcmp(leaf, t->dl->hash, 24) == 0 : db_dl_checkhash(t->dl->hash, num, leaf))
  544     return TRUE;
  545 
  546   g_static_mutex_lock(&t->dl->lock);
  547 
  548   /* Hash failure, remove the failed block from the bitmap and dl->have, and
  549    * reset this thread so that the block can be re-downloaded. */
  550   guint32 startchunk = num * (t->dl->hash_block / DLFILE_CHUNKSIZE);
  551   // Or: chunksinblock = MIN(t->dl->hash_block / DLFILE_CHUNKSIZE, dlfile_chunks(t->dl->size) - startchunk);
  552   guint32 chunksinblock = t->chunk - startchunk;
  553   t->chunk = startchunk;
  554   t->avail += chunksinblock;
  555   t->allocated += chunksinblock;
  556   t->dl->have -= MIN(t->dl->hash_block, t->dl->size - (guint64)startchunk * DLFILE_CHUNKSIZE);
  557 
  558   guint32 i;
  559   for(i=startchunk; i<startchunk+chunksinblock; i++)
  560     bita_set(t->dl->bitmap, i);
  561   dlfile_save_bitmap_defer(t->dl);
  562 
  563   g_static_mutex_unlock(&t->dl->lock);
  564 
  565   t->uerr = DLE_HASH;
  566   t->uerr_msg = g_strdup_printf("Hash for block %u (chunk %u-%u) does not match.", num, startchunk, startchunk+chunksinblock);
  567   return FALSE;
  568 }
  569 
  570 
  571 static gboolean dlfile_recv_write(dlfile_thread_t *t, const char *buf, int len) {
  572   off_t off = ((guint64)t->chunk * DLFILE_CHUNKSIZE) + t->len;
  573   off_t offi = off;
  574   size_t rem = len;
  575   const char *bufi = buf;
  576   while(rem > 0) {
  577     ssize_t r = pwrite(t->dl->incfd, bufi, rem, offi);
  578     if(r <= 0) {
  579       t->err = DLE_IO_INC;
  580       t->err_msg = g_strdup(g_strerror(errno));
  581       return FALSE;
  582     }
  583     offi += r;
  584     rem -= r;
  585     bufi += r;
  586   }
  587   fadv_oneshot(t->dl->incfd, off, len, VAR_FFC_DOWNLOAD);
  588   return TRUE;
  589 }
  590 
  591 
  592 /* Called when new data has been received from a downloading thread.  The data
  593  * is written to the file, the TTH calculation is updated and checked with the
  594  * DB, and the bitmap is updated.
  595  * This function may be called from another OS thread.
  596  * Returns TRUE to indicate success, FALSE on failure. */
  597 gboolean dlfile_recv(void *vt, const char *buf, int len) {
  598   dlfile_thread_t *t = vt;
  599   if(!dlfile_recv_write(t, buf, len))
  600     return FALSE;
  601 
  602   while(len > 0) {
  603     guint32 inchunk = MIN((guint32)len, DLFILE_CHUNKSIZE - t->len);
  604     t->len += inchunk;
  605     gboolean islast = ((guint64)t->chunk * DLFILE_CHUNKSIZE) + t->len == t->dl->size;
  606 
  607     if(!t->dl->islist)
  608       tth_update(&t->hash_tth, buf, inchunk);
  609     buf += inchunk;
  610     len -= inchunk;
  611 
  612     g_static_mutex_lock(&t->dl->lock);
  613     t->dl->have += inchunk;
  614 
  615     if(!islast && t->len < DLFILE_CHUNKSIZE) {
  616       g_static_mutex_unlock(&t->dl->lock);
  617       continue;
  618     }
  619 
  620     if(!t->dl->islist) {
  621       bita_set(t->dl->bitmap, t->chunk);
  622       dlfile_save_bitmap_defer(t->dl);
  623     }
  624     t->chunk++;
  625     t->allocated--;
  626     t->avail--;
  627     t->len = 0;
  628     g_static_mutex_unlock(&t->dl->lock);
  629 
  630     if(!t->dl->islist && (islast || t->chunk % (t->dl->hash_block / DLFILE_CHUNKSIZE) == 0)) {
  631       char leaf[24];
  632       tth_final(&t->hash_tth, leaf);
  633       tth_init(&t->hash_tth);
  634       if(!dlfile_recv_check(t, leaf))
  635         return FALSE;
  636     }
  637   }
  638   return TRUE;
  639 }
  640 
  641 
  642 void dlfile_recv_done(dlfile_thread_t *t) {
  643   dl_t *dl = t->dl;
  644   dl->active_threads--;
  645   t->busy = FALSE;
  646 
  647   gboolean freet = FALSE;
  648   if(dl->islist ? dl->hassize && dl->have == dl->size : !t->avail) {
  649     g_return_if_fail(!(t->err || t->uerr)); /* A failed thread can't be complete */
  650     dl->threads = g_slist_remove(dl->threads, t);
  651     freet = TRUE;
  652   } else {
  653     t->allocated = 0;
  654     dl->allbusy = FALSE;
  655   }
  656   dlfile_threaddump(dl, 3);
  657 
  658   /* File has been removed from the queue but the dl struct is still in memory
  659    * because this thread hadn't finished yet. Free it now. Note that the actual
  660    * call to dl_queue_rm() is deferred, because we can't access *t after
  661    * calling it. */
  662   gboolean doclose = !dl->bitmap_src && !dl->active_threads;
  663   gboolean dorm = FALSE;
  664   if(!dl->active_threads && !g_hash_table_lookup(dl_queue, dl->hash)) {
  665     dorm = TRUE;
  666     doclose = FALSE;
  667   } else if(t->err)
  668     dl_queue_seterr(t->dl, t->err, t->err_msg);
  669   else if(t->uerr)
  670     dl_queue_setuerr(t->uid, t->dl->hash, t->uerr, t->uerr_msg);
  671   else if(!dl->threads) {
  672     dlfile_finished(dl);
  673     doclose = FALSE;
  674   }
  675 
  676   if(doclose) {
  677     close(dl->incfd);
  678     dl->incfd = 0;
  679   }
  680 
  681   g_free(t->err_msg);
  682   g_free(t->uerr_msg);
  683   t->err = t->uerr = 0;
  684   t->err_msg = t->uerr_msg = NULL;
  685   if(freet)
  686     g_slice_free(dlfile_thread_t, t);
  687 
  688   if(dorm)
  689     dl_queue_rm(dl);
  690 }