lessfs  1.7.0
About: Lessfs is a inline data deduplicating file system for Linux (implemented in user space with FUSE; especially useful for backup purposes).
  Fossies Dox: lessfs-1.7.0.tar.gz  ("inofficial" and yet experimental doxygen-generated source code documentation)  

file_io.c
Go to the documentation of this file.
1 /*
2  * Lessfs: A data deduplicating filesystem.
3  * Copyright (C) 2008 Mark Ruijter <mruijter@lessfs.com>
4  *
5  * This program is free software.
6  * You can redistribute lessfs and/or modify it under the terms of either
7  * (1) the GNU General Public License; either version 3 of the License,
8  * or (at your option) any later version as published by
9  * the Free Software Foundation; or (2) obtain a commercial license
10  * by contacting the Author.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
15  * the GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20  */
21 #define _GNU_SOURCE
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 #ifndef LFATAL
26 #include "lib_log.h"
27 #endif
28 
29 #include <unistd.h>
30 #include <sys/types.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <errno.h>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <time.h>
38 #include <sys/file.h>
39 #include <fuse.h>
40 
41 #include <fcntl.h>
42 #include <pthread.h>
43 #include <libgen.h>
44 
45 #include <tcutil.h>
46 #include <tcbdb.h>
47 #include <tchdb.h>
48 #include <stdlib.h>
49 #include <stdbool.h>
50 #include <stdint.h>
51 #include <aio.h>
52 #include "lib_safe.h"
53 #include "lib_cfg.h"
54 #include "retcodes.h"
55 #ifdef LZO
56 #include "lib_lzo.h"
57 #endif
58 #include "lib_qlz.h"
59 #include "lib_qlz15.h"
60 #include "lib_common.h"
61 #ifndef COMMERCIAL
62 #include "lib_tc.h"
63 #else
64 #include "lib_hamster.h"
65 #endif
66 #include "lib_crypto.h"
67 #include "file_io.h"
68 #include "lib_repl.h"
69 #include "lib_net.h"
70 
71 extern char *logname;
72 extern char *function;
73 extern int debug;
74 extern int BLKSIZE;
75 extern int max_threads;
76 extern char *passwd;
77 
78 extern TCHDB *dbb;
79 extern TCHDB *dbu;
80 extern TCHDB *dbp;
81 extern TCBDB *dbl;
82 extern TCHDB *dbs;
83 extern TCHDB *dbdta;
84 extern TCBDB *dbdirent;
85 extern TCBDB *freelist;
86 extern TCTREE *workqtree;
87 extern TCTREE *readcachetree;
88 extern int fdbdta;
89 
90 extern unsigned long long nextoffset;
91 extern pthread_spinlock_t dbu_spinlock;
92 extern unsigned int dbu_qcount;
93 
94 #define die_dataerr(f...) { LFATAL(f); exit(EXIT_DATAERR); }
95 
96 INUSE *file_get_inuse(unsigned char *stiger)
97 {
98  INUSE *inuse;
99  OLDINUSE *oldinuse;
100 
101  DAT *data;
102 
103  FUNC;
104  if (NULL == stiger)
105  return NULL;
106  data = search_dbdata(DBU, stiger, config->hashlen, LOCK);
107  if (NULL == data) {
108  LDEBUG("file_get_inuse: nothing found return NULL.");
109  return NULL;
110  }
111  if ( data->size == sizeof(INUSE )) {
112  inuse = (INUSE *) data->data;
113  } else {
114  oldinuse=(OLDINUSE *) data->data;
115  inuse=s_zmalloc(sizeof(INUSE));
116  inuse->inuse=oldinuse->inuse;
117  inuse->offset=oldinuse->offset;
118  inuse->size=oldinuse->size;
119  s_free(data->data);
120  }
121  s_free(data);
122  EFUNC;
123  return inuse;
124 }
125 
126 void file_update_inuse(unsigned char *stiger, INUSE * inuse)
127 {
128  FUNC;
129  LDEBUG
130  ("file_update_inuse : update inuse->size = %lu, inuse->inuse = %llu",
131  inuse->size, inuse->inuse);
132  if (inuse) {
134  (unsigned char *) inuse, sizeof(INUSE));
135  }
136  EFUNC;
137  return;
138 }
139 
140 unsigned long long round_512(unsigned long long size)
141 {
142  unsigned long long so;
143  unsigned long long bytes;
144 
145  FUNC;
146  so = size / 512;
147  if (so != 0) {
148  so = so * 512;
149  }
150  if (so < size) {
151  bytes = 512 + so;
152  } else
153  bytes = so;
154  LDEBUG("round_512 : bytes is %llu : size %llu", bytes, size);
155  return bytes;
156 }
157 
158 void set_new_offset(unsigned long long size)
159 {
160  unsigned long long offset = 0;
161 
162  FUNC;
163  if (config->nospace == -ENOSPC) {
164  LINFO("set_new_offset : out of disk space : ENOSPC");
165  config->nospace = ENOSPC;
166  }
167  LDEBUG("oldoffset is %llu : add size %llu", nextoffset, size);
168  offset = round_512(size);
169  nextoffset = nextoffset + offset;
170  if (config->replication == 1 && config->replication_role == 0) {
172  (char *) &nextoffset, sizeof(unsigned long long),
173  NULL, 0, MAX_ALLOWED_THREADS - 2);
174  }
175  LDEBUG("nextoffset is now %llu", nextoffset);
176  EFUNC;
177  return;
178 }
179 
180 char *hash_to_path(unsigned char *thehash,unsigned long long chunk_store)
181 {
182  unsigned char *aschash;
183  char *path=NULL;
184  char *fullpath;
185  int len;
186  int d;
187 
188  aschash=hash_to_ascii(thehash);
189  switch(chunk_store)
190  {
191  case HIGH_PRIO:
192  path=s_zmalloc(1+strlen("/high")+strlen(config->blockdata)+2*config->chunk_depth);
193  sprintf(path,"%s/high",config->blockdata);
194  break;
195  case MEDIUM_PRIO:
196  path=s_zmalloc(1+strlen("/medium")+strlen(config->blockdata)+2*config->chunk_depth);
197  sprintf(path,"%s/medium",config->blockdata);
198  break;
199  path=s_zmalloc(1+strlen("/low")+strlen(config->blockdata)+2*config->chunk_depth);
200  sprintf(path,"%s/low",config->blockdata);
201  case LOW_PRIO:
202  break;
203  default:
204  die_dataerr("chunk_write : Unknown prio");
205  }
206  len=strlen(path);
207  len++;
208  for ( d=0; d<config->chunk_depth*2;d+=2){
209  path[len+d-1]='/';
210  path[len+d]=aschash[d/2];
211  }
212  fullpath=as_sprintf(__FILE__,__LINE__,"%s/%s",path,aschash);
213  s_free(aschash);
214  s_free(path);
215  return fullpath;
216 }
217 
218 DAT *chunk_read(unsigned char *thehash,INUSE *inuse)
219 {
220  DAT *encrypted;
221  char *fullpath;
222  int fd;
223 
224  fullpath=hash_to_path(thehash,inuse->offset);
225  encrypted=s_zmalloc(sizeof(DAT));
226  encrypted->data=s_zmalloc(inuse->size);
227  if (-1 ==
228  (fd =
229  s_open2(fullpath, O_RDONLY | O_NOATIME, S_IRWXU)))
230  die_syserr();
231  encrypted->size=fullRead(fd,encrypted->data,inuse->size);
232  if ( encrypted->size != inuse->size)
233  die_dataerr("chunk_read : unexpected short read %lu expected %lu: data corruption?",
234  encrypted->size,inuse->size);
235  close(fd);
236  return(encrypted);
237 }
238 
239 void chunk_delete(unsigned char *thehash,unsigned long long chunk_store)
240 {
241  char *fullpath;
242  fullpath=hash_to_path(thehash,chunk_store);
243  if ( -1 == unlink(fullpath)) die_syserr();
244  s_free(fullpath);
245 }
246 
247 void chunk_write(unsigned char *thehash,DAT *compressed, unsigned long long chunk_store)
248 {
249  int fd;
250  char *fullpath;
251 
252  fullpath=hash_to_path(thehash,chunk_store);
253  if (-1 ==
254  (fd =
255  s_open2(fullpath, O_CREAT | O_TRUNC| O_RDWR| O_NOATIME, S_IRWXU)))
256  die_syserr();
257  fullWrite(fd,compressed->data, compressed->size);
258  close(fd);
259  s_free(fullpath);
260 }
261 
262 void fl_write_cache(CCACHEDTA * ccachedta, INOBNO * inobno)
263 {
264  INUSE *inuse;
265  DAT *compressed;
266 
267  FUNC;
268  create_hash_note((unsigned char *) &ccachedta->hash);
269  file_delete_stored(inobno);
270  inuse = file_get_inuse((unsigned char *) &ccachedta->hash);
271  if (NULL == inuse) {
272  compressed = lfscompress(ccachedta->data, ccachedta->datasize);
273  if ( config->blockdata_io_type == CHUNK_IO ) {
274  inuse = s_zmalloc(sizeof(INUSE));
275  inuse->offset = MEDIUM_PRIO;
276  inuse->size = compressed->size;
277  inuse->allocated_size = compressed->size;
278  chunk_write((unsigned char *) &ccachedta->hash,compressed,inuse->offset);
279  } else {
280  inuse = get_offset(compressed->size);
281  s_lckpwrite(fdbdta, compressed->data, compressed->size,
282  inuse->offset);
283  }
284  if (config->replication == 1 && config->replication_role == 0) {
285  write_repl_data(FDBDTA, REPLWRITE, (char *) &inuse->offset,
286  sizeof(unsigned long long),
287  (char *) compressed->data, compressed->size,
288  MAX_ALLOWED_THREADS - 2);
289  }
290  if (ccachedta->newblock == 1)
291  update_meta(inobno->inode, compressed->size, 1);
292  if (ccachedta->updated != 0) {
293  if (compressed->size > ccachedta->updated) {
294  update_meta(inobno->inode,
295  compressed->size - ccachedta->updated, 1);
296  } else {
297  update_meta(inobno->inode,
298  ccachedta->updated - compressed->size, 0);
299  }
300  }
301  DATfree(compressed);
302  }
303  inuse->inuse++;
304  LDEBUG("inuse->inuse=%llu : %llu-%llu", inuse->inuse, inobno->inode,
305  inobno->blocknr);
306  file_update_inuse((unsigned char *) &ccachedta->hash, inuse);
307  delete_hash_note((unsigned char *) &ccachedta->hash);
308  bin_write_dbdata(DBB, (char *) inobno, sizeof(INOBNO), ccachedta->hash,
309  config->hashlen);
310  s_free(inuse);
311  ccachedta->dirty = 0;
312  ccachedta->pending = 0;
313  ccachedta->newblock = 0;
314  EFUNC;
315  return;
316 }
317 
318 /* delete = 1 Do delete dbdata
319  delete = 0 Do not delete dbdta */
320 unsigned int file_commit_block(unsigned char *dbdata,
321  INOBNO inobno, unsigned long dsize)
322 {
323  unsigned char *stiger;
324  DAT *compressed;
325  INUSE *inuse;
326  unsigned int ret = 0;
327 
328  FUNC;
329  stiger = thash(dbdata, dsize);
330  create_hash_note(stiger);
331  inuse = file_get_inuse(stiger);
332  if (NULL == inuse) {
333  compressed = lfscompress((unsigned char *) dbdata, dsize);
334  if ( config->blockdata_io_type == CHUNK_IO ) {
335  inuse = s_zmalloc(sizeof(INUSE));
336  inuse->offset = MEDIUM_PRIO;
337  inuse->size = compressed->size;
338  inuse->allocated_size = compressed->size;
339  chunk_write((unsigned char *) stiger,compressed,inuse->offset);
340  } else {
341  inuse = get_offset(compressed->size);
342  s_lckpwrite(fdbdta, compressed->data, compressed->size,
343  inuse->offset);
344  }
345  update_meta(inobno.inode, compressed->size, 1);
346  if (config->replication == 1 && config->replication_role == 0) {
347  write_repl_data(FDBDTA, REPLWRITE, (char *) &inuse->offset,
348  sizeof(unsigned long long),
349  (char *) compressed->data, compressed->size,
350  MAX_ALLOWED_THREADS - 2);
351  }
352  DATfree(compressed);
353  } else {
354  loghash("commit_block : only updated inuse for hash ", stiger);
355  }
356  inuse->inuse++;
357  file_update_inuse(stiger, inuse);
358  bin_write_dbdata(DBB, (char *) &inobno, sizeof(INOBNO), stiger,
359  config->hashlen);
360  delete_hash_note(stiger);
361  free(stiger);
362  s_free(inuse);
363  return (ret);
364 }
365 
366 /* Read a block of data from file */
367 DAT *file_tgr_read_data(unsigned char *stiger)
368 {
369  INUSE *inuse = NULL;
370  DAT *encrypted = NULL;
371 
372  FUNC;
373  inuse = file_get_inuse(stiger);
374  if (inuse) {
375  if (inuse->inuse == 0)
376  die_dataerr("file_tgr_read_data : read empty block");
377  LDEBUG("file_tgr_read_data : read offset %llu, size %lu",
378  inuse->offset, inuse->size);
379  if ( config->blockdata_io_type == CHUNK_IO ) {
380  encrypted=chunk_read(stiger,inuse);
381  } else {
382  encrypted = s_malloc(sizeof(DAT));
383  encrypted->data = s_malloc(inuse->size);
384  encrypted->size =
385  (unsigned long) s_lckpread(fdbdta, encrypted->data,
386  inuse->size, inuse->offset);
387  if ( inuse->size != encrypted->size ) {
388  log_fatal_hash("Please report this bug and include hash and offset",stiger);
389  die_dataerr("file_tgr_read_data : read block expected offset %llu, size %lu, got size %lu",
390  inuse->offset,inuse->size,encrypted->size);
391  }
392  }
393  s_free(inuse);
394  } else {
395  loghash("file_tgr_read_data - unable to find dbdta block hash :",
396  stiger);
397  encrypted = NULL;
398  }
399  EFUNC;
400  return encrypted;
401 }
402 
403 unsigned long long file_read_block(unsigned long long blocknr,
404  char *blockdata, size_t rsize,
405  size_t block_offset,
406  unsigned long long inode)
407 {
408  char *cachedata;
409  DAT *cdata;
410  DAT *data;
411  INOBNO inobno;
412  int ret = 0;
413  CCACHEDTA *ccachedta;
414  DAT *tdata;
415  int vsize;
416  uintptr_t p;
417  inobno.inode = inode;
418  inobno.blocknr = blocknr;
419  int locked;
420 
421  locked = inode_isnot_locked(inode);
422  cachedata =
423  (char *) tctreeget(readcachetree, (void *) &inobno, sizeof(INOBNO),
424  &vsize);
425  if (NULL == cachedata) {
426  tdata = check_block_exists(&inobno);
427  if (NULL == tdata) {
428  return (0);
429  }
430  cdata = file_tgr_read_data(tdata->data);
431  if (NULL == cdata) {
432  auto_repair(&inobno);
433  DATfree(tdata);
434  return (0);
435  }
436  DATfree(tdata);
437  data = lfsdecompress(cdata);
438  if ( NULL == data ) die_dataerr("file_read_block : inode %llu - %llu failed to decompress block", inobno.inode,inobno.blocknr);
439  if (block_offset < data->size) {
440  if (rsize > data->size - block_offset) {
441  memcpy(blockdata, data->data + block_offset,
442  data->size - block_offset);
443  } else {
444  memcpy(blockdata, data->data + block_offset, rsize);
445  }
446  }
447  DATfree(cdata);
448  if (locked) {
449 // When we read a block < BLKSIZE there it is likely that we need
450 // to read it again so it makes sense to put it in a cache.
451  if (rsize < BLKSIZE) {
452 // Make sure that we don't overflow the cache.
453  if (tctreernum(workqtree) * 2 > config->cachesize ||
454  tctreernum(readcachetree) * 2 > config->cachesize) {
455  flush_wait(inobno.inode);
456  purge_read_cache(inobno.inode, 1,
457  (char *) __PRETTY_FUNCTION__);
458  }
459  ccachedta = s_zmalloc(sizeof(CCACHEDTA));
460  p = (uintptr_t) ccachedta;
461  ccachedta->dirty = 0;
462  ccachedta->pending = 0;
463  ccachedta->creationtime = time(NULL);
464  memcpy(&ccachedta->data, data->data, data->size);
465  ccachedta->datasize = data->size;
466  tctreeput(readcachetree, (void *) &inobno, sizeof(INOBNO),
467  (void *) &p, sizeof(p));
468  }
469  }
470  DATfree(data);
471  ret = BLKSIZE;
472  return (ret);
473  }
474 // Fetch the block from disk and put it in the cache.
475  memcpy(&p, cachedata, vsize);
476  ccachedta = get_ccachedta(p);
477  ccachedta->creationtime = time(NULL);
478  if (rsize > BLKSIZE - block_offset) {
479  memcpy(blockdata, &ccachedta->data[block_offset],
480  BLKSIZE - block_offset);
481  } else {
482  memcpy(blockdata, &ccachedta->data[block_offset], rsize);
483  }
484  ret = BLKSIZE;
485  return (ret);
486 }
487 
488 /* The freelist is a btree with as key the number
489  of blocks (512 bytes). The value is offset.
490 */
491 void put_on_freelist(INUSE * inuse, unsigned long long inode)
492 {
493  FREEBLOCK freeblock;
494  unsigned long long scalc;
495  time_t thetime;
496 
497  FUNC;
498 
499  get_offset_lock((char *) __PRETTY_FUNCTION__);
500  // Delete these bytes from real_size;
501  update_meta(inode, inuse->size, 0);
502  thetime = time(NULL);
503  if (config->nospace != ENOSPC) {
504  freeblock.reuseafter = thetime + (config->flushtime * 10);
505  } else {
506  freeblock.reuseafter = thetime;
507  }
508  freeblock.offset = inuse->offset;
509  scalc = inuse->allocated_size / 512;
510  btbin_write_dup(FREELIST, (void *) &scalc, sizeof(unsigned long long),
511  (void *) &freeblock, sizeof(FREEBLOCK), LOCK);
512  LDEBUG("put_on_freelist : %llu blocks at offset %llu", scalc,
513  inuse->offset);
515  EFUNC;
516  return;
517 }
518 
519 CCACHEDTA *file_update_stored(unsigned char *hash, INOBNO * inobno,
520  off_t offsetblock)
521 {
522  DAT *data;
523  DAT *uncompdata;
524  CCACHEDTA *ccachedta;
525 
526  ccachedta = s_zmalloc(sizeof(CCACHEDTA));
527  ccachedta->creationtime = time(NULL);
528  ccachedta->dirty = 1;
529  ccachedta->pending = 0;
530  ccachedta->newblock = 0;
531 
532  data = file_tgr_read_data(hash);
533  if (NULL == data) {
534  s_free(ccachedta);
535  return NULL;
536  }
537  uncompdata = lfsdecompress(data);
538  if ( NULL == uncompdata ) die_dataerr("file_update_stored : inode %llu - %llu failed to decompress block", inobno->inode,inobno->blocknr);
539  memcpy(&ccachedta->data, uncompdata->data, uncompdata->size);
540  ccachedta->datasize = uncompdata->size;
541  ccachedta->updated = data->size;
542  DATfree(uncompdata);
543  DATfree(data);
544  file_delete_stored(inobno);
545  EFUNC;
546  return ccachedta;
547 }
548 
550 {
551  unsigned char *hash;
552  DAT *data;
553  INUSE *inuse;
554 
555  data = search_dbdata(DBB, inobno, sizeof(INOBNO), LOCK);
556  if (NULL == data)
557  return;
558  hash = data->data;
559  inuse = file_get_inuse(hash);
560  if (NULL == inuse)
561  return;
562  delete_dbb(inobno);
563 
564  if (inuse->inuse <= 1) {
565  if ( config->blockdata_io_type == CHUNK_IO ) {
566  chunk_delete(hash,inuse->offset);
567  } else {
568  put_on_freelist(inuse, inobno->inode);
569  }
571  } else {
572  inuse->inuse--;
573  file_update_inuse(hash, inuse);
574  }
575  DATfree(data);
576  s_free(inuse);
577  return;
578 }
579 
580 void *file_truncate_worker(void *threadarg)
581 {
582  unsigned long long blocknr;
583  unsigned long long lastblocknr;
584  struct truncate_thread_data *trunc_data;
585  unsigned char *stiger;
586  DAT *data;
587  INOBNO inobno;
588  unsigned int offsetblock;
589  INUSE *inuse;
590 
591  trunc_data = (struct truncate_thread_data *) threadarg;
592  lastblocknr = trunc_data->lastblocknr;
593  blocknr = trunc_data->blocknr;
594  offsetblock = trunc_data->offsetblock;
595  inobno.blocknr = trunc_data->blocknr;
596  inobno.inode = trunc_data->inode;
597 
598  LDEBUG("Start truncation thread : inode %llu", inobno.inode);
599 
600  while (lastblocknr >= blocknr) {
601  if (config->background_delete) {
602  truncation_wait();
603  write_lock((char *) __PRETTY_FUNCTION__);
604  }
605  if (offsetblock != 0 && lastblocknr == blocknr) {
608  break;
609  }
610  LDEBUG
611  ("file_fs_truncate : Enter loop lastblocknr %llu : blocknr %llu",
613  inobno.blocknr = lastblocknr;
614  data = search_dbdata(DBB, &inobno, sizeof(INOBNO), LOCK);
615  if (NULL == data) {
616  LDEBUG
617  ("file_fs_truncate: deletion of non existent block inode : %llu, blocknr %llu",
618  inobno.inode, inobno.blocknr);
619  if (lastblocknr > 0)
620  lastblocknr--;
621  else {
624  break;
625  }
628 // Need to continue in case of a sparse file.
629  continue;
630  }
631  stiger = s_malloc(data->size);
632  memcpy(stiger, data->data, data->size);
633  LDEBUG
634  ("file_fs_truncate : lessfs_truncate Search to delete blocknr %llu:",
635  lastblocknr);
636  DATfree(data);
637  create_hash_note(stiger);
638  delete_dbb(&inobno);
639  inuse = file_get_inuse(stiger);
640  if (inuse) {
641  if (inuse->inuse == 1) {
642  if ( config->blockdata_io_type == CHUNK_IO ) {
643  chunk_delete(stiger,inuse->offset);
644  } else {
645  put_on_freelist(inuse, inobno.inode);
646  }
647  delete_inuse(stiger);
648  LDEBUG("file_fs_truncate : delete dbb %llu-%llu",
649  inobno.inode, inobno.blocknr);
650  } else {
651  if (inuse->inuse > 1)
652  inuse->inuse--;
653  LDEBUG
654  ("file_fs_truncate : delete dbb %llu-%llu : inuse = %llu",
655  inobno.inode, inobno.blocknr, inuse->inuse);
656  file_update_inuse(stiger, inuse);
657  }
658  } else
659  LDEBUG("file_fs_truncate : inuse is NULL %llu-%llu",
660  inobno.inode, inobno.blocknr);
661  delete_hash_note(stiger);
662  s_free(inuse);
663  s_free(stiger);
666  if (lastblocknr > 0)
667  lastblocknr--;
668  else
669  break;
670  if (config->background_delete) {
671  if (config->shutdown) {
672  LINFO("Truncation thread aborts truncating inode %llu",
673  inobno.inode);
674  return NULL;
675  }
676  }
677  }
678  if (0 != offsetblock) {
680  write_lock((char *) __PRETTY_FUNCTION__);
682  offsetblock);
685  }
686  delete_inode_note(inobno.inode);
687  LDEBUG("Completed truncating of inode %llu", inobno.inode);
688  if ( config->background_delete ) write_lock((char *) __PRETTY_FUNCTION__);
689  btdelete_curkey(FREELIST, "TRNCTD", strlen("TRNCTD"),
690  threadarg, sizeof(struct truncate_thread_data),
691  (char *) __PRETTY_FUNCTION__);
693  s_free(threadarg);
694  return NULL;
695 }
696 
697 int file_fs_truncate(struct stat *stbuf, off_t size, char *bname,
698  bool unlink)
699 {
700  unsigned int offsetblock;
701  unsigned long long blocknr;
702  unsigned long long lastblocknr;
703  off_t oldsize;
704  time_t thetime;
705  pthread_t truncate_thread;
706  struct truncate_thread_data *trunc_data;
707 
708  FUNC;
709  trunc_data = s_zmalloc(sizeof(struct truncate_thread_data));
710  LDEBUG("file_fs_truncate inode %llu - size %llu", stbuf->st_ino,
711  (unsigned long long) size);
712  thetime = time(NULL);
713  blocknr = size / BLKSIZE;
714  offsetblock = size - (blocknr * BLKSIZE);
715  oldsize = stbuf->st_size;
716  lastblocknr = oldsize / BLKSIZE;
717  trunc_data->inode = stbuf->st_ino;
718  trunc_data->blocknr = blocknr;
719  trunc_data->lastblocknr = lastblocknr;
720  trunc_data->offsetblock = offsetblock;
721  trunc_data->unlink = unlink;
722  memcpy(&trunc_data->stbuf, stbuf, sizeof(struct stat));
723  // Truncate filesize.
724  if (!unlink)
726  if (config->background_delete == 0) {
727  file_truncate_worker((void *) trunc_data);
728  } else {
729  write_trunc_todolist(trunc_data);
730  if (0 !=
731  pthread_create(&truncate_thread, NULL, file_truncate_worker,
732  (void *) trunc_data))
733  die_syserr();
734  if (0 != pthread_detach(truncate_thread))
735  die_syserr();
736  }
737  return (0);
738 }
739 
740 void file_partial_truncate_block(unsigned long long inode,
741  unsigned long long blocknr,
742  unsigned int offset)
743 {
744  unsigned char *blockdata;
745  DAT *uncompdata;
746  INOBNO inobno;
747  DAT *data;
748  unsigned char *stiger;
749  INUSE *inuse;
750 
751  FUNC;
752  inobno.inode = inode;
753  inobno.blocknr = blocknr;
754 
755  data = search_dbdata(DBB, &inobno, sizeof(INOBNO), LOCK);
756  if (NULL == data) {
757  LDEBUG("Deletion of non existent block?");
758  return;
759  }
760  stiger = s_malloc(data->size);
761  memcpy(stiger, data->data, data->size);
762  DATfree(data);
763  blockdata = s_zmalloc(BLKSIZE);
764  data = file_tgr_read_data(stiger);
765  if (NULL == data) {
766  die_dataerr("Hmmm, did not expect this to happen.");
767  }
768  LDEBUG("file_partial_truncate_block : clz_decompress");
769  uncompdata = lfsdecompress(data);
770  if ( NULL == uncompdata ) die_dataerr("file_partial_truncate_block : inode %llu - %llu failed to decompress block", inobno.inode,inobno.blocknr);
771  if (uncompdata->size >= offset) {
772  memcpy(blockdata, uncompdata->data, offset);
773  } else {
774  memcpy(blockdata, uncompdata->data, uncompdata->size);
775  }
776  DATfree(uncompdata);
777  file_commit_block(blockdata, inobno, offset);
778  DATfree(data);
779  s_free(blockdata);
780  create_hash_note(stiger);
781  inuse = file_get_inuse(stiger);
782  if (NULL == inuse)
784  ("file_partial_truncate_block : unexpected block not found");
785  if (inuse->inuse == 1) {
786  loghash("file_partial_truncate_block : delete hash", stiger);
787  if (config->blockdata_io_type == CHUNK_IO ) {
788  chunk_delete(stiger,inuse->offset);
789  } else {
790  put_on_freelist(inuse, inobno.inode);
791  }
792  delete_inuse(stiger);
793  } else {
794  if (inuse->inuse > 1)
795  inuse->inuse--;
796  file_update_inuse(stiger, inuse);
797  }
798  delete_hash_note(stiger);
799  s_free(inuse);
800  s_free(stiger);
801  return;
802 }
803 
804 int file_unlink_file(const char *path)
805 {
806  int res = 0;
807  int haslinks = 0;
808  int dir_links = 0;
809  struct stat st;
810  struct stat dirst;
811  char *dname;
812  char *bname;
813  unsigned long long inode;
814  time_t thetime;
815  DAT *vdirnode;
816  DAT *ddbuf;
817  DAT *dataptr;
818  DDSTAT *ddstat;
819  DINOINO dinoino;
820  INOBNO inobno;
821  DAT *fname;
822 
823  FUNC;
824 
825  LDEBUG("unlink_file %s", path);
826  res = dbstat(path, &st, 0);
827  if (res == -ENOENT)
828  return (res);
829  inode = st.st_ino;
830  haslinks = st.st_nlink;
831  thetime = time(NULL);
832  dname = s_dirname((char *) path);
833  /* Change ctime and mtime of the parentdir Posix std posix behavior */
834  res = update_parent_time(dname, 0);
835  bname = s_basename((char *) path);
836  res = dbstat(dname, &dirst, 0);
837  if (S_ISLNK(st.st_mode) && haslinks == 1) {
838  LDEBUG("unlink symlink %s inode %llu", path, inode);
839  delete_key(DBS, &inode, sizeof(unsigned long long),
840  (char *) __PRETTY_FUNCTION__);
841  LDEBUG("unlink symlink done %s", path);
842  }
843  inobno.inode = inode;
844  inobno.blocknr = st.st_size / BLKSIZE;
845  if (inobno.blocknr * BLKSIZE < st.st_size)
846  inobno.blocknr++;
847 
848 // Start deleting the actual data blocks.
849  (void) file_fs_truncate(&st, 0, bname, 1);
850 // Remove this inode from the path_to_inode cache
851  LDEBUG("file_unlink_file %s haslinks =%u", path, haslinks);
852  if (haslinks == 1) {
853  if (0 !=
854  (res =
855  btdelete_curkey(DBDIRENT, &dirst.st_ino,
856  sizeof(unsigned long long), &inode,
857  sizeof(unsigned long long),
858  (char *) __PRETTY_FUNCTION__))) {
859  s_free(bname);
860  s_free(dname);
861  LDEBUG("file_unlink_file : %lu : %llu", dirst.st_ino, inode);
862  return (res);
863  }
864  delete_key(DBP, (unsigned char *) &inode,
865  sizeof(unsigned long long),
866  (char *) __PRETTY_FUNCTION__);
867  LDEBUG("file_unlink_file : unlinked DBDIRENT %lu : %llu",
868  dirst.st_ino, inode);
869  } else {
870  dataptr =
871  search_dbdata(DBP, (unsigned char *) &inode,
872  sizeof(unsigned long long), LOCK);
873  if (dataptr == NULL) {
874  die_dataerr("Failed to find file %llu", inode);
875  }
876  ddstat = value_to_ddstat(dataptr);
877  ddstat->stbuf.st_nlink--;
878  ddstat->stbuf.st_ctim.tv_sec = thetime;
879  ddstat->stbuf.st_ctim.tv_nsec = 0;
880  ddstat->stbuf.st_mtim.tv_sec = thetime;
881  ddstat->stbuf.st_mtim.tv_nsec = 0;
882  dinoino.dirnode = dirst.st_ino;
883  dinoino.inode = ddstat->stbuf.st_ino;
884  dir_links = count_dirlinks(&dinoino, sizeof(DINOINO));
885  res =
886  btdelete_curkey(DBL, &dinoino, sizeof(DINOINO), bname,
887  strlen(bname), (char *) __PRETTY_FUNCTION__);
888  btdelete_curkey(DBL, &ddstat->stbuf.st_ino,
889  sizeof(unsigned long long), &dinoino,
890  sizeof(DINOINO), (char *) __PRETTY_FUNCTION__);
891 // Restore to regular file settings and clean up.
892  if (ddstat->stbuf.st_nlink == 1) {
893  vdirnode =
894  btsearch_keyval(DBL, &ddstat->stbuf.st_ino,
895  sizeof(unsigned long long), NULL, 0, LOCK);
896  memcpy(&dinoino, vdirnode->data, vdirnode->size);
897  DATfree(vdirnode);
898  fname =
899  btsearch_keyval(DBL, &dinoino, sizeof(DINOINO),
900  NULL, 0, LOCK);
901  memcpy(&ddstat->filename, fname->data, fname->size);
902  DATfree(fname);
903  LDEBUG
904  ("unlink_file : Restore %s to regular file settings and clean up.",
905  ddstat->filename);
906  btdelete_curkey(DBL, &dinoino, sizeof(DINOINO),
907  ddstat->filename, strlen(ddstat->filename),
908  (char *) __PRETTY_FUNCTION__);
909  btdelete_curkey(DBL, &ddstat->stbuf.st_ino,
910  sizeof(unsigned long long), &dinoino,
911  sizeof(DINOINO), (char *) __PRETTY_FUNCTION__);
912  btdelete_curkey(DBL, &inode, sizeof(unsigned long long),
913  &dinoino, sizeof(DINOINO),
914  (char *) __PRETTY_FUNCTION__);
915  res = 0;
916  }
917  if (dir_links == 1) {
918  if (0 !=
919  (res =
920  btdelete_curkey(DBDIRENT, &dirst.st_ino,
921  sizeof(unsigned long long), &inode,
922  sizeof(unsigned long long),
923  (char *) __PRETTY_FUNCTION__))) {
924  die_dataerr("unlink_file : Failed to delete record.");
925  }
926  }
927  ddbuf =
928  create_ddbuf(ddstat->stbuf, ddstat->filename,
929  ddstat->real_size);
930  bin_write_dbdata(DBP, &inode, sizeof(unsigned long long),
931  (void *) ddbuf->data, ddbuf->size);
932  DATfree(dataptr);
933  DATfree(ddbuf);
934  ddstatfree(ddstat);
935  }
936  s_free(bname);
937  s_free(dname);
938  EFUNC;
939  return (res);
940 }
CCACHEDTA::updated
unsigned long updated
Definition: lib_common.h:101
file_tgr_read_data
DAT * file_tgr_read_data(unsigned char *stiger)
Definition: file_io.c:367
chunk_read
DAT * chunk_read(unsigned char *thehash, INUSE *inuse)
Definition: file_io.c:218
create_ddbuf
DAT * create_ddbuf(struct stat stbuf, char *filename, unsigned long long real_size)
Definition: lib_common.c:1023
CCACHEDTA
Definition: lib_common.h:93
fdbdta
int fdbdta
Definition: lib_tc.c:92
CCACHEDTA::hash
unsigned char hash[64]
Definition: lib_common.h:96
lib_hamster.h
lib_qlz.h
lib_safe.h
lfscompress
DAT * lfscompress(unsigned char *dbdata, unsigned long dsize)
Definition: lib_common.c:2191
dbdirent
TCBDB * dbdirent
Definition: lib_common.c:104
lib_common.h
OLDINUSE::size
unsigned long size
Definition: lib_common.h:44
DBL
#define DBL
Definition: lib_repl.h:26
FUNC
#define FUNC
Definition: lib_log.h:76
set_new_offset
void set_new_offset(unsigned long long size)
Definition: file_io.c:158
MAX_ALLOWED_THREADS
#define MAX_ALLOWED_THREADS
Definition: lib_common.h:13
fl_write_cache
void fl_write_cache(CCACHEDTA *ccachedta, INOBNO *inobno)
Definition: file_io.c:262
DATfree
void DATfree(DAT *data)
Definition: lib_common.c:1744
CCACHEDTA::newblock
char newblock
Definition: lib_common.h:100
EFUNC
#define EFUNC
Definition: lib_log.h:77
DAT
Definition: lib_common.h:59
nextoffset
unsigned long long nextoffset
Definition: lib_common.c:90
configdata::background_delete
int background_delete
Definition: lib_cfg.h:69
CCACHEDTA::dirty
int dirty
Definition: lib_common.h:97
dbdta
TCHDB * dbdta
Definition: lib_common.c:103
configdata::replication
int replication
Definition: lib_cfg.h:62
file_update_inuse
void file_update_inuse(unsigned char *stiger, INUSE *inuse)
Definition: file_io.c:126
dbl
TCBDB * dbl
Definition: lib_common.c:101
truncate_thread_data::stbuf
struct stat stbuf
Definition: lib_common.h:89
FREEBLOCK::reuseafter
time_t reuseafter
Definition: file_io.h:24
DAT::data
unsigned char * data
Definition: lib_common.h:61
dbu_spinlock
pthread_spinlock_t dbu_spinlock
INUSE::size
unsigned long size
Definition: lib_common.h:37
MEDIUM_PRIO
#define MEDIUM_PRIO
Definition: lib_common.h:21
ddstatfree
void ddstatfree(DDSTAT *ddstat)
Definition: lib_common.c:1261
s_basename
#define s_basename(path)
Definition: lib_safe.h:28
inode_isnot_locked
int inode_isnot_locked(unsigned long long inode)
Definition: lib_common.c:554
CCACHEDTA::data
unsigned char data[131072]
Definition: lib_common.h:94
passwd
char * passwd
die_dataerr
#define die_dataerr(f...)
Definition: file_io.c:94
lib_log.h
delete_dbb
void delete_dbb(INOBNO *inobno)
Definition: lib_common.c:1713
get_offset_lock
void get_offset_lock(const char *msg)
Definition: lib_common.c:820
fullWrite
int fullWrite(int fd, unsigned char *buf, int len)
Definition: lib_net.c:419
file_fs_truncate
int file_fs_truncate(struct stat *stbuf, off_t size, char *bname, bool unlink)
Definition: file_io.c:697
HIGH_PRIO
#define HIGH_PRIO
Definition: lib_common.h:20
file_read_block
unsigned long long file_read_block(unsigned long long blocknr, char *blockdata, size_t rsize, size_t block_offset, unsigned long long inode)
Definition: file_io.c:403
INUSE::offset
unsigned long long offset
Definition: lib_common.h:36
put_on_freelist
void put_on_freelist(INUSE *inuse, unsigned long long inode)
Definition: file_io.c:491
delete_inuse
void delete_inuse(unsigned char *stiger)
Definition: lib_common.c:1706
FDBDTA
#define FDBDTA
Definition: lib_repl.h:31
truncate_thread_data::inode
unsigned long long inode
Definition: lib_common.h:87
OLDINUSE
Definition: lib_common.h:42
CCACHEDTA::creationtime
time_t creationtime
Definition: lib_common.h:98
configdata::shutdown
int shutdown
Definition: lib_cfg.h:76
value_to_ddstat
DDSTAT * value_to_ddstat(DAT *vddstat)
Definition: lib_common.c:986
release_offset_lock
void release_offset_lock()
Definition: lib_common.c:842
DDSTAT::filename
char filename[256+1]
Definition: lib_common.h:67
DDSTAT::stbuf
struct stat stbuf
Definition: lib_common.h:65
flush_wait
void flush_wait(unsigned long long inode)
Definition: lib_common.c:2564
dbs
TCHDB * dbs
Definition: lib_common.c:102
INOBNO
Definition: lib_common.h:48
freelist
TCBDB * freelist
Definition: lib_common.c:105
file_truncate_worker
void * file_truncate_worker(void *threadarg)
Definition: file_io.c:580
configdata::blockdata
char * blockdata
Definition: lib_cfg.h:25
s_lckpread
int s_lckpread(int fd, void *buf, size_t len, off_t off)
Definition: lib_safe.c:149
DINOINO::dirnode
unsigned long long dirnode
Definition: lib_common.h:55
bin_write_dbdata
void bin_write_dbdata(int, void *, int, void *, int)
Definition: lib_tc.c:658
truncate_thread_data::unlink
bool unlink
Definition: lib_common.h:90
as_sprintf
void * as_sprintf(char *file, unsigned int line, const char *fmt,...)
Definition: lib_safe.c:564
INOBNO::blocknr
unsigned long long blocknr
Definition: lib_common.h:50
s_dirname
#define s_dirname(path)
Definition: lib_safe.h:27
btsearch_keyval
DAT * btsearch_keyval(int, void *, int, void *, int, bool)
Definition: lib_tc.c:919
INOBNO::inode
unsigned long long inode
Definition: lib_common.h:49
truncation_wait
void truncation_wait()
Definition: lib_common.c:966
DBB
#define DBB
Definition: lib_repl.h:23
chunk_delete
void chunk_delete(unsigned char *thehash, unsigned long long chunk_store)
Definition: file_io.c:239
file_update_stored
CCACHEDTA * file_update_stored(unsigned char *hash, INOBNO *inobno, off_t offsetblock)
Definition: file_io.c:519
release_write_lock
void release_write_lock()
Definition: lib_common.c:694
dbu
TCHDB * dbu
Definition: lib_common.c:99
logname
char * logname
Definition: commons.h:5
s_lckpwrite
int s_lckpwrite(int fd, const void *buf, size_t len, off_t off)
Definition: lib_safe.c:198
lib_tc.h
max_threads
int max_threads
Definition: commons.h:9
configdata::nospace
int nospace
Definition: lib_cfg.h:80
btdelete_curkey
int btdelete_curkey(int, void *, int, void *, int, const char *)
Definition: lib_tc.c:854
dbu_qcount
unsigned int dbu_qcount
configdata::flushtime
unsigned int flushtime
Definition: lib_cfg.h:53
LINFO
#define LINFO(f...)
Definition: lib_log.h:69
get_offset
INUSE * get_offset(unsigned long long size)
Definition: lib_common.c:249
configdata::blockdata_io_type
int blockdata_io_type
Definition: lib_cfg.h:27
s_zmalloc
#define s_zmalloc(size)
Definition: lib_safe.h:24
create_hash_note
void create_hash_note(unsigned char *hash)
Definition: lib_common.c:499
BLKSIZE
int BLKSIZE
Definition: commons.h:8
DINOINO
Definition: lib_common.h:53
count_dirlinks
int count_dirlinks(void *, int)
Definition: lib_tc.c:976
update_parent_time
int update_parent_time(char *path, int linkcount)
Definition: lib_common.c:2481
CCACHEDTA::datasize
unsigned long datasize
Definition: lib_common.h:95
write_trunc_todolist
void write_trunc_todolist(struct truncate_thread_data *)
Definition: lib_common.c:2847
lib_crypto.h
lib_net.h
FREEBLOCK
Definition: file_io.h:22
truncate_thread_data::lastblocknr
unsigned long long lastblocknr
Definition: lib_common.h:86
lfsdecompress
DAT * lfsdecompress(DAT *cdata)
Definition: lib_common.c:1527
chunk_write
void chunk_write(unsigned char *thehash, DAT *compressed, unsigned long long chunk_store)
Definition: file_io.c:247
LOCK
#define LOCK
Definition: lib_common.h:5
delete_hash_note
void delete_hash_note(unsigned char *hash)
Definition: lib_common.c:522
LDEBUG
#define LDEBUG(f...)
Definition: lib_log.h:78
file_delete_stored
void file_delete_stored(INOBNO *inobno)
Definition: file_io.c:549
dbstat
int dbstat(const char *filename, struct stat *stbuf, bool cache_request)
Definition: lib_common.c:1204
s_malloc
#define s_malloc(size)
Definition: lib_safe.h:23
file_io.h
file_unlink_file
int file_unlink_file(const char *path)
Definition: file_io.c:804
dbb
TCHDB * dbb
Definition: lib_common.c:98
btbin_write_dup
void btbin_write_dup(int, void *, int, void *, int, bool)
Definition: lib_tc.c:581
update_filesize_cache
int update_filesize_cache(struct stat *stbuf, off_t size)
Definition: lib_common.c:1815
DAT::size
unsigned long size
Definition: lib_common.h:60
file_get_inuse
INUSE * file_get_inuse(unsigned char *stiger)
Definition: file_io.c:96
DDSTAT::real_size
unsigned long long real_size
Definition: lib_common.h:66
REPLWRITE
#define REPLWRITE
Definition: lib_repl.h:12
DBP
#define DBP
Definition: lib_repl.h:24
hash_to_path
char * hash_to_path(unsigned char *thehash, unsigned long long chunk_store)
Definition: file_io.c:180
search_dbdata
DAT * search_dbdata(int, void *, int, bool)
Definition: lib_tc.c:816
retcodes.h
truncate_thread_data::blocknr
unsigned long long blocknr
Definition: lib_common.h:85
thash
unsigned char * thash(unsigned char *buf, int size)
Definition: lib_common.c:216
config
struct configdata * config
Definition: lib_cfg.h:91
INUSE
Definition: lib_common.h:35
configdata::hashlen
int hashlen
Definition: lib_cfg.h:60
debug
int debug
Definition: commons.h:7
lib_repl.h
OLDINUSE::offset
unsigned long long offset
Definition: lib_common.h:43
INUSE::allocated_size
unsigned long long allocated_size
Definition: lib_common.h:38
DBDIRENT
#define DBDIRENT
Definition: lib_repl.h:28
file_commit_block
unsigned int file_commit_block(unsigned char *dbdata, INOBNO inobno, unsigned long dsize)
Definition: file_io.c:320
DDSTAT
Definition: lib_common.h:64
auto_repair
void auto_repair(INOBNO *inobno)
Definition: lib_common.c:1605
lib_cfg.h
DBS
#define DBS
Definition: lib_repl.h:25
update_meta
void update_meta(unsigned long long inode, unsigned long size, int sign)
Definition: lib_common.c:2716
dbp
TCHDB * dbp
Definition: lib_common.c:100
truncate_thread_data
Definition: lib_common.h:84
lib_qlz15.h
purge_read_cache
void purge_read_cache(unsigned long long inode, bool force, char *caller)
Definition: lib_common.c:2645
delete_key
void delete_key(int, void *, int, const char *)
Definition: lib_tc.c:1007
REPLSETNEXTOFFSET
#define REPLSETNEXTOFFSET
Definition: lib_repl.h:17
DINOINO::inode
unsigned long long inode
Definition: lib_common.h:56
get_ccachedta
static CCACHEDTA * get_ccachedta(uintptr_t p)
Definition: lib_common.h:104
configdata::chunk_depth
int chunk_depth
Definition: lib_cfg.h:81
file_partial_truncate_block
void file_partial_truncate_block(unsigned long long inode, unsigned long long blocknr, unsigned int offset)
Definition: file_io.c:740
CHUNK_IO
Definition: lib_common.h:33
LOW_PRIO
#define LOW_PRIO
Definition: lib_common.h:22
workqtree
TCTREE * workqtree
Definition: lib_common.c:108
DBU
#define DBU
Definition: lib_repl.h:22
readcachetree
TCTREE * readcachetree
Definition: lib_common.c:109
OLDINUSE::inuse
unsigned long long inuse
Definition: lib_common.h:45
die_syserr
#define die_syserr()
Definition: lessfsck.c:71
fullRead
int fullRead(int fd, unsigned char *buf, int len)
Definition: lib_net.c:397
INUSE::inuse
unsigned long long inuse
Definition: lib_common.h:39
truncate_thread_data::offsetblock
unsigned int offsetblock
Definition: lib_common.h:88
s_open2
int s_open2(const char *pathname, int flags, mode_t mode)
Definition: lib_safe.c:489
delete_inode_note
void delete_inode_note(unsigned long long inode)
Definition: lib_common.c:587
configdata::hash
char * hash
Definition: lib_cfg.h:47
round_512
unsigned long long round_512(unsigned long long size)
Definition: file_io.c:140
lib_lzo.h
CCACHEDTA::pending
int pending
Definition: lib_common.h:99
FREELIST
#define FREELIST
Definition: lib_repl.h:27
check_block_exists
DAT * check_block_exists(INOBNO *inobno)
Definition: lib_common.c:1971
configdata::cachesize
unsigned long long cachesize
Definition: lib_cfg.h:52
NEXTOFFSET
#define NEXTOFFSET
Definition: lib_repl.h:30
write_repl_data
void write_repl_data(unsigned char db, unsigned char op, char *key, int ksize, char *value, int vsize, int threadnr)
Definition: lib_repl.c:450
loghash
void loghash(char *msg, unsigned char *bhash)
Definition: lib_common.c:451
configdata::replication_role
int replication_role
Definition: lib_cfg.h:63
FREEBLOCK::offset
unsigned long long offset
Definition: file_io.h:23
s_free
#define s_free(mem_ref)
Definition: lib_safe.h:25
write_lock
void write_lock(const char *msg)
Definition: lib_common.c:674
hash_to_ascii
unsigned char * hash_to_ascii(unsigned char *bhash)
Definition: lib_common.c:478
log_fatal_hash
void log_fatal_hash(char *msg, unsigned char *bhash)
Definition: lib_common.c:456