lessfs  1.7.0
About: Lessfs is a inline data deduplicating file system for Linux (implemented in user space with FUSE; especially useful for backup purposes).
  Fossies Dox: lessfs-1.7.0.tar.gz  ("inofficial" and yet experimental doxygen-generated source code documentation)  

replogtool.c
Go to the documentation of this file.
1 /*
2  * Lessfs: A data deduplicating filesystem.
3  * Copyright (C) 2008 Mark Ruijter <mruijter@lessfs.com>
4  *
5  * This program is s_free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
13  * the GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18  */
19 
20 #ifdef HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #ifndef LFATAL
24 #include "lib_log.h"
25 #endif
26 
27 #include <unistd.h>
28 #include <sys/types.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <sys/file.h>
36 #include <fcntl.h>
37 #include <semaphore.h>
38 #include <pthread.h>
39 #include <fuse.h>
40 
41 #include <tcutil.h>
42 #include <tchdb.h>
43 #include <tcbdb.h>
44 #include <stdlib.h>
45 #include <stdbool.h>
46 
47 #include "lib_cfg.h"
48 #include "lib_safe.h"
49 #include "lib_io.h"
50 #include "lib_net.h"
51 #include "lib_str.h"
52 #include "retcodes.h"
53 #ifdef LZO
54 #include "lib_lzo.h"
55 #else
56 #include "lib_qlz.h"
57 #endif
58 #include "lib_common.h"
59 #include "lib_tc.h"
60 #include "lib_repl.h"
61 #include "commons.h"
62 
63 #ifdef ENABLE_CRYPTO
64 unsigned char *passwd = NULL;
65 #endif
66 
67 extern struct configdata *config;
68 extern unsigned long long nextoffset;
69 extern int fdbdta;
70 extern int frepl;
71 extern int rrepl;
72 extern int freplbl; /* Backlog processing */
73 extern int BLKSIZE;
74 
75 #define die_dataerr(f...) { LFATAL(f); exit(EXIT_DATAERR); }
76 
77 struct option_info {
78  char *configfile;
79  char *replogfile;
80  bool notwritten;
81  unsigned int blocksize;
82 };
84 
85 unsigned long working;
86 
87 void usage(char *name)
88 {
89  printf
90  ("Usage:\nShow replog content :\n%s -r /path_to_replogfile -b blocksize \n",
91  name);
92  printf
93  ("\n\nMerge replog with a lessfs slave :\n%s -r /path_to_replogfile -c /path_to_config.cfg\n\n",
94  name);
95  printf
96  ("-r Path and name of the lessfs replication log : -b is mandatory\n");
97  printf
98  ("-c Path and name of the lessfs configuation file, the replog be merged with the lessfs slave filesystem\n");
99  printf("-h Displays this usage message\n");
100  exit(-1);
101 }
102 
103 int get_opts(int argc, char *argv[])
104 {
105 
106  int c;
107 
108  mkoptions.replogfile = NULL;
109  mkoptions.configfile = NULL;
110  mkoptions.notwritten = 0;
111  mkoptions.blocksize = 0;
112  while ((c = getopt(argc, argv, "b:whr:c:")) != -1)
113  switch (c) {
114  case 'b':
115  if (optopt == 'c')
116  printf
117  ("Option -%c requires a lessfs the blocksize as argument.\n",
118  optopt);
119  else {
120  mkoptions.blocksize = atoi(optarg);
122  }
123  break;
124  case 'c':
125  if (optopt == 'c')
126  printf
127  ("Option -%c requires a lessfs configuration file as argument.\n",
128  optopt);
129  else
130  mkoptions.configfile = optarg;
131  break;
132  case 'r':
133  if (optopt == 'r')
134  printf
135  ("Option -%c requires a lessfs replication log file as argument.\n",
136  optopt);
137  else
138  mkoptions.replogfile = optarg;
139  break;
140  case 'w':
141  mkoptions.notwritten = 1;
142  break;
143  case 'h':
144  usage(argv[0]);
145  break;
146  default:
147  abort();
148  }
149  return 0;
150 }
151 
152 char *asc_hash(unsigned char *bhash, int hashlen)
153 {
154  char *ascii_hash = NULL;
155  int n;
156  char *p1, *p2;
157 
158  for (n = 0; n < hashlen; n++) {
159  p1 = as_sprintf(__FILE__, __LINE__, "%02X", bhash[n]);
160  if (n == 0) {
161  ascii_hash = s_strdup(p1);
162  } else {
163  p2 = s_strdup(ascii_hash);
165  ascii_hash = as_sprintf(__FILE__, __LINE__, "%s%s", p2, p1);
166  s_free(p2);
167  }
168  s_free(p1);
169  }
170  return (ascii_hash);
171 }
172 
173 
174 void parsedbp(char *action, REPLICATIONMSG replicationmsg, char *wmsg)
175 {
176  unsigned long long *node;
177  DDSTAT *ddstat;
178 
179  if (replicationmsg.ksize == 3) {
180  node = (unsigned long long *) replicationmsg.value;
181  if (0 == memcmp("NFI", replicationmsg.value, replicationmsg.vsize)) {
182  printf("%s DBP\t:\tNFI %llu\t\t\t\t\t\t\t\t\t\t%s\n", action,
183  *node, wmsg);
184  } else
185  printf("%s DBP\t:\tSEQ %llu\t\t\t\t\t\t\t\t\t\t%s\n", action,
186  *node, wmsg);
187  return;
188  }
189  node = (unsigned long long *) replicationmsg.key;
190  ddstat = (DDSTAT *) replicationmsg.value;
191  printf
192  ("%s DBP\t:\tInode\t%llu\t:\tfilename %s\t- size\t%lu\t\t\t\t%s\n",
193  action, *node, ddstat->filename, ddstat->stbuf.st_size, wmsg);
194  return;
195 }
196 
197 void parseline(char *message, unsigned int msgsize, int is_written)
198 {
199 //message : db (char), op (char), ksize (int),vsize (int), key (char *),value (char *)
200  REPLICATIONMSG replicationmsg;
201  char *wmsg;
202  char *hsh;
203  INOBNO *inobno;
204 
205  if (is_written)
206  wmsg = as_sprintf(__FILE__, __LINE__, "IW");
207  else
208  wmsg = as_sprintf(__FILE__, __LINE__, "NW");
209 
210  replicationmsg.database = message[0];
211  replicationmsg.operation = message[1];
212  memcpy(&replicationmsg.ksize, &message[2], sizeof(int));
213  memcpy(&replicationmsg.vsize, &message[2 + sizeof(int)], sizeof(int));
214  replicationmsg.key = (unsigned char *) &message[2 + (2 * sizeof(int))];
215  replicationmsg.value =
216  (unsigned char *) &message[2 + (2 * sizeof(int)) +
217  replicationmsg.ksize];
218  if (replicationmsg.operation == REPLWRITE) {
219  if (replicationmsg.database == DBDTA) {
220  printf("WRT DBDTA\t: keysize %u\t:\tmsgsize\t%u\t%s\n",
221  replicationmsg.ksize, replicationmsg.vsize, wmsg);
222  }
223  if (replicationmsg.database == DBU) {
224  hsh = asc_hash(replicationmsg.key, replicationmsg.ksize);
225  printf("WRT DBU\t:\t%s\t:\tvalsize\t%u\t%s\n", hsh,
226  replicationmsg.ksize, wmsg);
227  s_free(hsh);
228  }
229  if (replicationmsg.database == DBB) {
230  inobno = (INOBNO *) replicationmsg.key;
231  hsh = asc_hash(replicationmsg.value, replicationmsg.vsize);
232  printf("WRT DBB\t:\t%llu-%llu\t: %s\t\t\t%s\n", inobno->inode,
233  inobno->blocknr, hsh, wmsg);
234  s_free(hsh);
235  }
236  if (replicationmsg.database == DBP) {
237  parsedbp("WRT", replicationmsg, wmsg);
238  }
239  if (replicationmsg.database == DBS) {
240  printf("WRT DBS\t: keysize %u\t: msgsize\t%u\t%s\n",
241  replicationmsg.ksize, replicationmsg.vsize, wmsg);
242  }
243  }
244  if (replicationmsg.operation == REPLDELETE) {
245  if (replicationmsg.database == DBDTA) {
246  printf("DEL DBDTA\t: keysize %u\t: msgsize\t%u\t%s\n",
247  replicationmsg.ksize, replicationmsg.vsize, wmsg);
248  }
249  if (replicationmsg.database == DBU) {
250  hsh = asc_hash(replicationmsg.key, replicationmsg.ksize);
251  printf("DEL DBU\t:\t%s\t\t\t\t%s\n", hsh, wmsg);
252  s_free(hsh);
253  }
254  if (replicationmsg.database == DBB) {
255  printf("DEL DBB\t: keysize %u\t: msgsize\t%u\t%s\n",
256  replicationmsg.ksize, replicationmsg.vsize, wmsg);
257  }
258  if (replicationmsg.database == DBP) {
259  parsedbp("DEL", replicationmsg, wmsg);
260  }
261  if (replicationmsg.database == DBS) {
262  printf("DEL DBS\t: keysize %u\t: msgsize\t%u\t%s\n",
263  replicationmsg.ksize, replicationmsg.vsize, wmsg);
264  }
265  }
266  if (replicationmsg.operation == TRANSACTIONCOMMIT) {
267  printf("TRC\t\t\t\t\t\t\t\t\t\t\t\t%s\n", wmsg);
268  }
269  if (replicationmsg.operation == TRANSACTIONABORT) {
270  printf("TRA\t\t\t\t\t\t\t\t\t\t\t\t%s\n", wmsg);
271  }
272  s_free(wmsg);
273  return;
274 }
275 
277 {
278  unsigned int msgsize;
279  int result;
280  char *data;
281  struct stat stbuf;
282  unsigned long offset = 0;
283  unsigned long line = 0;
284  int is_written = 0;
285 
286  if (-1 == fstat(rrepl, &stbuf))
287  die_syserr();
288  if (0 == stbuf.st_size) {
289  fprintf(stderr, "This replication logfile is empty\n");
290  return;
291  }
292  lseek(rrepl, offset, SEEK_SET);
293  while (1) {
294  line++;
295  result = fullRead(rrepl, (unsigned char *) &msgsize, sizeof(int));
296  if (sizeof(int) != result) {
297  break;
298  }
299  offset += sizeof(int);
300  if (msgsize > 2 * BLKSIZE || msgsize == 0 ) {
301  fprintf(stderr,
302  "create_report : corrupt replog file detected : msgsize %u",
303  msgsize);
304  die_dataerr("create_report : corrupt replog file detected");
305  }
306  data = s_zmalloc(msgsize);
307  result = fullRead(rrepl, (unsigned char *) data, msgsize);
308  if (result != msgsize)
309  die_syserr();
310  if (data[msgsize - 1] != '~') {
311  if (mkoptions.notwritten) {
312  offset += msgsize;
313  s_free(data);
314  lseek(rrepl, offset, SEEK_SET);
315  printf("SKIP\n");
316  continue;
317  }
318  is_written = 1;
319  }
320  is_written = 0;
321  parseline(data, msgsize - 1, is_written);
322  offset += msgsize;
323  s_free(data);
324  lseek(rrepl, offset, SEEK_SET);
325  }
326 }
327 
328 int main(int argc, char *argv[])
329 {
330  int analyze_only = 1;
331  char *p;
332 
333  debug = 1;
334  if (argc < 2)
335  usage(argv[0]);
336 
337  get_opts(argc, argv);
338  if (NULL != mkoptions.configfile) {
339  analyze_only = 0;
340  if (-1 == r_env_cfg(mkoptions.configfile))
341  usage(argv[0]);
342  }
343  if (0 == mkoptions.replogfile)
344  usage(argv[0]);
345  if (-1 == (rrepl = s_open2(mkoptions.replogfile, O_RDWR, S_IRWXU)))
346  die_syserr();
347  if (analyze_only) {
348  if (mkoptions.blocksize == 0)
349  usage(argv[0]);
350  create_report();
351  exit(0);
352  }
353  if (-1 == r_env_cfg(mkoptions.configfile))
354  usage(argv[0]);
355  p = getenv("BLKSIZE");
356  if (NULL != p) {
357  BLKSIZE = atoi(p);
358  if (BLKSIZE > 131072) {
359  LFATAL
360  ("Fuse does not support a blocksize greater then 131072 bytes.");
361  fprintf(stderr,
362  "Fuse does not support a blocksize greater then 131072 bytes.\n");
363  exit(EXIT_SYSTEM);
364  }
365  } else {
366  if (0 == mkoptions.blocksize) {
367  fprintf(stderr, "BLKSIZE not found in config nor specified\n");
368  exit(EXIT_USAGE);
369  }
370  }
371  parseconfig(0, 0);
372  merge_replog();
373  next_sequence();
374  db_close(0);
375  s_free(config);
376  printf("The replication log has been processed\n");
377 #ifdef MEMTRACE
378  leak_report();
379 #endif
380  exit(0);
381 }
REPLICATIONMSG::vsize
int vsize
Definition: lib_repl.h:8
lib_qlz.h
next_sequence
void next_sequence()
Definition: lib_common.c:1444
create_report
void create_report()
Definition: replogtool.c:276
lib_safe.h
lib_common.h
option_info::configfile
char * configfile
Definition: lessfsck.c:83
option_info::replogfile
char * replogfile
Definition: replogtool.c:79
frepl
int frepl
Definition: lib_tc.c:93
nextoffset
unsigned long long nextoffset
Definition: lib_common.c:90
parseline
void parseline(char *message, unsigned int msgsize, int is_written)
Definition: replogtool.c:197
TRANSACTIONABORT
#define TRANSACTIONABORT
Definition: lib_repl.h:19
db_close
void db_close(bool defrag)
Definition: lib_common.c:2551
REPLDELETE
#define REPLDELETE
Definition: lib_repl.h:13
DBDTA
#define DBDTA
Definition: lib_repl.h:21
passwd
char * passwd
REPLICATIONMSG
Definition: lib_repl.h:2
lib_log.h
rrepl
int rrepl
Definition: lib_repl.c:78
config
struct configdata * config
Definition: lib_cfg.h:91
fdbdta
int fdbdta
Definition: lib_tc.c:92
REPLICATIONMSG::key
unsigned char * key
Definition: lib_repl.h:5
r_env_cfg
int r_env_cfg(char *configfile)
Definition: lib_cfg.c:37
parseconfig
void parseconfig(int mklessfs, bool force_optimize)
Definition: lib_common.c:3343
REPLICATIONMSG::database
unsigned char database
Definition: lib_repl.h:3
DDSTAT::filename
char filename[256+1]
Definition: lib_common.h:67
usage
void usage(char *name)
Definition: replogtool.c:87
DDSTAT::stbuf
struct stat stbuf
Definition: lib_common.h:65
INOBNO
Definition: lib_common.h:48
TRANSACTIONCOMMIT
#define TRANSACTIONCOMMIT
Definition: lib_repl.h:18
asc_hash
char * asc_hash(unsigned char *bhash, int hashlen)
Definition: replogtool.c:152
die_dataerr
#define die_dataerr(f...)
Definition: replogtool.c:75
as_sprintf
void * as_sprintf(char *file, unsigned int line, const char *fmt,...)
Definition: lib_safe.c:564
INOBNO::blocknr
unsigned long long blocknr
Definition: lib_common.h:50
lib_str.h
INOBNO::inode
unsigned long long inode
Definition: lib_common.h:49
DBB
#define DBB
Definition: lib_repl.h:23
commons.h
merge_replog
void merge_replog()
Definition: lib_repl.c:106
lib_tc.h
main
int main(int argc, char *argv[])
Definition: replogtool.c:328
parsedbp
void parsedbp(char *action, REPLICATIONMSG replicationmsg, char *wmsg)
Definition: replogtool.c:174
s_zmalloc
#define s_zmalloc(size)
Definition: lib_safe.h:24
ascii_hash
char * ascii_hash(unsigned char *bhash)
Definition: lib_common.c:3880
freplbl
int freplbl
Definition: lib_tc.c:94
LFATAL
#define LFATAL(f...)
Definition: lib_log.h:68
mkoptions
struct option_info mkoptions
Definition: replogtool.c:83
lib_net.h
EXIT_SYSTEM
#define EXIT_SYSTEM
Definition: retcodes.h:22
option_info::blocksize
unsigned int blocksize
Definition: replogtool.c:81
debug
int debug
Definition: commons.h:7
REPLWRITE
#define REPLWRITE
Definition: lib_repl.h:12
DBP
#define DBP
Definition: lib_repl.h:24
BLKSIZE
int BLKSIZE
Definition: commons.h:8
action
action
Definition: fstest.c:56
retcodes.h
lib_repl.h
EXIT_USAGE
#define EXIT_USAGE
Definition: retcodes.h:23
option_info
Definition: lessfsck.c:82
DDSTAT
Definition: lib_common.h:64
lib_cfg.h
DBS
#define DBS
Definition: lib_repl.h:25
s_strdup
#define s_strdup(size)
Definition: lib_safe.h:26
configdata
Definition: lib_cfg.h:24
REPLICATIONMSG::value
unsigned char * value
Definition: lib_repl.h:7
REPLICATIONMSG::ksize
int ksize
Definition: lib_repl.h:6
option_info::notwritten
bool notwritten
Definition: replogtool.c:80
DBU
#define DBU
Definition: lib_repl.h:22
lib_io.h
die_syserr
#define die_syserr()
Definition: lessfsck.c:71
fullRead
int fullRead(int fd, unsigned char *buf, int len)
Definition: lib_net.c:397
s_open2
int s_open2(const char *pathname, int flags, mode_t mode)
Definition: lib_safe.c:489
lib_lzo.h
REPLICATIONMSG::operation
unsigned char operation
Definition: lib_repl.h:4
leak_report
void leak_report()
get_opts
int get_opts(int argc, char *argv[])
Definition: replogtool.c:103
working
unsigned long working
Definition: replogtool.c:85
s_free
#define s_free(mem_ref)
Definition: lib_safe.h:25