"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/tools/gfind_missing_files/gcrawler.c" (16 Sep 2020, 13761 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "gcrawler.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
    3   This file is part of GlusterFS.
    4 
    5   This file is licensed to you under your choice of the GNU Lesser
    6   General Public License, version 3 or any later version (LGPLv3 or
    7   later), or the GNU General Public License, version 2 (GPLv2), in all
    8   cases as published by the Free Software Foundation.
    9 */
   10 
   11 #include <stdio.h>
   12 #include <errno.h>
   13 #include <sys/stat.h>
   14 #include <unistd.h>
   15 #include <stdlib.h>
   16 #include <string.h>
   17 #include <dirent.h>
   18 #include <assert.h>
   19 #include <glusterfs/locking.h>
   20 
   21 #include <glusterfs/compat.h>
   22 #include <glusterfs/list.h>
   23 #include <glusterfs/syscall.h>
   24 
   25 #define THREAD_MAX 32
   26 #define BUMP(name) INC(name, 1)
   27 #define DEFAULT_WORKERS 4
   28 
   29 #define NEW(x)                                                                 \
   30     {                                                                          \
   31         x = calloc(1, sizeof(typeof(*x)));                                     \
   32     }
   33 
   34 #define err(x...) fprintf(stderr, x)
   35 #define out(x...) fprintf(stdout, x)
   36 #define dbg(x...)                                                              \
   37     do {                                                                       \
   38         if (debug)                                                             \
   39             fprintf(stdout, x);                                                \
   40     } while (0)
   41 #define tout(x...)                                                             \
   42     do {                                                                       \
   43         out("[%ld] ", pthread_self());                                         \
   44         out(x);                                                                \
   45     } while (0)
   46 #define terr(x...)                                                             \
   47     do {                                                                       \
   48         err("[%ld] ", pthread_self());                                         \
   49         err(x);                                                                \
   50     } while (0)
   51 #define tdbg(x...)                                                             \
   52     do {                                                                       \
   53         dbg("[%ld] ", pthread_self());                                         \
   54         dbg(x);                                                                \
   55     } while (0)
   56 
   57 int debug = 0;
   58 const char *slavemnt = NULL;
   59 int workers = 0;
   60 
   61 struct stats {
   62     unsigned long long int cnt_skipped_gfids;
   63 };
   64 
   65 pthread_spinlock_t stats_lock;
   66 
   67 struct stats stats_total;
   68 int stats = 0;
   69 
   70 #define INC(name, val)                                                         \
   71     do {                                                                       \
   72         if (!stats)                                                            \
   73             break;                                                             \
   74         pthread_spin_lock(&stats_lock);                                        \
   75         {                                                                      \
   76             stats_total.cnt_##name += val;                                     \
   77         }                                                                      \
   78         pthread_spin_unlock(&stats_lock);                                      \
   79     } while (0)
   80 
   81 void
   82 stats_dump()
   83 {
   84     if (!stats)
   85         return;
   86 
   87     out("-------------------------------------------\n");
   88     out("Skipped_Files : %10lld\n", stats_total.cnt_skipped_gfids);
   89     out("-------------------------------------------\n");
   90 }
   91 
   92 struct dirjob {
   93     struct list_head list;
   94 
   95     char *dirname;
   96 
   97     struct dirjob *parent;
   98     int ret;    /* final status of this subtree */
   99     int refcnt; /* how many dirjobs have this as parent */
  100 
  101     pthread_spinlock_t lock;
  102 };
  103 
  104 struct xwork {
  105     pthread_t cthreads[THREAD_MAX]; /* crawler threads */
  106     int count;
  107     int idle;
  108     int stop;
  109 
  110     struct dirjob crawl;
  111 
  112     struct dirjob *rootjob; /* to verify completion in xwork_fini() */
  113 
  114     pthread_mutex_t mutex;
  115     pthread_cond_t cond;
  116 };
  117 
  118 struct dirjob *
  119 dirjob_ref(struct dirjob *job)
  120 {
  121     pthread_spin_lock(&job->lock);
  122     {
  123         job->refcnt++;
  124     }
  125     pthread_spin_unlock(&job->lock);
  126 
  127     return job;
  128 }
  129 
  130 void
  131 dirjob_free(struct dirjob *job)
  132 {
  133     assert(list_empty(&job->list));
  134 
  135     pthread_spin_destroy(&job->lock);
  136     free(job->dirname);
  137     free(job);
  138 }
  139 
  140 void
  141 dirjob_ret(struct dirjob *job, int err)
  142 {
  143     int ret = 0;
  144     int refcnt = 0;
  145     struct dirjob *parent = NULL;
  146 
  147     pthread_spin_lock(&job->lock);
  148     {
  149         refcnt = --job->refcnt;
  150         job->ret = (job->ret || err);
  151     }
  152     pthread_spin_unlock(&job->lock);
  153 
  154     if (refcnt == 0) {
  155         ret = job->ret;
  156 
  157         if (ret)
  158             terr("Failed: %s (%d)\n", job->dirname, ret);
  159         else
  160             tdbg("Finished: %s\n", job->dirname);
  161 
  162         parent = job->parent;
  163         if (parent)
  164             dirjob_ret(parent, ret);
  165 
  166         dirjob_free(job);
  167         job = NULL;
  168     }
  169 }
  170 
  171 struct dirjob *
  172 dirjob_new(const char *dir, struct dirjob *parent)
  173 {
  174     struct dirjob *job = NULL;
  175 
  176     NEW(job);
  177     if (!job)
  178         return NULL;
  179 
  180     job->dirname = strdup(dir);
  181     if (!job->dirname) {
  182         free(job);
  183         return NULL;
  184     }
  185 
  186     INIT_LIST_HEAD(&job->list);
  187     pthread_spin_init(&job->lock, PTHREAD_PROCESS_PRIVATE);
  188     job->ret = 0;
  189 
  190     if (parent)
  191         job->parent = dirjob_ref(parent);
  192 
  193     job->refcnt = 1;
  194 
  195     return job;
  196 }
  197 
  198 void
  199 xwork_addcrawl(struct xwork *xwork, struct dirjob *job)
  200 {
  201     pthread_mutex_lock(&xwork->mutex);
  202     {
  203         list_add_tail(&job->list, &xwork->crawl.list);
  204         pthread_cond_broadcast(&xwork->cond);
  205     }
  206     pthread_mutex_unlock(&xwork->mutex);
  207 }
  208 
  209 int
  210 xwork_add(struct xwork *xwork, const char *dir, struct dirjob *parent)
  211 {
  212     struct dirjob *job = NULL;
  213 
  214     job = dirjob_new(dir, parent);
  215     if (!job)
  216         return -1;
  217 
  218     xwork_addcrawl(xwork, job);
  219 
  220     return 0;
  221 }
  222 
  223 struct dirjob *
  224 xwork_pick(struct xwork *xwork, int block)
  225 {
  226     struct dirjob *job = NULL;
  227     struct list_head *head = NULL;
  228 
  229     head = &xwork->crawl.list;
  230 
  231     pthread_mutex_lock(&xwork->mutex);
  232     {
  233         for (;;) {
  234             if (xwork->stop)
  235                 break;
  236 
  237             if (!list_empty(head)) {
  238                 job = list_entry(head->next, typeof(*job), list);
  239                 list_del_init(&job->list);
  240                 break;
  241             }
  242 
  243             if (((xwork->count * 2) == xwork->idle) &&
  244                 list_empty(&xwork->crawl.list)) {
  245                 /* no outstanding jobs, and no
  246                    active workers
  247                 */
  248                 tdbg("Jobless. Terminating\n");
  249                 xwork->stop = 1;
  250                 pthread_cond_broadcast(&xwork->cond);
  251                 break;
  252             }
  253 
  254             if (!block)
  255                 break;
  256 
  257             xwork->idle++;
  258             pthread_cond_wait(&xwork->cond, &xwork->mutex);
  259             xwork->idle--;
  260         }
  261     }
  262     pthread_mutex_unlock(&xwork->mutex);
  263 
  264     return job;
  265 }
  266 
  267 int
  268 skip_name(const char *dirname, const char *name)
  269 {
  270     if (strcmp(name, ".") == 0)
  271         return 1;
  272 
  273     if (strcmp(name, "..") == 0)
  274         return 1;
  275 
  276     if (strcmp(name, "changelogs") == 0)
  277         return 1;
  278 
  279     if (strcmp(name, "health_check") == 0)
  280         return 1;
  281 
  282     if (strcmp(name, "indices") == 0)
  283         return 1;
  284 
  285     if (strcmp(name, "landfill") == 0)
  286         return 1;
  287 
  288     return 0;
  289 }
  290 
  291 int
  292 skip_stat(struct dirjob *job, const char *name)
  293 {
  294     if (job == NULL)
  295         return 0;
  296 
  297     if (strcmp(job->dirname, ".glusterfs") == 0) {
  298         tdbg(
  299             "Directly adding directories under .glusterfs "
  300             "to global list: %s\n",
  301             name);
  302         return 1;
  303     }
  304 
  305     if (job->parent != NULL) {
  306         if (strcmp(job->parent->dirname, ".glusterfs") == 0) {
  307             tdbg(
  308                 "Directly adding directories under .glusterfs/XX "
  309                 "to global list: %s\n",
  310                 name);
  311             return 1;
  312         }
  313     }
  314 
  315     return 0;
  316 }
  317 
  318 int
  319 xworker_do_crawl(struct xwork *xwork, struct dirjob *job)
  320 {
  321     DIR *dirp = NULL;
  322     int ret = -1;
  323     int boff;
  324     int plen;
  325     char *path = NULL;
  326     struct dirjob *cjob = NULL;
  327     struct stat statbuf = {
  328         0,
  329     };
  330     struct dirent *entry;
  331     struct dirent scratch[2] = {
  332         {
  333             0,
  334         },
  335     };
  336     char gfid_path[PATH_MAX] = {
  337         0,
  338     };
  339 
  340     plen = strlen(job->dirname) + 256 + 2;
  341     path = alloca(plen);
  342 
  343     tdbg("Entering: %s\n", job->dirname);
  344 
  345     dirp = sys_opendir(job->dirname);
  346     if (!dirp) {
  347         terr("opendir failed on %s (%s)\n", job->dirname, strerror(errno));
  348         goto out;
  349     }
  350 
  351     boff = sprintf(path, "%s/", job->dirname);
  352 
  353     for (;;) {
  354         errno = 0;
  355         entry = sys_readdir(dirp, scratch);
  356         if (!entry || errno != 0) {
  357             if (errno != 0) {
  358                 err("readdir(%s): %s\n", job->dirname, strerror(errno));
  359                 ret = errno;
  360                 goto out;
  361             }
  362             break;
  363         }
  364 
  365         if (entry->d_ino == 0)
  366             continue;
  367 
  368         if (skip_name(job->dirname, entry->d_name))
  369             continue;
  370 
  371         /* It is sure that, children and grandchildren of .glusterfs
  372          * are directories, just add them to global queue.
  373          */
  374         if (skip_stat(job, entry->d_name)) {
  375             strncpy(path + boff, entry->d_name, (plen - boff));
  376             cjob = dirjob_new(path, job);
  377             if (!cjob) {
  378                 err("dirjob_new(%s): %s\n", path, strerror(errno));
  379                 ret = -1;
  380                 goto out;
  381             }
  382             xwork_addcrawl(xwork, cjob);
  383             continue;
  384         }
  385 
  386         (void)snprintf(gfid_path, sizeof(gfid_path), "%s/.gfid/%s", slavemnt,
  387                        entry->d_name);
  388         ret = sys_lstat(gfid_path, &statbuf);
  389 
  390         if (ret && errno == ENOENT) {
  391             out("%s\n", entry->d_name);
  392             BUMP(skipped_gfids);
  393         }
  394 
  395         if (ret && errno != ENOENT) {
  396             err("stat on slave failed(%s): %s\n", gfid_path, strerror(errno));
  397             goto out;
  398         }
  399     }
  400 
  401     ret = 0;
  402 out:
  403     if (dirp)
  404         (void)sys_closedir(dirp);
  405 
  406     return ret;
  407 }
  408 
  409 void *
  410 xworker_crawl(void *data)
  411 {
  412     struct xwork *xwork = data;
  413     struct dirjob *job = NULL;
  414     int ret = -1;
  415 
  416     while ((job = xwork_pick(xwork, 0))) {
  417         ret = xworker_do_crawl(xwork, job);
  418         dirjob_ret(job, ret);
  419     }
  420 
  421     return NULL;
  422 }
  423 
  424 int
  425 xwork_fini(struct xwork *xwork, int stop)
  426 {
  427     int i = 0;
  428     int ret = 0;
  429     void *tret = 0;
  430 
  431     pthread_mutex_lock(&xwork->mutex);
  432     {
  433         xwork->stop = (xwork->stop || stop);
  434         pthread_cond_broadcast(&xwork->cond);
  435     }
  436     pthread_mutex_unlock(&xwork->mutex);
  437 
  438     for (i = 0; i < xwork->count; i++) {
  439         pthread_join(xwork->cthreads[i], &tret);
  440         tdbg("CThread id %ld returned %p\n", xwork->cthreads[i], tret);
  441     }
  442 
  443     if (debug) {
  444         assert(xwork->rootjob->refcnt == 1);
  445         dirjob_ret(xwork->rootjob, 0);
  446     }
  447 
  448     if (stats)
  449         pthread_spin_destroy(&stats_lock);
  450 
  451     return ret;
  452 }
  453 
  454 int
  455 xwork_init(struct xwork *xwork, int count)
  456 {
  457     int i = 0;
  458     int ret = 0;
  459     struct dirjob *rootjob = NULL;
  460 
  461     if (stats)
  462         pthread_spin_init(&stats_lock, PTHREAD_PROCESS_PRIVATE);
  463 
  464     pthread_mutex_init(&xwork->mutex, NULL);
  465     pthread_cond_init(&xwork->cond, NULL);
  466 
  467     INIT_LIST_HEAD(&xwork->crawl.list);
  468 
  469     rootjob = dirjob_new(".glusterfs", NULL);
  470     if (debug)
  471         xwork->rootjob = dirjob_ref(rootjob);
  472 
  473     xwork_addcrawl(xwork, rootjob);
  474 
  475     xwork->count = count;
  476     for (i = 0; i < count; i++) {
  477         ret = pthread_create(&xwork->cthreads[i], NULL, xworker_crawl, xwork);
  478         if (ret)
  479             break;
  480         tdbg("Spawned crawler %d thread %ld\n", i, xwork->cthreads[i]);
  481     }
  482 
  483     return ret;
  484 }
  485 
  486 int
  487 xfind(const char *basedir)
  488 {
  489     struct xwork xwork;
  490     int ret = 0;
  491     char *cwd = NULL;
  492 
  493     ret = chdir(basedir);
  494     if (ret) {
  495         err("%s: %s\n", basedir, strerror(errno));
  496         return ret;
  497     }
  498 
  499     cwd = getcwd(0, 0);
  500     if (!cwd) {
  501         err("getcwd(): %s\n", strerror(errno));
  502         return -1;
  503     }
  504 
  505     tdbg("Working directory: %s\n", cwd);
  506     free(cwd);
  507 
  508     memset(&xwork, 0, sizeof(xwork));
  509 
  510     ret = xwork_init(&xwork, workers);
  511     if (ret == 0)
  512         xworker_crawl(&xwork);
  513 
  514     ret = xwork_fini(&xwork, ret);
  515     stats_dump();
  516 
  517     return ret;
  518 }
  519 
  520 static char *
  521 parse_and_validate_args(int argc, char *argv[])
  522 {
  523     char *basedir = NULL;
  524     struct stat d = {
  525         0,
  526     };
  527     int ret = -1;
  528 #ifndef __FreeBSD__
  529     unsigned char volume_id[16];
  530 #endif /* __FreeBSD__ */
  531     char *slv_mnt = NULL;
  532 
  533     if (argc != 4) {
  534         err("Usage: %s <DIR> <SLAVE-VOL-MOUNT> <CRAWL-THREAD-COUNT>\n",
  535             argv[0]);
  536         return NULL;
  537     }
  538 
  539     basedir = argv[1];
  540     ret = sys_lstat(basedir, &d);
  541     if (ret) {
  542         err("%s: %s\n", basedir, strerror(errno));
  543         return NULL;
  544     }
  545 
  546 #ifndef __FreeBSD__
  547     ret = sys_lgetxattr(basedir, "trusted.glusterfs.volume-id", volume_id, 16);
  548     if (ret != 16) {
  549         err("%s:Not a valid brick path.\n", basedir);
  550         return NULL;
  551     }
  552 #endif /* __FreeBSD__ */
  553 
  554     slv_mnt = argv[2];
  555     ret = sys_lstat(slv_mnt, &d);
  556     if (ret) {
  557         err("%s: %s\n", slv_mnt, strerror(errno));
  558         return NULL;
  559     }
  560     slavemnt = argv[2];
  561 
  562     workers = atoi(argv[3]);
  563     if (workers <= 0)
  564         workers = DEFAULT_WORKERS;
  565 
  566     return basedir;
  567 }
  568 
  569 int
  570 main(int argc, char *argv[])
  571 {
  572     char *basedir = NULL;
  573 
  574     basedir = parse_and_validate_args(argc, argv);
  575     if (!basedir)
  576         return 1;
  577 
  578     xfind(basedir);
  579 
  580     return 0;
  581 }