"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-6.9/xlators/cluster/afr/src/afr-self-heald.c" (23 Apr 2020, 39107 Bytes) of package /linux/misc/glusterfs-6.9.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "afr-self-heald.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
    3   This file is part of GlusterFS.
    4 
    5   This file is licensed to you under your choice of the GNU Lesser
    6   General Public License, version 3 or any later version (LGPLv3 or
    7   later), or the GNU General Public License, version 2 (GPLv2), in all
    8   cases as published by the Free Software Foundation.
    9 */
   10 
   11 #include "afr.h"
   12 #include "afr-self-heal.h"
   13 #include "afr-self-heald.h"
   14 #include "protocol-common.h"
   15 #include <glusterfs/syncop-utils.h>
   16 #include "afr-messages.h"
   17 #include <glusterfs/byte-order.h>
   18 
   19 #define AFR_EH_SPLIT_BRAIN_LIMIT 1024
   20 #define AFR_STATISTICS_HISTORY_SIZE 50
   21 
   22 #define ASSERT_LOCAL(this, healer)                                             \
   23     if (!afr_shd_is_subvol_local(this, healer->subvol)) {                      \
   24         healer->local = _gf_false;                                             \
   25         if (safe_break(healer)) {                                              \
   26             break;                                                             \
   27         } else {                                                               \
   28             continue;                                                          \
   29         }                                                                      \
   30     } else {                                                                   \
   31         healer->local = _gf_true;                                              \
   32     }
   33 
   34 #define NTH_INDEX_HEALER(this, n)                                              \
   35     &((((afr_private_t *)this->private))->shd.index_healers[n])
   36 #define NTH_FULL_HEALER(this, n)                                               \
   37     &((((afr_private_t *)this->private))->shd.full_healers[n])
   38 
   39 char *
   40 afr_subvol_name(xlator_t *this, int subvol)
   41 {
   42     afr_private_t *priv = NULL;
   43 
   44     priv = this->private;
   45     if (subvol < 0 || subvol > priv->child_count)
   46         return NULL;
   47 
   48     return priv->children[subvol]->name;
   49 }
   50 
   51 void
   52 afr_destroy_crawl_event_data(void *data)
   53 {
   54     return;
   55 }
   56 
   57 void
   58 afr_destroy_shd_event_data(void *data)
   59 {
   60     shd_event_t *shd_event = data;
   61 
   62     if (!shd_event)
   63         return;
   64     GF_FREE(shd_event->path);
   65 
   66     return;
   67 }
   68 
   69 gf_boolean_t
   70 afr_shd_is_subvol_local(xlator_t *this, int subvol)
   71 {
   72     afr_private_t *priv = NULL;
   73     gf_boolean_t is_local = _gf_false;
   74     loc_t loc = {
   75         0,
   76     };
   77 
   78     loc.inode = this->itable->root;
   79     gf_uuid_copy(loc.gfid, loc.inode->gfid);
   80     priv = this->private;
   81     syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
   82     return is_local;
   83 }
   84 
   85 int
   86 __afr_shd_healer_wait(struct subvol_healer *healer)
   87 {
   88     afr_private_t *priv = NULL;
   89     struct timespec wait_till = {
   90         0,
   91     };
   92     int ret = 0;
   93 
   94     priv = healer->this->private;
   95 
   96 disabled_loop:
   97     wait_till.tv_sec = time(NULL) + priv->shd.timeout;
   98 
   99     while (!healer->rerun) {
  100         ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
  101         if (ret == ETIMEDOUT)
  102             break;
  103     }
  104 
  105     ret = healer->rerun;
  106     healer->rerun = 0;
  107 
  108     if (!priv->shd.enabled)
  109         goto disabled_loop;
  110 
  111     return ret;
  112 }
  113 
  114 int
  115 afr_shd_healer_wait(struct subvol_healer *healer)
  116 {
  117     int ret = 0;
  118 
  119     pthread_mutex_lock(&healer->mutex);
  120     {
  121         ret = __afr_shd_healer_wait(healer);
  122     }
  123     pthread_mutex_unlock(&healer->mutex);
  124 
  125     return ret;
  126 }
  127 
  128 gf_boolean_t
  129 safe_break(struct subvol_healer *healer)
  130 {
  131     gf_boolean_t ret = _gf_false;
  132 
  133     pthread_mutex_lock(&healer->mutex);
  134     {
  135         if (healer->rerun)
  136             goto unlock;
  137 
  138         healer->running = _gf_false;
  139         ret = _gf_true;
  140     }
  141 unlock:
  142     pthread_mutex_unlock(&healer->mutex);
  143 
  144     return ret;
  145 }
  146 
  147 inode_t *
  148 afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
  149 {
  150     int ret = 0;
  151     uint64_t val = IA_INVAL;
  152     dict_t *xdata = NULL;
  153     dict_t *rsp_dict = NULL;
  154     inode_t *inode = NULL;
  155 
  156     xdata = dict_new();
  157     if (!xdata)
  158         goto out;
  159 
  160     ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
  161     if (ret)
  162         goto out;
  163 
  164     ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
  165     if (ret < 0)
  166         goto out;
  167 
  168     if (rsp_dict) {
  169         ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
  170         if (ret)
  171             goto out;
  172     }
  173     ret = inode_ctx_set2(inode, subvol, 0, &val);
  174 out:
  175     if (ret && inode) {
  176         inode_unref(inode);
  177         inode = NULL;
  178     }
  179     if (xdata)
  180         dict_unref(xdata);
  181     if (rsp_dict)
  182         dict_unref(rsp_dict);
  183     return inode;
  184 }
  185 
  186 inode_t *
  187 afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
  188 {
  189     loc_t rootloc = {
  190         0,
  191     };
  192     inode_t *inode = NULL;
  193     int ret = 0;
  194     dict_t *xattr = NULL;
  195     void *index_gfid = NULL;
  196 
  197     rootloc.inode = inode_ref(this->itable->root);
  198     gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
  199 
  200     ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
  201     if (ret || !xattr) {
  202         errno = -ret;
  203         goto out;
  204     }
  205 
  206     ret = dict_get_ptr(xattr, vgfid, &index_gfid);
  207     if (ret)
  208         goto out;
  209 
  210     gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
  211                  uuid_utoa(index_gfid));
  212 
  213     inode = afr_shd_inode_find(this, subvol, index_gfid);
  214 
  215 out:
  216     loc_wipe(&rootloc);
  217 
  218     if (xattr)
  219         dict_unref(xattr);
  220 
  221     return inode;
  222 }
  223 
  224 int
  225 afr_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name,
  226                     ia_type_t type)
  227 {
  228     int ret = 0;
  229     loc_t loc = {
  230         0,
  231     };
  232 
  233     loc.parent = inode_ref(inode);
  234     loc.name = name;
  235 
  236     if (IA_ISDIR(type))
  237         ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
  238     else
  239         ret = syncop_unlink(subvol, &loc, NULL, NULL);
  240 
  241     loc_wipe(&loc);
  242     return ret;
  243 }
  244 
  245 void
  246 afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
  247 {
  248     call_frame_t *frame = NULL;
  249     inode_t *inode = NULL;
  250     afr_private_t *priv = NULL;
  251     dict_t *xattr = NULL;
  252     int ret = 0;
  253     int i = 0;
  254     int raw[AFR_NUM_CHANGE_LOGS] = {0};
  255 
  256     priv = this->private;
  257     frame = afr_frame_create(this, NULL);
  258     if (!frame)
  259         goto out;
  260     inode = afr_inode_find(this, gfid);
  261     if (!inode)
  262         goto out;
  263     xattr = dict_new();
  264     if (!xattr)
  265         goto out;
  266     ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
  267                               sizeof(int) * AFR_NUM_CHANGE_LOGS);
  268     if (ret)
  269         goto out;
  270     for (i = 0; i < priv->child_count; i++) {
  271         ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
  272                                   sizeof(int) * AFR_NUM_CHANGE_LOGS);
  273         if (ret)
  274             goto out;
  275     }
  276 
  277     /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
  278      * has valid repies for this gfid seems a bit of an overkill.*/
  279     for (i = 0; i < priv->child_count; i++)
  280         afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
  281 
  282 out:
  283     if (frame)
  284         AFR_STACK_DESTROY(frame);
  285     if (inode)
  286         inode_unref(inode);
  287     if (xattr)
  288         dict_unref(xattr);
  289     return;
  290 }
  291 
  292 int
  293 afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
  294                       const char *bname)
  295 {
  296     int ret = -1;
  297 
  298     ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL);
  299 
  300     return ret;
  301 }
  302 
  303 int
  304 afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
  305 {
  306     int ret = 0;
  307     eh_t *eh = NULL;
  308     afr_private_t *priv = NULL;
  309     afr_self_heald_t *shd = NULL;
  310     shd_event_t *shd_event = NULL;
  311     char *path = NULL;
  312     xlator_t *subvol = NULL;
  313     xlator_t *this = NULL;
  314     crawl_event_t *crawl_event = NULL;
  315 
  316     this = healer->this;
  317     priv = this->private;
  318     shd = &priv->shd;
  319     crawl_event = &healer->crawl_event;
  320 
  321     subvol = priv->children[child];
  322 
  323     // If this fails with ENOENT/ESTALE index is stale
  324     ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
  325     if (ret < 0)
  326         return ret;
  327 
  328     ret = afr_selfheal(this, gfid);
  329 
  330     LOCK(&priv->lock);
  331     {
  332         if (ret == -EIO) {
  333             eh = shd->split_brain;
  334             crawl_event->split_brain_count++;
  335         } else if (ret < 0) {
  336             crawl_event->heal_failed_count++;
  337         } else if (ret == 0) {
  338             crawl_event->healed_count++;
  339         }
  340     }
  341     UNLOCK(&priv->lock);
  342 
  343     if (eh) {
  344         shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
  345         if (!shd_event)
  346             goto out;
  347 
  348         shd_event->child = child;
  349         shd_event->path = path;
  350 
  351         if (eh_save_history(eh, shd_event) < 0)
  352             goto out;
  353 
  354         shd_event = NULL;
  355         path = NULL;
  356     }
  357 out:
  358     GF_FREE(shd_event);
  359     GF_FREE(path);
  360     return ret;
  361 }
  362 
  363 void
  364 afr_shd_sweep_prepare(struct subvol_healer *healer)
  365 {
  366     crawl_event_t *event = NULL;
  367 
  368     event = &healer->crawl_event;
  369 
  370     event->healed_count = 0;
  371     event->split_brain_count = 0;
  372     event->heal_failed_count = 0;
  373 
  374     time(&event->start_time);
  375     event->end_time = 0;
  376 }
  377 
  378 void
  379 afr_shd_sweep_done(struct subvol_healer *healer)
  380 {
  381     crawl_event_t *event = NULL;
  382     crawl_event_t *history = NULL;
  383     afr_self_heald_t *shd = NULL;
  384 
  385     event = &healer->crawl_event;
  386     shd = &(((afr_private_t *)healer->this->private)->shd);
  387 
  388     time(&event->end_time);
  389     history = memdup(event, sizeof(*event));
  390     event->start_time = 0;
  391 
  392     if (!history)
  393         return;
  394 
  395     if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
  396         GF_FREE(history);
  397 }
  398 
  399 int
  400 afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
  401                    void *data)
  402 {
  403     struct subvol_healer *healer = data;
  404     afr_private_t *priv = NULL;
  405     uuid_t gfid = {0};
  406     int ret = 0;
  407     uint64_t val = IA_INVAL;
  408 
  409     priv = healer->this->private;
  410     if (!priv->shd.enabled)
  411         return -EBUSY;
  412 
  413     gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
  414                  priv->children[healer->subvol]->name);
  415 
  416     ret = gf_uuid_parse(entry->d_name, gfid);
  417     if (ret)
  418         return 0;
  419 
  420     inode_ctx_get2(parent->inode, subvol, NULL, &val);
  421 
  422     ret = afr_shd_selfheal(healer, healer->subvol, gfid);
  423 
  424     if (ret == -ENOENT || ret == -ESTALE)
  425         afr_shd_index_purge(subvol, parent->inode, entry->d_name, val);
  426 
  427     if (ret == 2)
  428         /* If bricks crashed in pre-op after creating indices/xattrop
  429          * link but before setting afr changelogs, we end up with stale
  430          * xattrop links but zero changelogs. Remove such entries by
  431          * sending a post-op with zero changelogs.
  432          */
  433         afr_shd_zero_xattrop(healer->this, gfid);
  434 
  435     return 0;
  436 }
  437 
  438 int
  439 afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
  440 {
  441     loc_t loc = {0};
  442     afr_private_t *priv = NULL;
  443     int ret = 0;
  444     xlator_t *subvol = NULL;
  445     dict_t *xdata = NULL;
  446     call_frame_t *frame = NULL;
  447 
  448     priv = healer->this->private;
  449     subvol = priv->children[healer->subvol];
  450 
  451     frame = afr_frame_create(healer->this, &ret);
  452     if (!frame) {
  453         ret = -ret;
  454         goto out;
  455     }
  456 
  457     loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
  458     if (!loc.inode) {
  459         gf_msg(healer->this->name, GF_LOG_WARNING, 0,
  460                AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
  461                subvol->name);
  462         ret = -errno;
  463         goto out;
  464     }
  465 
  466     xdata = dict_new();
  467     if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
  468         ret = -ENOMEM;
  469         goto out;
  470     }
  471 
  472     ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
  473                              healer, afr_shd_index_heal, xdata,
  474                              priv->shd.max_threads, priv->shd.wait_qlength);
  475 
  476     if (ret == 0)
  477         ret = healer->crawl_event.healed_count;
  478 
  479 out:
  480     loc_wipe(&loc);
  481 
  482     if (xdata)
  483         dict_unref(xdata);
  484     if (frame)
  485         AFR_STACK_DESTROY(frame);
  486     return ret;
  487 }
  488 
  489 int
  490 afr_shd_index_sweep_all(struct subvol_healer *healer)
  491 {
  492     int ret = 0;
  493     int count = 0;
  494 
  495     ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
  496     if (ret < 0)
  497         goto out;
  498     count = ret;
  499 
  500     ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
  501     if (ret < 0)
  502         goto out;
  503     count += ret;
  504 
  505     ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
  506     if (ret < 0)
  507         goto out;
  508     count += ret;
  509 out:
  510     if (ret < 0)
  511         return ret;
  512     else
  513         return count;
  514 }
  515 
  516 int
  517 afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
  518                   void *data)
  519 {
  520     struct subvol_healer *healer = data;
  521     xlator_t *this = healer->this;
  522     afr_private_t *priv = NULL;
  523 
  524     priv = this->private;
  525     if (!priv->shd.enabled)
  526         return -EBUSY;
  527 
  528     afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
  529                           entry->d_name);
  530 
  531     afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
  532 
  533     return 0;
  534 }
  535 
  536 int
  537 afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
  538 {
  539     afr_private_t *priv = NULL;
  540     loc_t loc = {0};
  541 
  542     priv = healer->this->private;
  543     loc.inode = inode;
  544     return syncop_ftw(priv->children[healer->subvol], &loc,
  545                       GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
  546 }
  547 
  548 int
  549 afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
  550 {
  551     afr_private_t *priv = NULL;
  552     struct iatt stbuf = {
  553         0,
  554     };
  555     int ret = -1;
  556 
  557     priv = this->private;
  558     loc->parent = inode_ref(this->itable->root);
  559     gf_uuid_copy(loc->pargfid, loc->parent->gfid);
  560     loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
  561     loc->inode = inode_new(loc->parent->table);
  562     GF_CHECK_ALLOC(loc->inode, ret, out);
  563 
  564     if (!gf_uuid_is_null(priv->ta_gfid))
  565         goto assign_gfid;
  566 
  567     ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
  568                         0, 0, 0);
  569     if (ret) {
  570         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
  571                "Failed lookup on file %s.", loc->name);
  572         goto out;
  573     }
  574 
  575     gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
  576 
  577 assign_gfid:
  578     gf_uuid_copy(loc->gfid, priv->ta_gfid);
  579     ret = 0;
  580 
  581 out:
  582     if (ret)
  583         loc_wipe(loc);
  584 
  585     return ret;
  586 }
  587 
  588 int
  589 _afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
  590 {
  591     afr_private_t *priv = NULL;
  592     dict_t *xattr = NULL;
  593     int raw[AFR_NUM_CHANGE_LOGS] = {
  594         0,
  595     };
  596     int ret = -1;
  597     int i = 0;
  598 
  599     priv = this->private;
  600 
  601     xattr = dict_new();
  602     if (!xattr) {
  603         gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
  604                "Failed to create dict.");
  605         goto out;
  606     }
  607     for (i = 0; i < priv->child_count; i++) {
  608         ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
  609                                   AFR_NUM_CHANGE_LOGS * sizeof(int));
  610         if (ret)
  611             goto out;
  612     }
  613 
  614     ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
  615                          GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
  616     if (ret || !(*xdata)) {
  617         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
  618                "Xattrop failed on %s.", loc->name);
  619     }
  620 
  621 out:
  622     if (xattr)
  623         dict_unref(xattr);
  624 
  625     return ret;
  626 }
  627 
  628 void
  629 afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
  630                       dict_t **xdata)
  631 {
  632     int ret = 0;
  633 
  634     loc_wipe(loc);
  635     if (afr_shd_fill_ta_loc(this, loc)) {
  636         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
  637                "Failed to populate thin-arbiter loc for: %s.", loc->name);
  638         ret = -1;
  639         goto out;
  640     }
  641 
  642     ret = afr_ta_post_op_lock(this, loc);
  643     if (ret)
  644         goto out;
  645 
  646     ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
  647     if (ret) {
  648         if (*xdata) {
  649             dict_unref(*xdata);
  650             *xdata = NULL;
  651         }
  652     }
  653 
  654     afr_ta_post_op_unlock(this, loc);
  655 
  656 out:
  657     if (ret)
  658         healer->rerun = 1;
  659 }
  660 
  661 int
  662 afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
  663 {
  664     afr_private_t *priv = NULL;
  665     dict_t *xattr = NULL;
  666     gf_boolean_t need_xattrop = _gf_false;
  667     void *pending_raw = NULL;
  668     int *raw = NULL;
  669     int pending[AFR_NUM_CHANGE_LOGS] = {
  670         0,
  671     };
  672     int i = 0;
  673     int j = 0;
  674     int val = 0;
  675     int ret = -1;
  676 
  677     priv = this->private;
  678 
  679     xattr = dict_new();
  680     if (!xattr) {
  681         goto out;
  682     }
  683 
  684     for (i = 0; i < priv->child_count; i++) {
  685         raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
  686         if (!raw) {
  687             goto out;
  688         }
  689 
  690         ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
  691         if (ret) {
  692             gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
  693                    "Error getting value "
  694                    "of pending key %s",
  695                    priv->pending_key[i]);
  696             GF_FREE(raw);
  697             goto out;
  698         }
  699 
  700         memcpy(pending, pending_raw, sizeof(pending));
  701         for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
  702             val = ntoh32(pending[j]);
  703             if (val) {
  704                 if (i == healer) {
  705                     gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
  706                            "I am "
  707                            "not the good shd. Skipping. "
  708                            "SHD = %d.",
  709                            healer);
  710                     ret = 0;
  711                     GF_FREE(raw);
  712                     goto out;
  713                 }
  714                 need_xattrop = _gf_true;
  715                 raw[j] = hton32(-val);
  716             }
  717         }
  718 
  719         ret = dict_set_bin(xattr, priv->pending_key[i], raw,
  720                            AFR_NUM_CHANGE_LOGS * sizeof(int));
  721         if (ret) {
  722             GF_FREE(raw);
  723             goto out;
  724         }
  725 
  726         if (need_xattrop)
  727             break;
  728     }
  729 
  730     if (!need_xattrop) {
  731         ret = 0;
  732         goto out;
  733     }
  734 
  735     ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
  736                          GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
  737     if (ret)
  738         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
  739                "Xattrop failed.");
  740 
  741 out:
  742     if (xattr)
  743         dict_unref(xattr);
  744 
  745     return ret;
  746 }
  747 
  748 void
  749 afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
  750                                   struct subvol_healer *healer,
  751                                   dict_t *pre_crawl_xdata)
  752 {
  753     int ret_lock = 0;
  754     int ret = 0;
  755     dict_t *post_crawl_xdata = NULL;
  756 
  757     ret_lock = afr_ta_post_op_lock(this, loc);
  758     if (ret_lock)
  759         goto unref;
  760 
  761     ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
  762     if (ret)
  763         goto unref;
  764 
  765     if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL)) {
  766         ret = -1;
  767         goto unref;
  768     }
  769 
  770     ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
  771 
  772 unref:
  773     if (post_crawl_xdata) {
  774         dict_unref(post_crawl_xdata);
  775         post_crawl_xdata = NULL;
  776     }
  777 
  778     if (ret || ret_lock)
  779         healer->rerun = 1;
  780 
  781     if (!ret_lock)
  782         afr_ta_post_op_unlock(this, loc);
  783 }
  784 
  785 gf_boolean_t
  786 afr_bricks_available_for_heal(afr_private_t *priv)
  787 {
  788     int up_children = 0;
  789 
  790     up_children = __afr_get_up_children_count(priv);
  791     if (up_children < 2) {
  792         return _gf_false;
  793     }
  794     return _gf_true;
  795 }
  796 
  797 void *
  798 afr_shd_index_healer(void *data)
  799 {
  800     struct subvol_healer *healer = NULL;
  801     xlator_t *this = NULL;
  802     int ret = 0;
  803     afr_private_t *priv = NULL;
  804     dict_t *pre_crawl_xdata = NULL;
  805     loc_t loc = {
  806         0,
  807     };
  808 
  809     healer = data;
  810     THIS = this = healer->this;
  811     priv = this->private;
  812 
  813     for (;;) {
  814         afr_shd_healer_wait(healer);
  815 
  816         if (!afr_bricks_available_for_heal(priv))
  817             continue;
  818 
  819         ASSERT_LOCAL(this, healer);
  820         priv->local[healer->subvol] = healer->local;
  821 
  822         if (priv->thin_arbiter_count) {
  823             afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
  824         }
  825 
  826         do {
  827             gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
  828                          afr_subvol_name(this, healer->subvol));
  829 
  830             afr_shd_sweep_prepare(healer);
  831 
  832             ret = afr_shd_index_sweep_all(healer);
  833 
  834             afr_shd_sweep_done(healer);
  835             /*
  836               As long as at least one gfid was
  837               healed, keep retrying. We may have
  838               just healed a directory and thereby
  839               created entries for other gfids which
  840               could not be healed thus far.
  841             */
  842 
  843             gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
  844                          afr_subvol_name(this, healer->subvol));
  845             /*
  846               Give a pause before retrying to avoid a busy loop
  847               in case the only entry in index is because of
  848               an ongoing I/O.
  849             */
  850             sleep(1);
  851         } while (ret > 0);
  852 
  853         if (ret == 0 && pre_crawl_xdata &&
  854             !healer->crawl_event.heal_failed_count) {
  855             afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
  856                                               pre_crawl_xdata);
  857         }
  858 
  859         if (pre_crawl_xdata) {
  860             dict_unref(pre_crawl_xdata);
  861             pre_crawl_xdata = NULL;
  862         }
  863     }
  864 
  865     return NULL;
  866 }
  867 
  868 void *
  869 afr_shd_full_healer(void *data)
  870 {
  871     struct subvol_healer *healer = NULL;
  872     xlator_t *this = NULL;
  873     int run = 0;
  874 
  875     healer = data;
  876     THIS = this = healer->this;
  877 
  878     for (;;) {
  879         pthread_mutex_lock(&healer->mutex);
  880         {
  881             run = __afr_shd_healer_wait(healer);
  882             if (!run)
  883                 healer->running = _gf_false;
  884         }
  885         pthread_mutex_unlock(&healer->mutex);
  886 
  887         if (!run)
  888             break;
  889 
  890         ASSERT_LOCAL(this, healer);
  891 
  892         gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
  893                "starting full sweep on subvol %s",
  894                afr_subvol_name(this, healer->subvol));
  895 
  896         afr_shd_sweep_prepare(healer);
  897 
  898         afr_shd_full_sweep(healer, this->itable->root);
  899 
  900         afr_shd_sweep_done(healer);
  901 
  902         gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
  903                "finished full sweep on subvol %s",
  904                afr_subvol_name(this, healer->subvol));
  905     }
  906 
  907     return NULL;
  908 }
  909 
  910 int
  911 afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
  912 {
  913     int ret = 0;
  914 
  915     ret = pthread_mutex_init(&healer->mutex, NULL);
  916     if (ret)
  917         goto out;
  918 
  919     ret = pthread_cond_init(&healer->cond, NULL);
  920     if (ret)
  921         goto out;
  922 
  923     healer->this = this;
  924     healer->running = _gf_false;
  925     healer->rerun = _gf_false;
  926     healer->local = _gf_false;
  927 out:
  928     return ret;
  929 }
  930 
  931 int
  932 afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
  933                      void *(threadfn)(void *))
  934 {
  935     int ret = 0;
  936 
  937     pthread_mutex_lock(&healer->mutex);
  938     {
  939         if (healer->running) {
  940             pthread_cond_signal(&healer->cond);
  941         } else {
  942             ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
  943                                    "shdheal");
  944             if (ret)
  945                 goto unlock;
  946             healer->running = 1;
  947         }
  948 
  949         healer->rerun = 1;
  950     }
  951 unlock:
  952     pthread_mutex_unlock(&healer->mutex);
  953 
  954     return ret;
  955 }
  956 
  957 int
  958 afr_shd_full_healer_spawn(xlator_t *this, int subvol)
  959 {
  960     return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
  961                                 afr_shd_full_healer);
  962 }
  963 
  964 int
  965 afr_shd_index_healer_spawn(xlator_t *this, int subvol)
  966 {
  967     return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
  968                                 afr_shd_index_healer);
  969 }
  970 
  971 int
  972 afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
  973                              crawl_event_t *crawl_event)
  974 {
  975     int ret = 0;
  976     uint64_t count = 0;
  977     char key[256] = {0};
  978     int keylen = 0;
  979     char suffix[64] = {0};
  980     int xl_id = 0;
  981     uint64_t healed_count = 0;
  982     uint64_t split_brain_count = 0;
  983     uint64_t heal_failed_count = 0;
  984     char *start_time_str = 0;
  985     char *end_time_str = NULL;
  986     char *crawl_type = NULL;
  987     int progress = -1;
  988     int child = -1;
  989 
  990     child = crawl_event->child;
  991     healed_count = crawl_event->healed_count;
  992     split_brain_count = crawl_event->split_brain_count;
  993     heal_failed_count = crawl_event->heal_failed_count;
  994     crawl_type = crawl_event->crawl_type;
  995 
  996     if (!crawl_event->start_time)
  997         goto out;
  998 
  999     start_time_str = gf_strdup(ctime(&crawl_event->start_time));
 1000 
 1001     if (crawl_event->end_time)
 1002         end_time_str = gf_strdup(ctime(&crawl_event->end_time));
 1003 
 1004     ret = dict_get_int32(output, this->name, &xl_id);
 1005     if (ret) {
 1006         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
 1007                "xl does not have id");
 1008         goto out;
 1009     }
 1010 
 1011     snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
 1012     ret = dict_get_uint64(output, key, &count);
 1013 
 1014     snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
 1015     snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
 1016     ret = dict_set_uint64(output, key, healed_count);
 1017     if (ret) {
 1018         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1019                "Could not add statistics_healed_count to output");
 1020         goto out;
 1021     }
 1022 
 1023     snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
 1024     ret = dict_set_uint64(output, key, split_brain_count);
 1025     if (ret) {
 1026         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1027                "Could not add statistics_split_brain_count to output");
 1028         goto out;
 1029     }
 1030 
 1031     keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
 1032     ret = dict_set_strn(output, key, keylen, crawl_type);
 1033     if (ret) {
 1034         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1035                "Could not add statistics_crawl_type to output");
 1036         goto out;
 1037     }
 1038 
 1039     snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
 1040     ret = dict_set_uint64(output, key, heal_failed_count);
 1041     if (ret) {
 1042         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1043                "Could not add statistics_healed_failed_count to output");
 1044         goto out;
 1045     }
 1046 
 1047     keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
 1048     ret = dict_set_dynstrn(output, key, keylen, start_time_str);
 1049     if (ret) {
 1050         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1051                "Could not add statistics_crawl_start_time to output");
 1052         goto out;
 1053     } else {
 1054         start_time_str = NULL;
 1055     }
 1056 
 1057     if (!end_time_str)
 1058         progress = 1;
 1059     else
 1060         progress = 0;
 1061 
 1062     keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
 1063     if (!end_time_str)
 1064         end_time_str = gf_strdup("Could not determine the end time");
 1065     ret = dict_set_dynstrn(output, key, keylen, end_time_str);
 1066     if (ret) {
 1067         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1068                "Could not add statistics_crawl_end_time to output");
 1069         goto out;
 1070     } else {
 1071         end_time_str = NULL;
 1072     }
 1073 
 1074     keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
 1075 
 1076     ret = dict_set_int32n(output, key, keylen, progress);
 1077     if (ret) {
 1078         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1079                "Could not add statistics_inprogress to output");
 1080         goto out;
 1081     }
 1082 
 1083     snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
 1084     ret = dict_set_uint64(output, key, count + 1);
 1085     if (ret) {
 1086         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1087                "Could not increment the counter.");
 1088         goto out;
 1089     }
 1090 out:
 1091     GF_FREE(start_time_str);
 1092     GF_FREE(end_time_str);
 1093     return ret;
 1094 }
 1095 
 1096 int
 1097 afr_shd_dict_add_path(xlator_t *this, dict_t *output, int child, char *path,
 1098                       struct timeval *tv)
 1099 {
 1100     int ret = -1;
 1101     uint64_t count = 0;
 1102     char key[256] = {0};
 1103     int keylen = 0;
 1104     char xl_id_child_str[64] = {0};
 1105     int xl_id = 0;
 1106 
 1107     ret = dict_get_int32(output, this->name, &xl_id);
 1108     if (ret) {
 1109         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
 1110                "xl does not have id");
 1111         goto out;
 1112     }
 1113 
 1114     snprintf(xl_id_child_str, sizeof(xl_id_child_str), "%d-%d", xl_id, child);
 1115     snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 1116     ret = dict_get_uint64(output, key, &count);
 1117 
 1118     keylen = snprintf(key, sizeof(key), "%s-%" PRIu64, xl_id_child_str, count);
 1119     ret = dict_set_dynstrn(output, key, keylen, path);
 1120 
 1121     if (ret) {
 1122         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1123                "%s: Could not add to output", path);
 1124         goto out;
 1125     }
 1126 
 1127     if (tv) {
 1128         snprintf(key, sizeof(key), "%s-%" PRIu64 "-time", xl_id_child_str,
 1129                  count);
 1130         ret = dict_set_uint32(output, key, tv->tv_sec);
 1131         if (ret) {
 1132             gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1133                    "%s: Could not set time", path);
 1134             goto out;
 1135         }
 1136     }
 1137 
 1138     snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 1139 
 1140     ret = dict_set_uint64(output, key, count + 1);
 1141     if (ret) {
 1142         gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
 1143                "Could not increment count");
 1144         goto out;
 1145     }
 1146 
 1147     ret = 0;
 1148 out:
 1149     return ret;
 1150 }
 1151 
 1152 int
 1153 afr_add_shd_event(circular_buffer_t *cb, void *data)
 1154 {
 1155     dict_t *output = NULL;
 1156     xlator_t *this = THIS;
 1157     afr_private_t *priv = NULL;
 1158     afr_self_heald_t *shd = NULL;
 1159     shd_event_t *shd_event = NULL;
 1160     char *path = NULL;
 1161 
 1162     output = data;
 1163     priv = this->private;
 1164     shd = &priv->shd;
 1165     shd_event = cb->data;
 1166 
 1167     if (!shd->index_healers[shd_event->child].local)
 1168         return 0;
 1169 
 1170     path = gf_strdup(shd_event->path);
 1171     if (!path)
 1172         return -ENOMEM;
 1173 
 1174     afr_shd_dict_add_path(this, output, shd_event->child, path, &cb->tv);
 1175     return 0;
 1176 }
 1177 
 1178 int
 1179 afr_add_crawl_event(circular_buffer_t *cb, void *data)
 1180 {
 1181     dict_t *output = NULL;
 1182     xlator_t *this = THIS;
 1183     afr_private_t *priv = NULL;
 1184     afr_self_heald_t *shd = NULL;
 1185     crawl_event_t *crawl_event = NULL;
 1186 
 1187     output = data;
 1188     priv = this->private;
 1189     shd = &priv->shd;
 1190     crawl_event = cb->data;
 1191 
 1192     if (!shd->index_healers[crawl_event->child].local)
 1193         return 0;
 1194 
 1195     afr_shd_dict_add_crawl_event(this, output, crawl_event);
 1196 
 1197     return 0;
 1198 }
 1199 
 1200 int
 1201 afr_selfheal_daemon_init(xlator_t *this)
 1202 {
 1203     afr_private_t *priv = NULL;
 1204     afr_self_heald_t *shd = NULL;
 1205     int ret = -1;
 1206     int i = 0;
 1207 
 1208     priv = this->private;
 1209     shd = &priv->shd;
 1210 
 1211     shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
 1212                                    priv->child_count,
 1213                                    gf_afr_mt_subvol_healer_t);
 1214     if (!shd->index_healers)
 1215         goto out;
 1216 
 1217     for (i = 0; i < priv->child_count; i++) {
 1218         shd->index_healers[i].subvol = i;
 1219         ret = afr_shd_healer_init(this, &shd->index_healers[i]);
 1220         if (ret)
 1221             goto out;
 1222     }
 1223 
 1224     shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
 1225                                   gf_afr_mt_subvol_healer_t);
 1226     if (!shd->full_healers)
 1227         goto out;
 1228     for (i = 0; i < priv->child_count; i++) {
 1229         shd->full_healers[i].subvol = i;
 1230         ret = afr_shd_healer_init(this, &shd->full_healers[i]);
 1231         if (ret)
 1232             goto out;
 1233     }
 1234 
 1235     shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
 1236                               afr_destroy_shd_event_data);
 1237     if (!shd->split_brain)
 1238         goto out;
 1239 
 1240     shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
 1241                                 gf_common_mt_eh_t);
 1242     if (!shd->statistics)
 1243         goto out;
 1244 
 1245     for (i = 0; i < priv->child_count; i++) {
 1246         shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
 1247                                     afr_destroy_crawl_event_data);
 1248         if (!shd->statistics[i])
 1249             goto out;
 1250         shd->full_healers[i].crawl_event.child = i;
 1251         shd->full_healers[i].crawl_event.crawl_type = "FULL";
 1252         shd->index_healers[i].crawl_event.child = i;
 1253         shd->index_healers[i].crawl_event.crawl_type = "INDEX";
 1254     }
 1255 
 1256     ret = 0;
 1257 out:
 1258     return ret;
 1259 }
 1260 
 1261 void
 1262 afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
 1263 {
 1264     int subvol = 0;
 1265 
 1266     if (!priv->shd.iamshd)
 1267         return;
 1268     for (subvol = 0; subvol < priv->child_count; subvol++)
 1269         if (priv->child_up[subvol])
 1270             afr_shd_index_healer_spawn(this, subvol);
 1271 
 1272     return;
 1273 }
 1274 
 1275 int
 1276 afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
 1277 {
 1278     afr_private_t *priv = NULL;
 1279     xlator_t *subvol = NULL;
 1280     loc_t rootloc = {
 1281         0,
 1282     };
 1283     dict_t *xattr = NULL;
 1284     int ret = -1;
 1285 
 1286     priv = this->private;
 1287     subvol = priv->children[i];
 1288 
 1289     rootloc.inode = inode_ref(this->itable->root);
 1290     gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 1291 
 1292     ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
 1293                           NULL, NULL);
 1294     if (ret < 0)
 1295         goto out;
 1296 
 1297     ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
 1298     if (ret)
 1299         goto out;
 1300 
 1301     ret = 0;
 1302 
 1303 out:
 1304     if (xattr)
 1305         dict_unref(xattr);
 1306     loc_wipe(&rootloc);
 1307 
 1308     return ret;
 1309 }
 1310 
 1311 int
 1312 afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 1313 {
 1314     gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 1315     int ret = 0;
 1316     int xl_id = 0;
 1317     afr_private_t *priv = NULL;
 1318     afr_self_heald_t *shd = NULL;
 1319     struct subvol_healer *healer = NULL;
 1320     int i = 0;
 1321     char key[64];
 1322     int keylen = 0;
 1323     int this_name_len = 0;
 1324     int op_ret = 0;
 1325     uint64_t cnt = 0;
 1326 
 1327     priv = this->private;
 1328     shd = &priv->shd;
 1329 
 1330     ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
 1331     if (ret)
 1332         goto out;
 1333     this_name_len = strlen(this->name);
 1334     ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
 1335     if (ret)
 1336         goto out;
 1337     ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
 1338     if (ret)
 1339         goto out;
 1340     switch (op) {
 1341         case GF_SHD_OP_HEAL_INDEX:
 1342             op_ret = 0;
 1343 
 1344             for (i = 0; i < priv->child_count; i++) {
 1345                 healer = &shd->index_healers[i];
 1346                 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
 1347 
 1348                 if (!priv->child_up[i]) {
 1349                     ret = dict_set_nstrn(output, key, keylen,
 1350                                          SBRICK_NOT_CONNECTED,
 1351                                          SLEN(SBRICK_NOT_CONNECTED));
 1352                     op_ret = -1;
 1353                 } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
 1354                     ret = dict_set_nstrn(output, key, keylen,
 1355                                          SLESS_THAN2_BRICKS_in_REP,
 1356                                          SLEN(SLESS_THAN2_BRICKS_in_REP));
 1357                     op_ret = -1;
 1358                 } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
 1359                     ret = dict_set_nstrn(output, key, keylen, SBRICK_IS_REMOTE,
 1360                                          SLEN(SBRICK_IS_REMOTE));
 1361                 } else {
 1362                     ret = dict_set_nstrn(output, key, keylen,
 1363                                          SSTARTED_SELF_HEAL,
 1364                                          SLEN(SSTARTED_SELF_HEAL));
 1365                     afr_shd_index_healer_spawn(this, i);
 1366                 }
 1367             }
 1368             break;
 1369         case GF_SHD_OP_HEAL_FULL:
 1370             op_ret = -1;
 1371 
 1372             for (i = 0; i < priv->child_count; i++) {
 1373                 healer = &shd->full_healers[i];
 1374                 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
 1375 
 1376                 if (!priv->child_up[i]) {
 1377                     ret = dict_set_nstrn(output, key, keylen,
 1378                                          SBRICK_NOT_CONNECTED,
 1379                                          SLEN(SBRICK_NOT_CONNECTED));
 1380                 } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
 1381                     ret = dict_set_nstrn(output, key, keylen,
 1382                                          SLESS_THAN2_BRICKS_in_REP,
 1383                                          SLEN(SLESS_THAN2_BRICKS_in_REP));
 1384                 } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
 1385                     ret = dict_set_nstrn(output, key, keylen, SBRICK_IS_REMOTE,
 1386                                          SLEN(SBRICK_IS_REMOTE));
 1387                 } else {
 1388                     ret = dict_set_nstrn(output, key, keylen,
 1389                                          SSTARTED_SELF_HEAL,
 1390                                          SLEN(SSTARTED_SELF_HEAL));
 1391                     afr_shd_full_healer_spawn(this, i);
 1392                     op_ret = 0;
 1393                 }
 1394             }
 1395             break;
 1396         case GF_SHD_OP_INDEX_SUMMARY:
 1397             /* this case has been handled in glfs-heal.c */
 1398             break;
 1399         case GF_SHD_OP_HEALED_FILES:
 1400         case GF_SHD_OP_HEAL_FAILED_FILES:
 1401             for (i = 0; i < priv->child_count; i++) {
 1402                 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
 1403                 ret = dict_set_nstrn(output, key, keylen, SOP_NOT_SUPPORTED,
 1404                                      SLEN(SOP_NOT_SUPPORTED));
 1405             }
 1406             break;
 1407         case GF_SHD_OP_SPLIT_BRAIN_FILES:
 1408             eh_dump(shd->split_brain, output, afr_add_shd_event);
 1409             break;
 1410         case GF_SHD_OP_STATISTICS:
 1411             for (i = 0; i < priv->child_count; i++) {
 1412                 eh_dump(shd->statistics[i], output, afr_add_crawl_event);
 1413                 afr_shd_dict_add_crawl_event(
 1414                     this, output, &shd->index_healers[i].crawl_event);
 1415                 afr_shd_dict_add_crawl_event(this, output,
 1416                                              &shd->full_healers[i].crawl_event);
 1417             }
 1418             break;
 1419         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
 1420         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
 1421             op_ret = -1;
 1422 
 1423             for (i = 0; i < priv->child_count; i++) {
 1424                 if (!priv->child_up[i]) {
 1425                     keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
 1426                                       i);
 1427                     ret = dict_set_nstrn(output, key, keylen,
 1428                                          SBRICK_NOT_CONNECTED,
 1429                                          SLEN(SBRICK_NOT_CONNECTED));
 1430                 } else {
 1431                     snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
 1432                     ret = afr_shd_get_index_count(this, i, &cnt);
 1433                     if (ret == 0) {
 1434                         ret = dict_set_uint64(output, key, cnt);
 1435                     }
 1436                     op_ret = 0;
 1437                 }
 1438             }
 1439 
 1440             break;
 1441 
 1442         default:
 1443             gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG,
 1444                    "Unknown set op %d", op);
 1445             break;
 1446     }
 1447 out:
 1448     dict_deln(output, this->name, this_name_len);
 1449     return op_ret;
 1450 }