"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/xlators/mgmt/glusterd/src/glusterd-rebalance.c" (16 Sep 2020, 47897 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "glusterd-rebalance.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2    Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
    3    This file is part of GlusterFS.
    4 
    5    This file is licensed to you under your choice of the GNU Lesser
    6    General Public License, version 3 or any later version (LGPLv3 or
    7    later), or the GNU General Public License, version 2 (GPLv2), in all
    8    cases as published by the Free Software Foundation.
    9 */
   10 #include <inttypes.h>
   11 #include <sys/types.h>
   12 #include <unistd.h>
   13 #include <sys/resource.h>
   14 #include <sys/statvfs.h>
   15 
   16 #include <glusterfs/compat.h>
   17 #include "protocol-common.h"
   18 #include <glusterfs/xlator.h>
   19 #include <glusterfs/logging.h>
   20 #include <glusterfs/timer.h>
   21 #include "glusterd-mem-types.h"
   22 #include "glusterd.h"
   23 #include "glusterd-sm.h"
   24 #include "glusterd-op-sm.h"
   25 #include "glusterd-utils.h"
   26 #include "glusterd-mgmt.h"
   27 #include "glusterd-messages.h"
   28 #include "glusterd-store.h"
   29 #include <glusterfs/run.h>
   30 #include "glusterd-volgen.h"
   31 #include "glusterd-messages.h"
   32 
   33 #include <glusterfs/syscall.h>
   34 #include "cli1-xdr.h"
   35 #include "xdr-generic.h"
   36 
   37 #define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo)                           \
   38     do {                                                                       \
   39         int32_t _defrag_sockfile_len;                                          \
   40         char tmppath[PATH_MAX] = {                                             \
   41             0,                                                                 \
   42         };                                                                     \
   43         _defrag_sockfile_len = snprintf(                                       \
   44             tmppath, PATH_MAX,                                                 \
   45             DEFAULT_VAR_RUN_DIRECTORY "/gluster-%s-%s-%s.sock", "rebalance",   \
   46             volinfo->volname, uuid_utoa(MY_UUID));                             \
   47         if ((_defrag_sockfile_len < 0) ||                                      \
   48             (_defrag_sockfile_len >= PATH_MAX)) {                              \
   49             path[0] = 0;                                                       \
   50         } else {                                                               \
   51             glusterd_set_socket_filepath(tmppath, path, sizeof(path));         \
   52         }                                                                      \
   53     } while (0)
   54 
   55 int32_t
   56 glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
   57                       void *myframe);
   58 int
   59 glusterd_defrag_start_validate(glusterd_volinfo_t *volinfo, char *op_errstr,
   60                                size_t len, glusterd_op_t op)
   61 {
   62     int ret = -1;
   63     xlator_t *this = NULL;
   64 
   65     this = THIS;
   66     GF_ASSERT(this);
   67 
   68     /* Check only if operation is not remove-brick */
   69     if ((GD_OP_REMOVE_BRICK != op) && !gd_is_remove_brick_committed(volinfo)) {
   70         gf_msg_debug(this->name, 0,
   71                      "A remove-brick task on "
   72                      "volume %s is not yet committed",
   73                      volinfo->volname);
   74         snprintf(op_errstr, len,
   75                  "A remove-brick task on volume %s is"
   76                  " not yet committed. Either commit or stop the "
   77                  "remove-brick task.",
   78                  volinfo->volname);
   79         goto out;
   80     }
   81 
   82     if (glusterd_is_defrag_on(volinfo)) {
   83         gf_msg_debug(this->name, 0, "rebalance on volume %s already started",
   84                      volinfo->volname);
   85         snprintf(op_errstr, len, "Rebalance on %s is already started",
   86                  volinfo->volname);
   87         goto out;
   88     }
   89 
   90     ret = 0;
   91 out:
   92     gf_msg_debug(this->name, 0, "Returning %d", ret);
   93     return ret;
   94 }
   95 
   96 int32_t
   97 __glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
   98                          rpc_clnt_event_t event, void *data)
   99 {
  100     glusterd_volinfo_t *volinfo = NULL;
  101     glusterd_defrag_info_t *defrag = NULL;
  102     int ret = 0;
  103     char pidfile[PATH_MAX];
  104     glusterd_conf_t *priv = NULL;
  105     xlator_t *this = NULL;
  106     int pid = -1;
  107 
  108     this = THIS;
  109     if (!this)
  110         return 0;
  111 
  112     priv = this->private;
  113     if (!priv)
  114         return 0;
  115 
  116     volinfo = mydata;
  117     if (!volinfo)
  118         return 0;
  119 
  120     defrag = volinfo->rebal.defrag;
  121     if (!defrag)
  122         return 0;
  123 
  124     if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
  125         volinfo->rebal.defrag = NULL;
  126 
  127     GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
  128 
  129     switch (event) {
  130         case RPC_CLNT_CONNECT: {
  131             if (defrag->connected)
  132                 return 0;
  133 
  134             LOCK(&defrag->lock);
  135             {
  136                 defrag->connected = 1;
  137             }
  138             UNLOCK(&defrag->lock);
  139 
  140             gf_msg_debug(this->name, 0, "%s got RPC_CLNT_CONNECT",
  141                          rpc->conn.name);
  142             break;
  143         }
  144 
  145         case RPC_CLNT_DISCONNECT: {
  146             if (!defrag->connected)
  147                 return 0;
  148 
  149             LOCK(&defrag->lock);
  150             {
  151                 defrag->connected = 0;
  152             }
  153             UNLOCK(&defrag->lock);
  154 
  155             if (!gf_is_service_running(pidfile, &pid)) {
  156                 if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED) {
  157                     volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_FAILED;
  158                 }
  159             }
  160 
  161             glusterd_store_perform_node_state_store(volinfo);
  162 
  163             rpc_clnt_disable(defrag->rpc);
  164             glusterd_defrag_rpc_put(defrag);
  165             if (defrag->cbk_fn)
  166                 defrag->cbk_fn(volinfo, volinfo->rebal.defrag_status);
  167 
  168             GF_FREE(defrag);
  169             gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REBALANCE_DISCONNECTED,
  170                    "Rebalance process for volume %s has disconnected.",
  171                    volinfo->volname);
  172             break;
  173         }
  174         case RPC_CLNT_DESTROY:
  175             glusterd_volinfo_unref(volinfo);
  176             break;
  177         default:
  178             gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
  179             ret = 0;
  180             break;
  181     }
  182 
  183     return ret;
  184 }
  185 
  186 int32_t
  187 glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
  188                        rpc_clnt_event_t event, void *data)
  189 {
  190     return glusterd_big_locked_notify(rpc, mydata, event, data,
  191                                       __glusterd_defrag_notify);
  192 }
  193 
  194 int
  195 glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr,
  196                              size_t len, int cmd, defrag_cbk_fn_t cbk,
  197                              glusterd_op_t op)
  198 {
  199     xlator_t *this = NULL;
  200     int ret = -1;
  201     glusterd_defrag_info_t *defrag = NULL;
  202     runner_t runner = {
  203         0,
  204     };
  205     glusterd_conf_t *priv = NULL;
  206     char defrag_path[PATH_MAX];
  207     char sockfile[PATH_MAX] = {
  208         0,
  209     };
  210     char pidfile[PATH_MAX] = {
  211         0,
  212     };
  213     char logfile[PATH_MAX] = {
  214         0,
  215     };
  216     char volname[PATH_MAX] = {
  217         0,
  218     };
  219     char valgrind_logfile[PATH_MAX] = {
  220         0,
  221     };
  222     char *volfileserver = NULL;
  223     char *localtime_logging = NULL;
  224 
  225     this = THIS;
  226     GF_VALIDATE_OR_GOTO("glusterd", this, out);
  227 
  228     priv = this->private;
  229     GF_VALIDATE_OR_GOTO("glusterd", priv, out);
  230 
  231     GF_ASSERT(volinfo);
  232     GF_ASSERT(op_errstr);
  233 
  234     ret = glusterd_defrag_start_validate(volinfo, op_errstr, len, op);
  235     if (ret)
  236         goto out;
  237     if (!volinfo->rebal.defrag)
  238         volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
  239                                           gf_gld_mt_defrag_info);
  240     if (!volinfo->rebal.defrag)
  241         goto out;
  242 
  243     defrag = volinfo->rebal.defrag;
  244 
  245     defrag->cmd = cmd;
  246 
  247     volinfo->rebal.defrag_cmd = cmd;
  248     volinfo->rebal.op = op;
  249 
  250     LOCK_INIT(&defrag->lock);
  251 
  252     volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
  253 
  254     glusterd_volinfo_reset_defrag_stats(volinfo);
  255     glusterd_store_perform_node_state_store(volinfo);
  256 
  257     GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);
  258     ret = mkdir_p(defrag_path, 0755, _gf_true);
  259     if (ret) {
  260         gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
  261                "Failed to create "
  262                "directory %s",
  263                defrag_path);
  264         goto out;
  265     }
  266 
  267     GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
  268     GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
  269     snprintf(logfile, PATH_MAX, "%s/%s-%s.log", priv->logdir, volinfo->volname,
  270              "rebalance");
  271     runinit(&runner);
  272 
  273     if (this->ctx->cmd_args.valgrind) {
  274         snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-rebalance.log",
  275                  priv->logdir, volinfo->volname);
  276 
  277         runner_add_args(&runner, "valgrind", "--leak-check=full",
  278                         "--trace-children=yes", "--track-origins=yes", NULL);
  279         runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
  280     }
  281 
  282     snprintf(volname, sizeof(volname), "rebalance/%s", volinfo->volname);
  283 
  284     if (dict_get_strn(this->options, "transport.socket.bind-address",
  285                       SLEN("transport.socket.bind-address"),
  286                       &volfileserver) != 0) {
  287         volfileserver = "localhost";
  288     }
  289 
  290     runner_add_args(
  291         &runner, SBIN_DIR "/glusterfs", "-s", volfileserver, "--volfile-id",
  292         volname, "--xlator-option", "*dht.use-readdirp=yes", "--xlator-option",
  293         "*dht.lookup-unhashed=yes", "--xlator-option",
  294         "*dht.assert-no-child-down=yes", "--xlator-option",
  295         "*dht.readdir-optimize=on", "--process-name", "rebalance", NULL);
  296 
  297     runner_add_arg(&runner, "--xlator-option");
  298     runner_argprintf(&runner, "*dht.rebalance-cmd=%d", cmd);
  299     runner_add_arg(&runner, "--xlator-option");
  300     runner_argprintf(&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
  301     runner_add_arg(&runner, "--xlator-option");
  302     runner_argprintf(&runner, "*dht.commit-hash=%u",
  303                      volinfo->rebal.commit_hash);
  304     runner_add_arg(&runner, "--socket-file");
  305     runner_argprintf(&runner, "%s", sockfile);
  306     runner_add_arg(&runner, "--pid-file");
  307     runner_argprintf(&runner, "%s", pidfile);
  308     runner_add_arg(&runner, "-l");
  309     runner_argprintf(&runner, "%s", logfile);
  310     if (volinfo->memory_accounting)
  311         runner_add_arg(&runner, "--mem-accounting");
  312     if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
  313                       SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
  314                       &localtime_logging) == 0) {
  315         if (strcmp(localtime_logging, "enable") == 0)
  316             runner_add_arg(&runner, "--localtime-logging");
  317     }
  318 
  319     ret = runner_run_nowait(&runner);
  320     if (ret) {
  321         gf_msg_debug("glusterd", 0, "rebalance command failed");
  322         goto out;
  323     }
  324 
  325     sleep(5);
  326 
  327     ret = glusterd_rebalance_rpc_create(volinfo);
  328 
  329     // FIXME: this cbk is passed as NULL in all occurrences. May be
  330     // we never needed it.
  331     if (cbk)
  332         defrag->cbk_fn = cbk;
  333 
  334 out:
  335     gf_msg_debug("glusterd", 0, "Returning %d", ret);
  336     return ret;
  337 }
  338 
  339 int
  340 glusterd_rebalance_defrag_init(glusterd_volinfo_t *volinfo, defrag_cbk_fn_t cbk)
  341 
  342 {
  343     glusterd_defrag_info_t *defrag = NULL;
  344     int ret = -1;
  345 
  346     if (!volinfo->rebal.defrag) {
  347         volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
  348                                           gf_gld_mt_defrag_info);
  349     } else {
  350         /*
  351          * if defrag variable is already initialized,
  352          * we skip the initialization.
  353          */
  354         ret = 0;
  355         goto out;
  356     }
  357 
  358     if (!volinfo->rebal.defrag)
  359         goto out;
  360     defrag = volinfo->rebal.defrag;
  361 
  362     defrag->cmd = volinfo->rebal.defrag_cmd;
  363     LOCK_INIT(&defrag->lock);
  364     if (cbk)
  365         defrag->cbk_fn = cbk;
  366     ret = 0;
  367 out:
  368     return ret;
  369 }
  370 
  371 int
  372 glusterd_rebalance_rpc_create(glusterd_volinfo_t *volinfo)
  373 {
  374     dict_t *options = NULL;
  375     char sockfile[PATH_MAX] = {
  376         0,
  377     };
  378     int ret = -1;
  379     glusterd_defrag_info_t *defrag = volinfo->rebal.defrag;
  380     glusterd_conf_t *priv = NULL;
  381     xlator_t *this = NULL;
  382 
  383     this = THIS;
  384     GF_ASSERT(this);
  385     priv = this->private;
  386     GF_ASSERT(priv);
  387 
  388     // rebalance process is not started
  389     if (!defrag)
  390         goto out;
  391 
  392     options = dict_new();
  393     if (!options)
  394         goto out;
  395 
  396     GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
  397 
  398     /* Setting frame-timeout to 10mins (600seconds).
  399      * Unix domain sockets ensures that the connection is reliable. The
  400      * default timeout of 30mins used for unreliable network connections is
  401      * too long for unix domain socket connections.
  402      */
  403     ret = rpc_transport_unix_options_build(options, sockfile, 600);
  404     if (ret) {
  405         gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNIX_OP_BUILD_FAIL,
  406                "Unix options build failed");
  407         goto out;
  408     }
  409 
  410     glusterd_volinfo_ref(volinfo);
  411     ret = glusterd_rpc_create(&defrag->rpc, options, glusterd_defrag_notify,
  412                               volinfo, _gf_true);
  413     if (ret) {
  414         gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
  415                "Glusterd RPC creation failed");
  416         goto out;
  417     }
  418     ret = 0;
  419 out:
  420     if (options)
  421         dict_unref(options);
  422     return ret;
  423 }
  424 
  425 int
  426 glusterd_rebalance_cmd_validate(int cmd, char *volname,
  427                                 glusterd_volinfo_t **volinfo, char *op_errstr,
  428                                 size_t len)
  429 {
  430     int ret = -1;
  431 
  432     if (glusterd_volinfo_find(volname, volinfo)) {
  433         gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
  434                "Received rebalance on invalid"
  435                " volname %s",
  436                volname);
  437         snprintf(op_errstr, len, "Volume %s does not exist", volname);
  438         goto out;
  439     }
  440     if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
  441         gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_DISTRIBUTE,
  442                "Volume %s is not a "
  443                "distribute type or contains only 1 brick",
  444                volname);
  445         snprintf(op_errstr, len,
  446                  "Volume %s is not a distribute "
  447                  "volume or contains only 1 brick.\n"
  448                  "Not performing rebalance",
  449                  volname);
  450         goto out;
  451     }
  452 
  453     if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
  454         gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_STOPPED,
  455                "Received rebalance on stopped"
  456                " volname %s",
  457                volname);
  458         snprintf(op_errstr, len,
  459                  "Volume %s needs to "
  460                  "be started to perform rebalance",
  461                  volname);
  462         goto out;
  463     }
  464 
  465     ret = 0;
  466 
  467 out:
  468     gf_msg_debug("glusterd", 0, "Returning %d", ret);
  469     return ret;
  470 }
  471 
  472 int
  473 __glusterd_handle_defrag_volume(rpcsvc_request_t *req)
  474 {
  475     int32_t ret = -1;
  476     gf_cli_req cli_req = {{
  477         0,
  478     }};
  479     glusterd_conf_t *priv = NULL;
  480     int32_t op = GD_OP_NONE;
  481     dict_t *dict = NULL;
  482     char *volname = NULL;
  483     gf_cli_defrag_type cmd = 0;
  484     char msg[2048] = {
  485         0,
  486     };
  487     xlator_t *this = NULL;
  488 
  489     GF_ASSERT(req);
  490     this = THIS;
  491     GF_ASSERT(this);
  492 
  493     priv = this->private;
  494     GF_ASSERT(priv);
  495 
  496     ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
  497     if (ret < 0) {
  498         // failed to decode msg;
  499         req->rpc_err = GARBAGE_ARGS;
  500         goto out;
  501     }
  502 
  503     if (cli_req.dict.dict_len) {
  504         /* Unserialize the dictionary */
  505         dict = dict_new();
  506 
  507         ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
  508                                &dict);
  509         if (ret < 0) {
  510             gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
  511                    "failed to "
  512                    "unserialize req-buffer to dictionary");
  513             snprintf(msg, sizeof(msg),
  514                      "Unable to decode the "
  515                      "command");
  516             goto out;
  517         }
  518     }
  519 
  520     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
  521     if (ret) {
  522         snprintf(msg, sizeof(msg), "Failed to get volume name");
  523         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
  524         goto out;
  525     }
  526 
  527     ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
  528                           (int32_t *)&cmd);
  529     if (ret) {
  530         snprintf(msg, sizeof(msg), "Failed to get command");
  531         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
  532         goto out;
  533     }
  534 
  535     ret = dict_set_static_bin(dict, "node-uuid", MY_UUID, 16);
  536     if (ret)
  537         goto out;
  538 
  539     if ((cmd == GF_DEFRAG_CMD_STATUS) || (cmd == GF_DEFRAG_CMD_STOP)) {
  540         op = GD_OP_DEFRAG_BRICK_VOLUME;
  541     } else
  542         op = GD_OP_REBALANCE;
  543 
  544     if (priv->op_version < GD_OP_VERSION_6_0) {
  545         gf_msg_debug(this->name, 0,
  546                      "The cluster is operating at "
  547                      "version less than %d. Falling back "
  548                      "to op-sm framework.",
  549                      GD_OP_VERSION_6_0);
  550         ret = glusterd_op_begin(req, op, dict, msg, sizeof(msg));
  551         glusterd_friend_sm();
  552         glusterd_op_sm();
  553     } else {
  554         ret = glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(req, op,
  555                                                                       dict);
  556     }
  557 out:
  558     if (ret) {
  559         if (msg[0] == '\0')
  560             snprintf(msg, sizeof(msg), "Operation failed");
  561         ret = glusterd_op_send_cli_response(GD_OP_REBALANCE, ret, 0, req, dict,
  562                                             msg);
  563     }
  564 
  565     free(cli_req.dict.dict_val);  // malloced by xdr
  566     gf_msg_debug(this->name, 0, "Returning %d", ret);
  567     return ret;
  568 }
  569 
  570 int
  571 glusterd_handle_defrag_volume(rpcsvc_request_t *req)
  572 {
  573     return glusterd_big_locked_handler(req, __glusterd_handle_defrag_volume);
  574 }
  575 
  576 static int
  577 glusterd_brick_validation(dict_t *dict, char *key, data_t *value, void *data)
  578 {
  579     int32_t ret = -1;
  580     xlator_t *this = NULL;
  581     glusterd_volinfo_t *volinfo = data;
  582     glusterd_brickinfo_t *brickinfo = NULL;
  583 
  584     this = THIS;
  585     GF_ASSERT(this);
  586 
  587     ret = glusterd_volume_brickinfo_get_by_brick(value->data, volinfo,
  588                                                  &brickinfo, _gf_false);
  589     if (ret) {
  590         gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
  591                "Incorrect brick %s for "
  592                "volume %s",
  593                value->data, volinfo->volname);
  594         return ret;
  595     }
  596 
  597     if (!brickinfo->decommissioned) {
  598         gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
  599                "Incorrect brick %s for "
  600                "volume %s",
  601                value->data, volinfo->volname);
  602         ret = -1;
  603         return ret;
  604     }
  605 
  606     return ret;
  607 }
  608 
  609 int
  610 glusterd_set_rebalance_id_in_rsp_dict(dict_t *req_dict, dict_t *rsp_dict)
  611 {
  612     int ret = -1;
  613     int32_t cmd = 0;
  614     char *volname = NULL;
  615     glusterd_volinfo_t *volinfo = NULL;
  616     char msg[2048] = {0};
  617     char *task_id_str = NULL;
  618     xlator_t *this = NULL;
  619 
  620     this = THIS;
  621     GF_ASSERT(this);
  622 
  623     GF_ASSERT(rsp_dict);
  624     GF_ASSERT(req_dict);
  625 
  626     ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname);
  627     if (ret) {
  628         gf_msg_debug(this->name, 0, "volname not found");
  629         goto out;
  630     }
  631 
  632     ret = dict_get_int32n(rsp_dict, "rebalance-command",
  633                           SLEN("rebalance-command"), &cmd);
  634     if (ret) {
  635         gf_msg_debug(this->name, 0, "cmd not found");
  636         goto out;
  637     }
  638 
  639     ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
  640                                           sizeof(msg));
  641     if (ret) {
  642         gf_msg_debug(this->name, 0, "failed to validate");
  643         goto out;
  644     }
  645 
  646     /* reblance id is generted in glusterd_mgmt_v3_op_stage_rebalance(), but
  647      * rsp_dict is unavailable there. So copying it to rsp_dict from req_dict
  648      * here. So that cli can display the rebalance id.*/
  649     if ((cmd == GF_DEFRAG_CMD_START) ||
  650         (cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX) ||
  651         (cmd == GF_DEFRAG_CMD_START_FORCE)) {
  652         if (is_origin_glusterd(rsp_dict)) {
  653             ret = dict_get_strn(req_dict, GF_REBALANCE_TID_KEY,
  654                                 SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
  655             if (ret) {
  656                 snprintf(msg, sizeof(msg), "Missing rebalance-id");
  657                 gf_msg(this->name, GF_LOG_WARNING, 0,
  658                        GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
  659                 ret = 0;
  660             } else {
  661                 gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
  662                 ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
  663                                                  rsp_dict, GF_REBALANCE_TID_KEY,
  664                                                  SLEN(GF_REBALANCE_TID_KEY));
  665                 if (ret) {
  666                     snprintf(msg, sizeof(msg),
  667                              "Failed to set rebalance id for volume %s",
  668                              volname);
  669                     gf_msg(this->name, GF_LOG_WARNING, 0,
  670                            GD_MSG_DICT_SET_FAILED, "%s", msg);
  671                 }
  672             }
  673         }
  674     }
  675 
  676     /* Set task-id, if available, in rsp_dict for operations other than
  677      * start. This is needed when we want rebalance id in xml output
  678      */
  679     if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
  680         if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
  681             if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
  682                 ret = glusterd_copy_uuid_to_dict(
  683                     volinfo->rebal.rebalance_id, rsp_dict,
  684                     GF_REMOVE_BRICK_TID_KEY, SLEN(GF_REMOVE_BRICK_TID_KEY));
  685             else
  686                 ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
  687                                                  rsp_dict, GF_REBALANCE_TID_KEY,
  688                                                  SLEN(GF_REBALANCE_TID_KEY));
  689             if (ret) {
  690                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
  691                        "Failed to set task-id for volume %s", volname);
  692                 goto out;
  693             }
  694         }
  695     }
  696 out:
  697     return ret;
  698 }
  699 
  700 int
  701 glusterd_mgmt_v3_op_stage_rebalance(dict_t *dict, char **op_errstr)
  702 {
  703     char *volname = NULL;
  704     char *cmd_str = NULL;
  705     int ret = 0;
  706     int32_t cmd = 0;
  707     char msg[2048] = {0};
  708     glusterd_volinfo_t *volinfo = NULL;
  709     char *task_id_str = NULL;
  710     xlator_t *this = 0;
  711 
  712     this = THIS;
  713     GF_ASSERT(this);
  714 
  715     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
  716     if (ret) {
  717         gf_msg_debug(this->name, 0, "volname not found");
  718         goto out;
  719     }
  720 
  721     ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
  722                           &cmd);
  723     if (ret) {
  724         gf_msg_debug(this->name, 0, "cmd not found");
  725         goto out;
  726     }
  727 
  728     ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
  729                                           sizeof(msg));
  730     if (ret) {
  731         gf_msg_debug(this->name, 0, "failed to validate");
  732         goto out;
  733     }
  734     switch (cmd) {
  735         case GF_DEFRAG_CMD_START:
  736         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
  737             /* Check if the connected clients are all of version
  738              * glusterfs-3.6 and higher. This is needed to prevent some data
  739              * loss issues that could occur when older clients are connected
  740              * when rebalance is run. This check can be bypassed by using
  741              * 'force'
  742              */
  743             ret = glusterd_check_client_op_version_support(
  744                 volname, GD_OP_VERSION_3_6_0, NULL);
  745             if (ret) {
  746                 ret = gf_asprintf(op_errstr,
  747                                   "Volume %s has one or "
  748                                   "more connected clients of a version"
  749                                   " lower than GlusterFS-v3.6.0. "
  750                                   "Starting rebalance in this state "
  751                                   "could lead to data loss.\nPlease "
  752                                   "disconnect those clients before "
  753                                   "attempting this command again.",
  754                                   volname);
  755                 goto out;
  756             }
  757             /* Fall through */
  758         case GF_DEFRAG_CMD_START_FORCE:
  759             if (is_origin_glusterd(dict)) {
  760                 ret = glusterd_generate_and_set_task_id(
  761                     dict, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
  762                 if (ret) {
  763                     gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
  764                            "Failed to generate task-id");
  765                     goto out;
  766                 }
  767             } else {
  768                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
  769                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
  770                 if (ret) {
  771                     snprintf(msg, sizeof(msg), "Missing rebalance-id");
  772                     gf_msg(this->name, GF_LOG_WARNING, 0,
  773                            GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
  774                     ret = 0;
  775                 }
  776             }
  777             ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
  778                                                  GD_OP_REBALANCE);
  779             if (ret) {
  780                 gf_msg_debug(this->name, 0,
  781                              "defrag start validate "
  782                              "failed for volume %s.",
  783                              volinfo->volname);
  784                 goto out;
  785             }
  786             break;
  787         case GF_DEFRAG_CMD_STATUS:
  788         case GF_DEFRAG_CMD_STOP:
  789 
  790             ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
  791             if (ret) {
  792                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
  793                        "Failed to get "
  794                        "command string");
  795                 ret = -1;
  796                 goto out;
  797             }
  798             if ((strstr(cmd_str, "rebalance") != NULL) &&
  799                 (volinfo->rebal.op != GD_OP_REBALANCE)) {
  800                 snprintf(msg, sizeof(msg),
  801                          "Rebalance not started "
  802                          "for volume %s.",
  803                          volinfo->volname);
  804                 ret = -1;
  805                 goto out;
  806             }
  807 
  808             if (strstr(cmd_str, "remove-brick") != NULL) {
  809                 if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
  810                     snprintf(msg, sizeof(msg),
  811                              "remove-brick not "
  812                              "started for volume %s.",
  813                              volinfo->volname);
  814                     ret = -1;
  815                     goto out;
  816                 }
  817 
  818                 /* For remove-brick status/stop command check whether
  819                  * given input brick is part of volume or not.*/
  820 
  821                 ret = dict_foreach_fnmatch(dict, "brick*",
  822                                            glusterd_brick_validation, volinfo);
  823                 if (ret == -1) {
  824                     snprintf(msg, sizeof(msg),
  825                              "Incorrect brick"
  826                              " for volume %s",
  827                              volinfo->volname);
  828                     goto out;
  829                 }
  830             }
  831             break;
  832 
  833         default:
  834             break;
  835     }
  836 
  837     ret = 0;
  838 out:
  839     if (ret && op_errstr && msg[0])
  840         *op_errstr = gf_strdup(msg);
  841 
  842     return ret;
  843 }
  844 
  845 int
  846 glusterd_mgmt_v3_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
  847 {
  848     char *volname = NULL;
  849     int ret = 0;
  850     int32_t cmd = 0;
  851     char msg[2048] = {0};
  852     glusterd_volinfo_t *volinfo = NULL;
  853     glusterd_brickinfo_t *brickinfo = NULL;
  854     glusterd_brickinfo_t *tmp = NULL;
  855     gf_boolean_t volfile_update = _gf_false;
  856     char *task_id_str = NULL;
  857     xlator_t *this = NULL;
  858     uint32_t commit_hash;
  859     int32_t is_force = 0;
  860 
  861     this = THIS;
  862     GF_ASSERT(this);
  863 
  864     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
  865     if (ret) {
  866         gf_msg_debug(this->name, 0, "volname not given");
  867         goto out;
  868     }
  869 
  870     ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
  871                           &cmd);
  872     if (ret) {
  873         gf_msg_debug(this->name, 0, "command not given");
  874         goto out;
  875     }
  876 
  877     ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
  878                                           sizeof(msg));
  879     if (ret) {
  880         gf_msg_debug(this->name, 0, "cmd validate failed");
  881         goto out;
  882     }
  883 
  884     switch (cmd) {
  885         case GF_DEFRAG_CMD_START:
  886         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
  887         case GF_DEFRAG_CMD_START_FORCE:
  888 
  889             ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
  890             if (ret)
  891                 is_force = 0;
  892             if (!is_force) {
  893                 /* Reset defrag status to 'NOT STARTED' whenever a
  894                  * remove-brick/rebalance command is issued to remove
  895                  * stale information from previous run.
  896                  */
  897                 volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
  898 
  899                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
  900                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
  901                 if (ret) {
  902                     gf_msg_debug(this->name, 0,
  903                                  "Missing rebalance"
  904                                  " id");
  905                     ret = 0;
  906                 } else {
  907                     gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
  908                     volinfo->rebal.op = GD_OP_REBALANCE;
  909                 }
  910                 if (!gd_should_i_start_rebalance(volinfo)) {
  911                     /* Store the rebalance-id and rebalance command
  912                      * even if the peer isn't starting a rebalance
  913                      * process. On peers where a rebalance process
  914                      * is started, glusterd_handle_defrag_start
  915                      * performs the storing.
  916                      * Storing this is needed for having
  917                      * 'volume status' work correctly.
  918                      */
  919                     glusterd_store_perform_node_state_store(volinfo);
  920                     break;
  921                 }
  922                 if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
  923                     volinfo->rebal.commit_hash = commit_hash;
  924                 }
  925                 ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
  926                                                    cmd, NULL, GD_OP_REBALANCE);
  927                 break;
  928             } else {
  929                 /* Reset defrag status to 'STARTED' so that the
  930                  * pid is checked and restarted accordingly.
  931                  * If the pid is not running it executes the
  932                  * "NOT_STARTED" case and restarts the process
  933                  */
  934                 volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
  935                 volinfo->rebal.defrag_cmd = cmd;
  936                 volinfo->rebal.op = GD_OP_REBALANCE;
  937 
  938                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
  939                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
  940                 if (ret) {
  941                     gf_msg_debug(this->name, 0,
  942                                  "Missing rebalance"
  943                                  " id");
  944                     ret = 0;
  945                 } else {
  946                     gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
  947                     volinfo->rebal.op = GD_OP_REBALANCE;
  948                 }
  949                 if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
  950                     volinfo->rebal.commit_hash = commit_hash;
  951                 }
  952                 ret = glusterd_restart_rebalance_for_volume(volinfo);
  953                 break;
  954             }
  955         case GF_DEFRAG_CMD_STOP:
  956             /* Clear task-id only on explicitly stopping rebalance.
  957              * Also clear the stored operation, so it doesn't cause trouble
  958              * with future rebalance/remove-brick starts
  959              */
  960             gf_uuid_clear(volinfo->rebal.rebalance_id);
  961             volinfo->rebal.op = GD_OP_NONE;
  962 
  963             /* Fall back to the old volume file in case of decommission*/
  964             cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
  965                                          brick_list)
  966             {
  967                 if (!brickinfo->decommissioned)
  968                     continue;
  969                 brickinfo->decommissioned = 0;
  970                 volfile_update = _gf_true;
  971             }
  972 
  973             if (volfile_update == _gf_false) {
  974                 ret = 0;
  975                 break;
  976             }
  977 
  978             ret = glusterd_create_volfiles_and_notify_services(volinfo);
  979             if (ret) {
  980                 gf_msg(this->name, GF_LOG_WARNING, 0,
  981                        GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
  982                 goto out;
  983             }
  984 
  985             ret = glusterd_store_volinfo(volinfo,
  986                                          GLUSTERD_VOLINFO_VER_AC_INCREMENT);
  987             if (ret) {
  988                 gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
  989                        "failed to store volinfo");
  990                 goto out;
  991             }
  992 
  993             ret = 0;
  994             break;
  995 
  996         case GF_DEFRAG_CMD_STATUS:
  997             break;
  998         default:
  999             break;
 1000     }
 1001 
 1002 out:
 1003     if (ret && op_errstr && msg[0])
 1004         *op_errstr = gf_strdup(msg);
 1005 
 1006     return ret;
 1007 }
 1008 
 1009 int
 1010 glusterd_op_stage_rebalance(dict_t *dict, char **op_errstr)
 1011 {
 1012     char *volname = NULL;
 1013     char *cmd_str = NULL;
 1014     int ret = 0;
 1015     int32_t cmd = 0;
 1016     char msg[2048] = {0};
 1017     glusterd_volinfo_t *volinfo = NULL;
 1018     char *task_id_str = NULL;
 1019     dict_t *op_ctx = NULL;
 1020     xlator_t *this = 0;
 1021 
 1022     this = THIS;
 1023     GF_ASSERT(this);
 1024 
 1025     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 1026     if (ret) {
 1027         gf_msg_debug(this->name, 0, "volname not found");
 1028         goto out;
 1029     }
 1030 
 1031     ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
 1032                           &cmd);
 1033     if (ret) {
 1034         gf_msg_debug(this->name, 0, "cmd not found");
 1035         goto out;
 1036     }
 1037 
 1038     ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
 1039                                           sizeof(msg));
 1040     if (ret) {
 1041         gf_msg_debug(this->name, 0, "failed to validate");
 1042         goto out;
 1043     }
 1044     switch (cmd) {
 1045         case GF_DEFRAG_CMD_START:
 1046         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
 1047             /* Check if the connected clients are all of version
 1048              * glusterfs-3.6 and higher. This is needed to prevent some data
 1049              * loss issues that could occur when older clients are connected
 1050              * when rebalance is run. This check can be bypassed by using
 1051              * 'force'
 1052              */
 1053             ret = glusterd_check_client_op_version_support(
 1054                 volname, GD_OP_VERSION_3_6_0, NULL);
 1055             if (ret) {
 1056                 ret = gf_asprintf(op_errstr,
 1057                                   "Volume %s has one or "
 1058                                   "more connected clients of a version"
 1059                                   " lower than GlusterFS-v3.6.0. "
 1060                                   "Starting rebalance in this state "
 1061                                   "could lead to data loss.\nPlease "
 1062                                   "disconnect those clients before "
 1063                                   "attempting this command again.",
 1064                                   volname);
 1065                 goto out;
 1066             }
 1067             /* Fall through */
 1068         case GF_DEFRAG_CMD_START_FORCE:
 1069             if (is_origin_glusterd(dict)) {
 1070                 op_ctx = glusterd_op_get_ctx();
 1071                 if (!op_ctx) {
 1072                     ret = -1;
 1073                     gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
 1074                            "Failed to get op_ctx");
 1075                     goto out;
 1076                 }
 1077 
 1078                 ret = glusterd_generate_and_set_task_id(
 1079                     op_ctx, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
 1080                 if (ret) {
 1081                     gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
 1082                            "Failed to generate task-id");
 1083                     goto out;
 1084                 }
 1085             } else {
 1086                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
 1087                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
 1088                 if (ret) {
 1089                     snprintf(msg, sizeof(msg), "Missing rebalance-id");
 1090                     gf_msg(this->name, GF_LOG_WARNING, 0,
 1091                            GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
 1092                     ret = 0;
 1093                 }
 1094             }
 1095             ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
 1096                                                  GD_OP_REBALANCE);
 1097             if (ret) {
 1098                 gf_msg_debug(this->name, 0,
 1099                              "defrag start validate "
 1100                              "failed for volume %s.",
 1101                              volinfo->volname);
 1102                 goto out;
 1103             }
 1104             break;
 1105         case GF_DEFRAG_CMD_STATUS:
 1106         case GF_DEFRAG_CMD_STOP:
 1107 
 1108             ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
 1109             if (ret) {
 1110                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
 1111                        "Failed to get "
 1112                        "command string");
 1113                 ret = -1;
 1114                 goto out;
 1115             }
 1116             if ((strstr(cmd_str, "rebalance") != NULL) &&
 1117                 (volinfo->rebal.op != GD_OP_REBALANCE)) {
 1118                 snprintf(msg, sizeof(msg),
 1119                          "Rebalance not started "
 1120                          "for volume %s.",
 1121                          volinfo->volname);
 1122                 ret = -1;
 1123                 goto out;
 1124             }
 1125 
 1126             if (strstr(cmd_str, "remove-brick") != NULL) {
 1127                 if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
 1128                     snprintf(msg, sizeof(msg),
 1129                              "remove-brick not "
 1130                              "started for volume %s.",
 1131                              volinfo->volname);
 1132                     ret = -1;
 1133                     goto out;
 1134                 }
 1135 
 1136                 /* For remove-brick status/stop command check whether
 1137                  * given input brick is part of volume or not.*/
 1138 
 1139                 ret = dict_foreach_fnmatch(dict, "brick*",
 1140                                            glusterd_brick_validation, volinfo);
 1141                 if (ret == -1) {
 1142                     snprintf(msg, sizeof(msg),
 1143                              "Incorrect brick"
 1144                              " for volume %s",
 1145                              volinfo->volname);
 1146                     goto out;
 1147                 }
 1148             }
 1149             break;
 1150 
 1151         default:
 1152             break;
 1153     }
 1154 
 1155     ret = 0;
 1156 out:
 1157     if (ret && op_errstr && msg[0])
 1158         *op_errstr = gf_strdup(msg);
 1159 
 1160     return ret;
 1161 }
 1162 
 1163 int
 1164 glusterd_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 1165 {
 1166     char *volname = NULL;
 1167     int ret = 0;
 1168     int32_t cmd = 0;
 1169     char msg[2048] = {0};
 1170     glusterd_volinfo_t *volinfo = NULL;
 1171     glusterd_brickinfo_t *brickinfo = NULL;
 1172     glusterd_brickinfo_t *tmp = NULL;
 1173     gf_boolean_t volfile_update = _gf_false;
 1174     char *task_id_str = NULL;
 1175     dict_t *ctx = NULL;
 1176     xlator_t *this = NULL;
 1177     uint32_t commit_hash;
 1178     int32_t is_force = 0;
 1179 
 1180     this = THIS;
 1181     GF_ASSERT(this);
 1182 
 1183     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 1184     if (ret) {
 1185         gf_msg_debug(this->name, 0, "volname not given");
 1186         goto out;
 1187     }
 1188 
 1189     ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
 1190                           &cmd);
 1191     if (ret) {
 1192         gf_msg_debug(this->name, 0, "command not given");
 1193         goto out;
 1194     }
 1195 
 1196     ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
 1197                                           sizeof(msg));
 1198     if (ret) {
 1199         gf_msg_debug(this->name, 0, "cmd validate failed");
 1200         goto out;
 1201     }
 1202 
 1203     /* Set task-id, if available, in op_ctx dict for operations other than
 1204      * start
 1205      */
 1206     if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
 1207         if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
 1208             ctx = glusterd_op_get_ctx();
 1209             if (!ctx) {
 1210                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
 1211                        "Failed to get op_ctx");
 1212                 ret = -1;
 1213                 goto out;
 1214             }
 1215 
 1216             if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
 1217                 ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
 1218                                                  ctx, GF_REMOVE_BRICK_TID_KEY,
 1219                                                  SLEN(GF_REMOVE_BRICK_TID_KEY));
 1220             else
 1221                 ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
 1222                                                  ctx, GF_REBALANCE_TID_KEY,
 1223                                                  SLEN(GF_REBALANCE_TID_KEY));
 1224             if (ret) {
 1225                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
 1226                        "Failed to set task-id");
 1227                 goto out;
 1228             }
 1229         }
 1230     }
 1231 
 1232     switch (cmd) {
 1233         case GF_DEFRAG_CMD_START:
 1234         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
 1235         case GF_DEFRAG_CMD_START_FORCE:
 1236 
 1237             ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
 1238             if (ret)
 1239                 is_force = 0;
 1240             if (!is_force) {
 1241                 /* Reset defrag status to 'NOT STARTED' whenever a
 1242                  * remove-brick/rebalance command is issued to remove
 1243                  * stale information from previous run.
 1244                  */
 1245                 volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 1246 
 1247                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
 1248                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
 1249                 if (ret) {
 1250                     gf_msg_debug(this->name, 0,
 1251                                  "Missing rebalance"
 1252                                  " id");
 1253                     ret = 0;
 1254                 } else {
 1255                     gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
 1256                     volinfo->rebal.op = GD_OP_REBALANCE;
 1257                 }
 1258                 if (!gd_should_i_start_rebalance(volinfo)) {
 1259                     /* Store the rebalance-id and rebalance command
 1260                      * even if the peer isn't starting a rebalance
 1261                      * process. On peers where a rebalance process
 1262                      * is started, glusterd_handle_defrag_start
 1263                      * performs the storing.
 1264                      * Storing this is needed for having
 1265                      * 'volume status' work correctly.
 1266                      */
 1267                     glusterd_store_perform_node_state_store(volinfo);
 1268                     break;
 1269                 }
 1270                 if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
 1271                     volinfo->rebal.commit_hash = commit_hash;
 1272                 }
 1273                 ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
 1274                                                    cmd, NULL, GD_OP_REBALANCE);
 1275                 break;
 1276             } else {
 1277                 /* Reset defrag status to 'STARTED' so that the
 1278                  * pid is checked and restarted accordingly.
 1279                  * If the pid is not running it executes the
 1280                  * "NOT_STARTED" case and restarts the process
 1281                  */
 1282                 volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
 1283                 volinfo->rebal.defrag_cmd = cmd;
 1284                 volinfo->rebal.op = GD_OP_REBALANCE;
 1285 
 1286                 ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
 1287                                     SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
 1288                 if (ret) {
 1289                     gf_msg_debug(this->name, 0,
 1290                                  "Missing rebalance"
 1291                                  " id");
 1292                     ret = 0;
 1293                 } else {
 1294                     gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
 1295                     volinfo->rebal.op = GD_OP_REBALANCE;
 1296                 }
 1297                 if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
 1298                     volinfo->rebal.commit_hash = commit_hash;
 1299                 }
 1300                 ret = glusterd_restart_rebalance_for_volume(volinfo);
 1301                 break;
 1302             }
 1303         case GF_DEFRAG_CMD_STOP:
 1304             /* Clear task-id only on explicitly stopping rebalance.
 1305              * Also clear the stored operation, so it doesn't cause trouble
 1306              * with future rebalance/remove-brick starts
 1307              */
 1308             gf_uuid_clear(volinfo->rebal.rebalance_id);
 1309             volinfo->rebal.op = GD_OP_NONE;
 1310 
 1311             /* Fall back to the old volume file in case of decommission*/
 1312             cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
 1313                                          brick_list)
 1314             {
 1315                 if (!brickinfo->decommissioned)
 1316                     continue;
 1317                 brickinfo->decommissioned = 0;
 1318                 volfile_update = _gf_true;
 1319             }
 1320 
 1321             if (volfile_update == _gf_false) {
 1322                 ret = 0;
 1323                 break;
 1324             }
 1325 
 1326             ret = glusterd_create_volfiles_and_notify_services(volinfo);
 1327             if (ret) {
 1328                 gf_msg(this->name, GF_LOG_WARNING, 0,
 1329                        GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
 1330                 goto out;
 1331             }
 1332 
 1333             ret = glusterd_store_volinfo(volinfo,
 1334                                          GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 1335             if (ret) {
 1336                 gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
 1337                        "failed to store volinfo");
 1338                 goto out;
 1339             }
 1340 
 1341             ret = 0;
 1342             break;
 1343 
 1344         case GF_DEFRAG_CMD_STATUS:
 1345             break;
 1346         default:
 1347             break;
 1348     }
 1349 
 1350 out:
 1351     if (ret && op_errstr && msg[0])
 1352         *op_errstr = gf_strdup(msg);
 1353 
 1354     return ret;
 1355 }
 1356 
 1357 int32_t
 1358 glusterd_defrag_event_notify_handle(dict_t *dict)
 1359 {
 1360     glusterd_volinfo_t *volinfo = NULL;
 1361     char *volname = NULL;
 1362     char *volname_ptr = NULL;
 1363     int32_t ret = -1;
 1364     xlator_t *this = NULL;
 1365 
 1366     this = THIS;
 1367     GF_ASSERT(this);
 1368     GF_ASSERT(dict);
 1369 
 1370     ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 1371     if (ret) {
 1372         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
 1373                "Failed to get volname");
 1374         return ret;
 1375     }
 1376 
 1377     volname_ptr = strstr(volname, "rebalance/");
 1378     if (volname_ptr) {
 1379         volname_ptr = strchr(volname_ptr, '/');
 1380         volname = volname_ptr + 1;
 1381     } else {
 1382         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
 1383                "volname received (%s) is not prefixed with rebalance.",
 1384                volname);
 1385         ret = -1;
 1386         goto out;
 1387     }
 1388 
 1389     ret = glusterd_volinfo_find(volname, &volinfo);
 1390     if (ret) {
 1391         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
 1392                "Failed to get volinfo for %s", volname);
 1393         return ret;
 1394     }
 1395 
 1396     ret = glusterd_defrag_volume_status_update(volinfo, dict, 0);
 1397 
 1398     if (ret) {
 1399         gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
 1400                "Failed to update status");
 1401         gf_event(EVENT_REBALANCE_STATUS_UPDATE_FAILED, "volume=%s",
 1402                  volinfo->volname);
 1403     }
 1404 
 1405 out:
 1406     return ret;
 1407 }