"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/xlators/mgmt/glusterd/src/glusterd-server-quorum.c" (16 Sep 2020, 13682 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "glusterd-server-quorum.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2    Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
    3    This file is part of GlusterFS.
    4 
    5    This file is licensed to you under your choice of the GNU Lesser
    6    General Public License, version 3 or any later version (LGPLv3 or
    7    later), or the GNU General Public License, version 2 (GPLv2), in all
    8    cases as published by the Free Software Foundation.
    9 */
   10 #include <glusterfs/common-utils.h>
   11 #include "glusterd.h"
   12 #include "glusterd-utils.h"
   13 #include "glusterd-messages.h"
   14 #include "glusterd-server-quorum.h"
   15 #include "glusterd-store.h"
   16 #include "glusterd-syncop.h"
   17 #include "glusterd-op-sm.h"
   18 
   19 #define CEILING_POS(X) (((X) - (int)(X)) > 0 ? (int)((X) + 1) : (int)(X))
   20 
   21 static gf_boolean_t
   22 glusterd_is_get_op(xlator_t *this, glusterd_op_t op, dict_t *dict)
   23 {
   24     char *key = NULL;
   25     char *volname = NULL;
   26     int ret = 0;
   27 
   28     if (op == GD_OP_STATUS_VOLUME)
   29         return _gf_true;
   30 
   31     if (op == GD_OP_SET_VOLUME) {
   32         /*check for set volume help*/
   33         ret = dict_get_str(dict, "volname", &volname);
   34         if (volname && ((strcmp(volname, "help") == 0) ||
   35                         (strcmp(volname, "help-xml") == 0))) {
   36             ret = dict_get_str(dict, "key1", &key);
   37             if (ret < 0)
   38                 return _gf_true;
   39         }
   40     }
   41     return _gf_false;
   42 }
   43 
   44 gf_boolean_t
   45 glusterd_is_quorum_validation_required(xlator_t *this, glusterd_op_t op,
   46                                        dict_t *dict)
   47 {
   48     gf_boolean_t required = _gf_true;
   49     char *key = NULL;
   50     char *key_fixed = NULL;
   51     int ret = -1;
   52 
   53     if (glusterd_is_get_op(this, op, dict)) {
   54         required = _gf_false;
   55         goto out;
   56     }
   57     if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
   58         goto out;
   59     if (op == GD_OP_SET_VOLUME)
   60         ret = dict_get_str(dict, "key1", &key);
   61     else if (op == GD_OP_RESET_VOLUME)
   62         ret = dict_get_str(dict, "key", &key);
   63     if (ret)
   64         goto out;
   65     ret = glusterd_check_option_exists(key, &key_fixed);
   66     if (ret <= 0)
   67         goto out;
   68     if (key_fixed)
   69         key = key_fixed;
   70     if (glusterd_is_quorum_option(key))
   71         required = _gf_false;
   72 out:
   73     GF_FREE(key_fixed);
   74     return required;
   75 }
   76 
   77 int
   78 glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
   79                          char **op_errstr)
   80 {
   81     int ret = 0;
   82     char *volname = NULL;
   83     glusterd_volinfo_t *volinfo = NULL;
   84     char *errstr = NULL;
   85 
   86     errstr = "Quorum not met. Volume operation not allowed.";
   87     if (!glusterd_is_quorum_validation_required(this, op, dict))
   88         goto out;
   89 
   90     ret = dict_get_str(dict, "volname", &volname);
   91     if (ret) {
   92         ret = 0;
   93         goto out;
   94     }
   95 
   96     ret = glusterd_volinfo_find(volname, &volinfo);
   97     if (ret) {
   98         ret = 0;
   99         goto out;
  100     }
  101 
  102     if (!glusterd_is_volume_in_server_quorum(volinfo)) {
  103         ret = 0;
  104         goto out;
  105     }
  106 
  107     if (does_gd_meet_server_quorum(this)) {
  108         ret = 0;
  109         goto out;
  110     }
  111 
  112     ret = -1;
  113     *op_errstr = gf_strdup(errstr);
  114 
  115 out:
  116     return ret;
  117 }
  118 
  119 gf_boolean_t
  120 glusterd_is_quorum_option(char *option)
  121 {
  122     gf_boolean_t res = _gf_false;
  123     int i = 0;
  124     static const char *const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
  125                                        GLUSTERD_QUORUM_RATIO_KEY, NULL};
  126 
  127     for (i = 0; keys[i]; i++) {
  128         if (strcmp(option, keys[i]) == 0) {
  129             res = _gf_true;
  130             break;
  131         }
  132     }
  133     return res;
  134 }
  135 
  136 gf_boolean_t
  137 glusterd_is_quorum_changed(dict_t *options, char *option, char *value)
  138 {
  139     int ret = 0;
  140     gf_boolean_t reconfigured = _gf_false;
  141     gf_boolean_t all = _gf_false;
  142     char *oldquorum = NULL;
  143     char *newquorum = NULL;
  144     char *oldratio = NULL;
  145     char *newratio = NULL;
  146     xlator_t *this = NULL;
  147 
  148     this = THIS;
  149 
  150     if ((strcmp("all", option) != 0) && !glusterd_is_quorum_option(option))
  151         goto out;
  152 
  153     if (strcmp("all", option) == 0)
  154         all = _gf_true;
  155 
  156     if (all || (strcmp(GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
  157         newquorum = value;
  158         ret = dict_get_str(options, GLUSTERD_QUORUM_TYPE_KEY, &oldquorum);
  159         if (ret)
  160             gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
  161                    "dict_get_str failed on %s", GLUSTERD_QUORUM_TYPE_KEY);
  162     }
  163 
  164     if (all || (strcmp(GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
  165         newratio = value;
  166         ret = dict_get_str(options, GLUSTERD_QUORUM_RATIO_KEY, &oldratio);
  167         if (ret)
  168             gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
  169                    "dict_get_str failed on %s", GLUSTERD_QUORUM_RATIO_KEY);
  170     }
  171 
  172     reconfigured = _gf_true;
  173 
  174     if (oldquorum && newquorum && (strcmp(oldquorum, newquorum) == 0))
  175         reconfigured = _gf_false;
  176     if (oldratio && newratio && (strcmp(oldratio, newratio) == 0))
  177         reconfigured = _gf_false;
  178 
  179     if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
  180         (newquorum == NULL))
  181         reconfigured = _gf_false;
  182 out:
  183     return reconfigured;
  184 }
  185 
  186 static gf_boolean_t
  187 _is_contributing_to_quorum(gd_quorum_contrib_t contrib)
  188 {
  189     if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
  190         return _gf_true;
  191     return _gf_false;
  192 }
  193 
  194 gf_boolean_t
  195 does_quorum_meet(int active_count, int quorum_count)
  196 {
  197     return (active_count >= quorum_count);
  198 }
  199 
  200 int
  201 glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
  202                                    int *quorum_count)
  203 {
  204     glusterd_peerinfo_t *peerinfo = NULL;
  205     glusterd_conf_t *conf = NULL;
  206     int ret = -1;
  207     int inquorum_count = 0;
  208     char *val = NULL;
  209     double quorum_percentage = 0.0;
  210     gf_boolean_t ratio = _gf_false;
  211     int count = 0;
  212 
  213     conf = this->private;
  214 
  215     /* Start with counting self */
  216     inquorum_count = 1;
  217     if (active_count)
  218         *active_count = 1;
  219 
  220     RCU_READ_LOCK;
  221     cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
  222     {
  223         if (_is_contributing_to_quorum(peerinfo->quorum_contrib))
  224             inquorum_count = inquorum_count + 1;
  225         if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
  226             *active_count = *active_count + 1;
  227     }
  228     RCU_READ_UNLOCK;
  229 
  230     ret = dict_get_str(conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
  231     if (ret == 0) {
  232         ret = gf_string2percent(val, &quorum_percentage);
  233         if (ret == 0)
  234             ratio = _gf_true;
  235     }
  236     if (ratio)
  237         count = CEILING_POS(inquorum_count * quorum_percentage / 100.0);
  238     else
  239         count = (inquorum_count * 50 / 100) + 1;
  240 
  241     *quorum_count = count;
  242     ret = 0;
  243 
  244     return ret;
  245 }
  246 
  247 gf_boolean_t
  248 glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo)
  249 {
  250     gf_boolean_t res = _gf_false;
  251     char *quorum_type = NULL;
  252     int ret = 0;
  253 
  254     ret = dict_get_str(volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY, &quorum_type);
  255     if (ret)
  256         goto out;
  257 
  258     if (strcmp(quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
  259         res = _gf_true;
  260 out:
  261     return res;
  262 }
  263 
  264 gf_boolean_t
  265 glusterd_is_any_volume_in_server_quorum(xlator_t *this)
  266 {
  267     glusterd_conf_t *conf = NULL;
  268     glusterd_volinfo_t *volinfo = NULL;
  269 
  270     conf = this->private;
  271     list_for_each_entry(volinfo, &conf->volumes, vol_list)
  272     {
  273         if (glusterd_is_volume_in_server_quorum(volinfo)) {
  274             return _gf_true;
  275         }
  276     }
  277     return _gf_false;
  278 }
  279 
  280 gf_boolean_t
  281 does_gd_meet_server_quorum(xlator_t *this)
  282 {
  283     int quorum_count = 0;
  284     int active_count = 0;
  285     gf_boolean_t in = _gf_false;
  286     int ret = -1;
  287 
  288     ret = glusterd_get_quorum_cluster_counts(this, &active_count,
  289                                              &quorum_count);
  290     if (ret)
  291         goto out;
  292 
  293     if (!does_quorum_meet(active_count, quorum_count)) {
  294         goto out;
  295     }
  296 
  297     in = _gf_true;
  298 out:
  299     return in;
  300 }
  301 
  302 void
  303 glusterd_do_volume_quorum_action(xlator_t *this, glusterd_volinfo_t *volinfo,
  304                                  gf_boolean_t meets_quorum)
  305 {
  306     int ret = -1;
  307     glusterd_brickinfo_t *brickinfo = NULL;
  308     gd_quorum_status_t quorum_status = NOT_APPLICABLE_QUORUM;
  309     gf_boolean_t follows_quorum = _gf_false;
  310     gf_boolean_t quorum_status_unchanged = _gf_false;
  311 
  312     if (volinfo->status != GLUSTERD_STATUS_STARTED) {
  313         volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
  314         goto out;
  315     }
  316 
  317     follows_quorum = glusterd_is_volume_in_server_quorum(volinfo);
  318     if (follows_quorum) {
  319         if (meets_quorum)
  320             quorum_status = MEETS_QUORUM;
  321         else
  322             quorum_status = DOESNT_MEET_QUORUM;
  323     } else {
  324         quorum_status = NOT_APPLICABLE_QUORUM;
  325     }
  326 
  327     /*
  328      * The following check is added to prevent spurious brick starts when
  329      * events occur that affect quorum.
  330      * Example:
  331      * There is a cluster of 10 peers. Volume is in quorum. User
  332      * takes down one brick from the volume to perform maintenance.
  333      * Suddenly one of the peers go down. Cluster is still in quorum. But
  334      * because of this 'peer going down' event, quorum is calculated and
  335      * the bricks that are down are brought up again. In this process it
  336      * also brings up the brick that is purposefully taken down.
  337      */
  338     if (volinfo->quorum_status == quorum_status) {
  339         quorum_status_unchanged = _gf_true;
  340         goto out;
  341     }
  342 
  343     if (quorum_status == MEETS_QUORUM) {
  344         gf_msg(this->name, GF_LOG_CRITICAL, 0,
  345                GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
  346                "Server quorum regained for volume %s. Starting local "
  347                "bricks.",
  348                volinfo->volname);
  349         gf_event(EVENT_QUORUM_REGAINED, "volume=%s", volinfo->volname);
  350     } else if (quorum_status == DOESNT_MEET_QUORUM) {
  351         gf_msg(this->name, GF_LOG_CRITICAL, 0,
  352                GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
  353                "Server quorum lost for volume %s. Stopping local "
  354                "bricks.",
  355                volinfo->volname);
  356         gf_event(EVENT_QUORUM_LOST, "volume=%s", volinfo->volname);
  357     }
  358 
  359     list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
  360     {
  361         if (!glusterd_is_local_brick(this, volinfo, brickinfo))
  362             continue;
  363         if (quorum_status == DOESNT_MEET_QUORUM) {
  364             ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
  365             if (ret) {
  366                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
  367                        "Failed to "
  368                        "stop brick %s:%s",
  369                        brickinfo->hostname, brickinfo->path);
  370             }
  371         } else {
  372             if (!brickinfo->start_triggered) {
  373                 pthread_mutex_lock(&brickinfo->restart_mutex);
  374                 {
  375                     /* coverity[SLEEP] */
  376                     ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
  377                                                _gf_false);
  378                 }
  379                 pthread_mutex_unlock(&brickinfo->restart_mutex);
  380                 if (ret) {
  381                     gf_msg(this->name, GF_LOG_ERROR, 0,
  382                            GD_MSG_BRICK_DISCONNECTED, "Failed to start %s:%s",
  383                            brickinfo->hostname, brickinfo->path);
  384                 }
  385             }
  386         }
  387     }
  388     volinfo->quorum_status = quorum_status;
  389     if (quorum_status == MEETS_QUORUM) {
  390         /* bricks might have been restarted and so as the port change
  391          * might have happened
  392          */
  393         ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
  394         if (ret) {
  395             gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
  396                    "Failed to write volinfo for volume %s", volinfo->volname);
  397             goto out;
  398         }
  399     }
  400 out:
  401     if (quorum_status_unchanged) {
  402         list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
  403         {
  404             if (!glusterd_is_local_brick(this, volinfo, brickinfo))
  405                 continue;
  406             ret = glusterd_brick_start(volinfo, brickinfo, _gf_false, _gf_true);
  407             if (ret) {
  408                 gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
  409                        "Failed to "
  410                        "connect to %s:%s",
  411                        brickinfo->hostname, brickinfo->path);
  412             }
  413         }
  414     }
  415     return;
  416 }
  417 
  418 int
  419 glusterd_do_quorum_action()
  420 {
  421     xlator_t *this = NULL;
  422     glusterd_conf_t *conf = NULL;
  423     glusterd_volinfo_t *volinfo = NULL;
  424     int ret = 0;
  425     int active_count = 0;
  426     int quorum_count = 0;
  427     gf_boolean_t meets = _gf_false;
  428 
  429     this = THIS;
  430     conf = this->private;
  431 
  432     conf->pending_quorum_action = _gf_true;
  433     ret = glusterd_lock(conf->uuid);
  434     if (ret)
  435         goto out;
  436 
  437     {
  438         ret = glusterd_get_quorum_cluster_counts(this, &active_count,
  439                                                  &quorum_count);
  440         if (ret)
  441             goto unlock;
  442 
  443         if (does_quorum_meet(active_count, quorum_count))
  444             meets = _gf_true;
  445         list_for_each_entry(volinfo, &conf->volumes, vol_list)
  446         {
  447             glusterd_do_volume_quorum_action(this, volinfo, meets);
  448         }
  449     }
  450 unlock:
  451     (void)glusterd_unlock(conf->uuid);
  452     conf->pending_quorum_action = _gf_false;
  453 out:
  454     return ret;
  455 }
  456 
  457 /* ret = 0 represents quorum is not met
  458  * ret = 1 represents quorum is met
  459  * ret = 2 represents quorum not applicable
  460  */
  461 
  462 int
  463 check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
  464                              gf_boolean_t node_quorum)
  465 {
  466     gf_boolean_t volume_quorum = _gf_false;
  467     int ret = 0;
  468 
  469     volume_quorum = glusterd_is_volume_in_server_quorum(volinfo);
  470     if (volume_quorum) {
  471         if (node_quorum)
  472             ret = 1;
  473     } else {
  474         ret = 2;
  475     }
  476     return ret;
  477 }