"Fossies" - the Fresh Open Source Software Archive

Member "monasca-api-4.0.0/devstack/lib/monasca-log.sh" (13 May 2020, 28310 Bytes) of package /linux/misc/openstack/monasca-api-4.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Bash source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "monasca-log.sh": 4.0.0_vs_5.0.0.

    1 #!/bin/bash
    2 
    3 #
    4 # Copyright 2016-2017 FUJITSU LIMITED
    5 #
    6 # Licensed under the Apache License, Version 2.0 (the "License");
    7 # you may not use this file except in compliance with the License.
    8 # You may obtain a copy of the License at
    9 #
   10 #    http://www.apache.org/licenses/LICENSE-2.0
   11 #
   12 # Unless required by applicable law or agreed to in writing, software
   13 # distributed under the License is distributed on an "AS IS" BASIS,
   14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
   15 # implied.
   16 # See the License for the specific language governing permissions and
   17 # limitations under the License.
   18 #
   19 
   20 _XTRACE_MON_LOG=$(set +o | grep xtrace)
   21 set +o xtrace
   22 
   23 _ERREXIT_MON_LOG=$(set +o | grep errexit)
   24 set -o errexit
   25 
   26 # configuration bits of various services
   27 LOG_PERSISTER_DIR=$DEST/monasca-log-persister
   28 LOG_TRANSFORMER_DIR=$DEST/monasca-log-transformer
   29 LOG_METRICS_DIR=$DEST/monasca-log-metrics
   30 LOG_AGENT_DIR=$DEST/monasca-log-agent
   31 
   32 ELASTICSEARCH_DIR=$DEST/elasticsearch
   33 ELASTICSEARCH_CFG_DIR=$ELASTICSEARCH_DIR/config
   34 ELASTICSEARCH_LOG_DIR=$LOGDIR/elasticsearch
   35 ELASTICSEARCH_DATA_DIR=$DATA_DIR/elasticsearch
   36 
   37 KIBANA_DIR=$DEST/kibana
   38 KIBANA_CFG_DIR=$KIBANA_DIR/config
   39 KIBANA_DEV_DIR=$DEST/kibana_dev
   40 KIBANA_DEV_NODE_JS_VERSION=${KIBANA_DEV_NODE_JS_VERSION:-"10.15.2"}
   41 
   42 LOGSTASH_DIR=$DEST/logstash
   43 LOGSTASH_DATA_DIR=$DEST/logstash-data
   44 
   45 ES_SERVICE_BIND_HOST=${ES_SERVICE_BIND_HOST:-${SERVICE_HOST}}
   46 ES_SERVICE_BIND_PORT=${ES_SERVICE_BIND_PORT:-9200}
   47 ES_SERVICE_PUBLISH_HOST=${ES_SERVICE_PUBLISH_HOST:-${SERVICE_HOST}}
   48 ES_SERVICE_PUBLISH_PORT=${ES_SERVICE_PUBLISH_PORT:-9300}
   49 
   50 KIBANA_SERVICE_HOST=${KIBANA_SERVICE_HOST:-${SERVICE_HOST}}
   51 KIBANA_SERVICE_PORT=${KIBANA_SERVICE_PORT:-5601}
   52 KIBANA_SERVER_BASE_PATH=${KIBANA_SERVER_BASE_PATH:-"/dashboard/monitoring/logs_proxy"}
   53 
   54 # Settings needed for Elasticsearch
   55 # Elasticsearch uses a lot of file descriptors or file handles.
   56 # Increase the limit on the number of open files descriptors for the user running Elasticsearch to 65,536 or higher.
   57 LIMIT_NOFILE=${LIMIT_NOFILE:-65535}
   58 # Elasticsearch uses a mmapfs directory by default to store its indices.
   59 # The default operating system limits on mmap counts is likely to be too low,
   60 # which may result in out of memory exceptions, increase to at least 262144.
   61 VM_MAX_MAP_COUNT=${VM_MAX_MAP_COUNT:-262144}
   62 
   63 MONASCA_LOG_API_BASE_URI=https://${MONASCA_API_BASE_URI}/logs
   64 
   65 
   66 run_process_sleep() {
   67     local name=$1
   68     local cmd=$2
   69     local sleepTime=${3:-1}
   70     run_process "$name" "$cmd"
   71     sleep ${sleepTime}
   72 }
   73 
   74 is_logstash_required() {
   75     is_service_enabled monasca-log-persister \
   76         || is_service_enabled monasca-log-transformer \
   77         || is_service_enabled monasca-log-metrics \
   78         || is_service_enabled monasca-log-agent \
   79         && return 0
   80 }
   81 
   82 # TOP_LEVEL functions called from devstack coordinator
   83 ###############################################################################
   84 function pre_install_logs_services {
   85     install_elk
   86     install_nodejs
   87     install_gate_config_holder
   88 }
   89 
   90 function install_monasca_log {
   91     configure_nvm
   92     configure_yarn
   93     build_kibana_plugin
   94     install_log_agent
   95     if $USE_OLD_LOG_API = true; then
   96         install_old_log_api
   97     fi
   98 }
   99 
  100 function install_elk {
  101     install_logstash
  102     install_elasticsearch
  103     install_kibana
  104 }
  105 
  106 function install_gate_config_holder {
  107     sudo install -d -o $STACK_USER $GATE_CONFIGURATION_DIR
  108 }
  109 
  110 function install_monasca_statsd {
  111     if use_library_from_git "monasca-statsd"; then
  112         git_clone_by_name "monasca-statsd"
  113         setup_dev_lib "monasca-statsd"
  114     fi
  115 }
  116 
  117 function configure_monasca_log {
  118     configure_kafka
  119     configure_elasticsearch
  120     configure_kibana
  121     install_kibana_plugin
  122     if $USE_OLD_LOG_API = true; then
  123         configure_old_monasca_log_api
  124     fi
  125     configure_monasca_log_api
  126     configure_monasca_log_transformer
  127     configure_monasca_log_metrics
  128     configure_monasca_log_persister
  129     configure_monasca_log_agent
  130 
  131 
  132 }
  133 
  134 function init_monasca_log {
  135     enable_log_management
  136 }
  137 
  138 function init_monasca_grafana_dashboards {
  139     if is_service_enabled horizon; then
  140         echo_summary "Init Grafana dashboards"
  141 
  142         sudo python "${PLUGIN_FILES}"/grafana/grafana.py "${PLUGIN_FILES}"/grafana/dashboards.d
  143     fi
  144 }
  145 
  146 function install_old_log_api {
  147 
  148     if python3_enabled; then
  149             enable_python3_package monasca-log-api
  150     fi
  151 
  152     echo_summary "Installing monasca-log-api"
  153 
  154     git_clone $MONASCA_LOG_API_REPO $MONASCA_LOG_API_DIR $MONASCA_LOG_API_BRANCH
  155     setup_develop $MONASCA_LOG_API_DIR
  156 
  157     install_keystonemiddleware
  158     install_monasca_statsd
  159 
  160     if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then
  161         install_apache_wsgi
  162     elif [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then
  163         pip_install uwsgi
  164     else
  165         pip_install gunicorn
  166     fi
  167 
  168     if [ "$MONASCA_LOG_API_DEPLOY" != "gunicorn" ]; then
  169         if is_ssl_enabled_service "monasca-log-api"; then
  170             enable_mod_ssl
  171         fi
  172     fi
  173 
  174 }
  175 
  176 
  177 function configure_old_monasca_log_api {
  178     MONASCA_LOG_API_BIN_DIR=$(get_python_exec_prefix)
  179     MONASCA_LOG_API_WSGI=$MONASCA_LOG_API_BIN_DIR/monasca-log-api-wsgi
  180 
  181     if is_service_enabled monasca-log-api; then
  182         echo_summary "Configuring monasca-log-api"
  183         rm -rf $MONASCA_LOG_API_UWSGI_CONF
  184         install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-uwsgi.ini $MONASCA_LOG_API_UWSGI_CONF
  185 
  186         write_uwsgi_config "$MONASCA_LOG_API_UWSGI_CONF" "$MONASCA_LOG_API_WSGI" "/logs"
  187 
  188     fi
  189 }
  190 
  191 function configure_old_monasca_log_api_core {
  192     # Put config files in ``$MONASCA_LOG_API_CONF_DIR`` for everyone to find
  193     sudo install -d -o $STACK_USER $MONASCA_LOG_API_CONF_DIR
  194     sudo install -m 700 -d -o $STACK_USER $MONASCA_LOG_API_CACHE_DIR
  195     sudo install -d -o $STACK_USER $MONASCA_LOG_API_LOG_DIR
  196 
  197     # ensure fresh installation of configuration files
  198     rm -rf $MONASCA_LOG_API_CONF $MONASCA_LOG_API_PASTE $MONASCA_LOG_API_LOGGING_CONF
  199 
  200     $MONASCA_LOG_API_BIN_DIR/oslo-config-generator \
  201         --config-file $MONASCA_LOG_API_DIR/config-generator/monasca-log-api.conf \
  202         --output-file /tmp/monasca-log-api.conf
  203 
  204     install -m 600 /tmp/monasca-log-api.conf $MONASCA_LOG_API_CONF && rm -rf /tmp/monasca-log-api.conf
  205     install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE
  206     install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF
  207 
  208     # configure monasca-log-api.conf
  209     iniset "$MONASCA_LOG_API_CONF" DEFAULT log_config_append $MONASCA_LOG_API_LOGGING_CONF
  210     iniset "$MONASCA_LOG_API_CONF" service region $REGION_NAME
  211 
  212     iniset "$MONASCA_LOG_API_CONF" log_publisher kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT
  213     iniset "$MONASCA_LOG_API_CONF" log_publisher topics log
  214 
  215     iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT
  216     iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_topics log
  217 
  218     iniset "$MONASCA_LOG_API_CONF" roles_middleware path "/v2.0/log"
  219     iniset "$MONASCA_LOG_API_CONF" roles_middleware default_roles monasca-user
  220     iniset "$MONASCA_LOG_API_CONF" roles_middleware agent_roles monasca-agent
  221     iniset "$MONASCA_LOG_API_CONF" roles_middleware delegate_roles admin
  222 
  223     # configure keystone middleware
  224     configure_auth_token_middleware "$MONASCA_LOG_API_CONF" "admin" $MONASCA_LOG_API_CACHE_DIR
  225     iniset "$MONASCA_LOG_API_CONF" keystone_authtoken region_name $REGION_NAME
  226     iniset "$MONASCA_LOG_API_CONF" keystone_authtoken project_name "admin"
  227     iniset "$MONASCA_LOG_API_CONF" keystone_authtoken password $ADMIN_PASSWORD
  228 
  229     # insecure
  230     if is_service_enabled tls-proxy; then
  231         iniset "$MONASCA_LOG_API_CONF" keystone_authtoken insecure False
  232     fi
  233 
  234     # configure log-api-paste.ini
  235     iniset "$MONASCA_LOG_API_PASTE" server:main bind $MONASCA_LOG_API_SERVICE_HOST:$MONASCA_LOG_API_SERVICE_PORT
  236     iniset "$MONASCA_LOG_API_PASTE" server:main chdir $MONASCA_LOG_API_DIR
  237     iniset "$MONASCA_LOG_API_PASTE" server:main workers $API_WORKERS
  238 }
  239 
  240 function init_agent {
  241     echo_summary "Init Monasca agent"
  242 
  243     sudo cp -f "${PLUGIN_FILES}"/monasca-agent/http_check.yaml /etc/monasca/agent/conf.d/http_check.yaml
  244     sudo cp -f "${PLUGIN_FILES}"/monasca-agent/process.yaml /etc/monasca/agent/conf.d/process.yaml
  245     sudo cp -f "${PLUGIN_FILES}"/monasca-agent/elastic.yaml /etc/monasca/agent/conf.d/elastic.yaml
  246 
  247     sudo sed -i "s/{{IP}}/$(ip -o -4 addr list eth1 | awk '{print $4}' | cut -d/ -f1 | head -1)/" /etc/monasca/agent/conf.d/*.yaml
  248     sudo sed -i "s/127\.0\.0\.1/$(hostname)/" /etc/monasca/agent/conf.d/*.yaml
  249     sudo systemctl restart monasca-collector
  250 }
  251 
  252 function stop_monasca_log {
  253     stop_process "monasca-log-agent" || true
  254     stop_monasca_log_api
  255     stop_process "monasca-log-metrics" || true
  256     stop_process "monasca-log-persister" || true
  257     stop_process "monasca-log-transformer" || true
  258     stop_process "kibana" || true
  259     stop_process "elasticsearch" || true
  260 }
  261 
  262 function start_monasca_log {
  263     start_elasticsearch
  264     start_kibana
  265     start_monasca_log_transformer
  266     start_monasca_log_metrics
  267     start_monasca_log_persister
  268     if $USE_OLD_LOG_API = true; then
  269         start_monasca_log_api
  270     fi
  271     start_monasca_log_agent
  272 }
  273 
  274 function clean_monasca_log {
  275     clean_monasca_log_agent
  276     clean_monasca_log_api
  277     clean_monasca_log_persister
  278     clean_monasca_log_transformer
  279     clean_kibana
  280     clean_elasticsearch
  281     clean_logstash
  282     clean_nodejs
  283     clean_nvm
  284     clean_yarn
  285     clean_gate_config_holder
  286 }
  287 ###############################################################################
  288 
  289 function configure_monasca_log_api {
  290     if is_service_enabled monasca-log; then
  291         echo_summary "Configuring monasca-api"
  292         iniset "$MONASCA_API_CONF" DEFAULT enable_logs_api "true"
  293         iniset "$MONASCA_API_CONF" kafka logs_topics "log"
  294 
  295         create_log_management_accounts
  296     fi
  297 }
  298 
  299 function install_logstash {
  300     if is_logstash_required; then
  301         echo_summary "Installing Logstash ${LOGSTASH_VERSION}"
  302 
  303         local logstash_tarball=logstash-oss-${LOGSTASH_VERSION}.tar.gz
  304         local logstash_url=https://artifacts.elastic.co/downloads/logstash/${logstash_tarball}
  305 
  306         local logstash_dest
  307         logstash_dest=`get_extra_file ${logstash_url}`
  308 
  309         tar xzf ${logstash_dest} -C $DEST
  310 
  311         sudo chown -R $STACK_USER $DEST/logstash-${LOGSTASH_VERSION}
  312         ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR
  313 
  314         sudo mkdir -p $LOGSTASH_DATA_DIR
  315         sudo chown $STACK_USER:monasca $LOGSTASH_DATA_DIR
  316     fi
  317 }
  318 
  319 function clean_logstash {
  320     if is_logstash_required; then
  321         echo_summary "Cleaning Logstash ${LOGSTASH_VERSION}"
  322 
  323         sudo rm -rf $LOGSTASH_DIR || true
  324         sudo rm -rf $FILES/logstash-${LOGSTASH_VERSION}.tar.gz ||  true
  325         sudo rm -rf $DEST/logstash-${LOGSTASH_VERSION} || true
  326     fi
  327 }
  328 
  329 function install_elasticsearch {
  330     if is_service_enabled elasticsearch; then
  331         echo_summary "Installing ElasticSearch ${ELASTICSEARCH_VERSION}"
  332 
  333         local es_tarball=elasticsearch-oss-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz
  334         local es_url=https://artifacts.elastic.co/downloads/elasticsearch/${es_tarball}
  335 
  336         local es_dest
  337         es_dest=`get_extra_file ${es_url}`
  338 
  339         tar xzf ${es_dest} -C $DEST
  340 
  341         sudo chown -R $STACK_USER $DEST/elasticsearch-${ELASTICSEARCH_VERSION}
  342         ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR
  343     fi
  344 }
  345 
  346 function configure_elasticsearch {
  347     if is_service_enabled elasticsearch; then
  348         echo_summary "Configuring ElasticSearch ${ELASTICSEARCH_VERSION}"
  349 
  350         local templateDir=$ELASTICSEARCH_CFG_DIR/templates
  351 
  352         for dir in $ELASTICSEARCH_LOG_DIR $templateDir $ELASTICSEARCH_DATA_DIR; do
  353             sudo install -m 755 -d -o $STACK_USER $dir
  354         done
  355 
  356         sudo cp -f "${PLUGIN_FILES}"/elasticsearch/elasticsearch.yml $ELASTICSEARCH_CFG_DIR/elasticsearch.yml
  357         sudo chown -R $STACK_USER $ELASTICSEARCH_CFG_DIR/elasticsearch.yml
  358         sudo chmod 0644 $ELASTICSEARCH_CFG_DIR/elasticsearch.yml
  359 
  360         sudo sed -e "
  361             s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g;
  362             s|%ES_SERVICE_BIND_PORT%|$ES_SERVICE_BIND_PORT|g;
  363             s|%ES_DATA_DIR%|$ELASTICSEARCH_DATA_DIR|g;
  364             s|%ES_LOG_DIR%|$ELASTICSEARCH_LOG_DIR|g;
  365         " -i $ELASTICSEARCH_CFG_DIR/elasticsearch.yml
  366 
  367         ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml
  368 
  369         echo "[Service]" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null
  370         echo "LimitNOFILE=$LIMIT_NOFILE" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null
  371 
  372         echo "vm.max_map_count=$VM_MAX_MAP_COUNT" | sudo tee --append /etc/sysctl.conf > /dev/null
  373         sudo sysctl -w vm.max_map_count=$VM_MAX_MAP_COUNT
  374     fi
  375 }
  376 
  377 function clean_elasticsearch {
  378     if is_service_enabled elasticsearch; then
  379         echo_summary "Cleaning Elasticsearch ${ELASTICSEARCH_VERSION}"
  380 
  381         sudo rm -rf ELASTICSEARCH_DIR || true
  382         sudo rm -rf ELASTICSEARCH_CFG_DIR || true
  383         sudo rm -rf ELASTICSEARCH_LOG_DIR || true
  384         sudo rm -rf ELASTICSEARCH_DATA_DIR || true
  385         sudo rm -rf $FILES/elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz || true
  386         sudo rm -rf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} || true
  387     fi
  388 }
  389 
  390 function start_elasticsearch {
  391     if is_service_enabled elasticsearch; then
  392         echo_summary "Starting ElasticSearch ${ELASTICSEARCH_VERSION}"
  393         # 5 extra seconds to ensure that ES started properly
  394         local esSleepTime=${ELASTICSEARCH_SLEEP_TIME:-5}
  395         run_process_sleep "elasticsearch" "$ELASTICSEARCH_DIR/bin/elasticsearch -E logger.org.elasticsearch=DEBUG" $esSleepTime
  396     fi
  397 }
  398 
  399 function install_kibana {
  400     if is_service_enabled kibana; then
  401         echo_summary "Installing Kibana ${KIBANA_VERSION}"
  402 
  403         local kibana_tarball=kibana-oss-${KIBANA_VERSION}.tar.gz
  404         local kibana_tarball_url=https://artifacts.elastic.co/downloads/kibana/${kibana_tarball}
  405         local kibana_tarball_dest
  406         kibana_tarball_dest=`get_extra_file ${kibana_tarball_url}`
  407 
  408         tar xzf ${kibana_tarball_dest} -C $DEST
  409 
  410         sudo chown -R $STACK_USER $DEST/kibana-${KIBANA_VERSION}
  411         ln -sf $DEST/kibana-${KIBANA_VERSION} $KIBANA_DIR
  412     fi
  413 }
  414 
  415 function configure_kibana {
  416     if is_service_enabled kibana; then
  417         echo_summary "Configuring Kibana ${KIBANA_VERSION}"
  418 
  419         sudo install -m 755 -d -o $STACK_USER $KIBANA_CFG_DIR
  420 
  421         sudo cp -f "${PLUGIN_FILES}"/kibana/kibana.yml $KIBANA_CFG_DIR/kibana.yml
  422         sudo chown -R $STACK_USER $KIBANA_CFG_DIR/kibana.yml
  423         sudo chmod 0644 $KIBANA_CFG_DIR/kibana.yml
  424 
  425         sudo sed -e "
  426             s|%KIBANA_SERVICE_HOST%|$KIBANA_SERVICE_HOST|g;
  427             s|%KIBANA_SERVICE_PORT%|$KIBANA_SERVICE_PORT|g;
  428             s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g;
  429             s|%ES_SERVICE_BIND_PORT%|$ES_SERVICE_BIND_PORT|g;
  430             s|%KIBANA_SERVER_BASE_PATH%|$KIBANA_SERVER_BASE_PATH|g;
  431             s|%KEYSTONE_AUTH_URI%|$KEYSTONE_AUTH_URI|g;
  432         " -i $KIBANA_CFG_DIR/kibana.yml
  433 
  434         ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml
  435     fi
  436 }
  437 
  438 function clean_kibana {
  439     if is_service_enabled kibana; then
  440         echo_summary "Cleaning Kibana ${KIBANA_VERSION}"
  441 
  442         sudo rm -rf $KIBANA_DIR || true
  443         sudo rm -rf $FILES/kibana-${KIBANA_VERSION}.tar.gz || true
  444         sudo rm -rf $KIBANA_CFG_DIR || true
  445     fi
  446 }
  447 
  448 function start_kibana {
  449     if is_service_enabled kibana; then
  450         echo_summary "Starting Kibana ${KIBANA_VERSION}"
  451         local kibanaSleepTime=${KIBANA_SLEEP_TIME:-120}     # kibana takes some time to load up
  452         local kibanaCFG="$KIBANA_CFG_DIR/kibana.yml"
  453         run_process_sleep "kibana" "$KIBANA_DIR/bin/kibana --config $kibanaCFG" $kibanaSleepTime
  454     fi
  455 }
  456 
  457 function configure_nvm {
  458     if is_service_enabled kibana; then
  459         echo_summary "Configuring NVM"
  460         curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
  461         source ~/.nvm/nvm.sh
  462         nvm install $KIBANA_DEV_NODE_JS_VERSION
  463         nvm use $KIBANA_DEV_NODE_JS_VERSION
  464     fi
  465 }
  466 
  467 function configure_yarn {
  468     if is_service_enabled kibana; then
  469         echo_summary "Configuring Yarn"
  470         REPOS_UPDATED=False
  471         curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
  472         echo "deb https://dl.yarnpkg.com/debian/ stable main" | \
  473             sudo tee /etc/apt/sources.list.d/yarn.list
  474         apt_get_update
  475         apt_get install yarn
  476     fi
  477 }
  478 
  479 function clean_nvm {
  480     if is_service_enabled kibana; then
  481         echo_summary "Cleaning NVM"
  482         rm -rf ~/.nvm
  483         rm -rf ~/.bower
  484     fi
  485 }
  486 
  487 function clean_yarn {
  488     if is_service_enabled kibana; then
  489         echo_summary "Cleaning Yarn"
  490         apt_get purge yarn
  491     fi
  492 }
  493 
  494 function build_kibana_plugin {
  495     if is_service_enabled kibana; then
  496         echo "Building Kibana plugin"
  497 
  498         echo_summary "Cloning and initializing Kibana development environment"
  499 
  500         git clone $KIBANA_DEV_REPO $KIBANA_DEV_DIR --branch $KIBANA_DEV_BRANCH --depth 1
  501 
  502         git_clone $MONASCA_KIBANA_PLUGIN_REPO $MONASCA_KIBANA_PLUGIN_DIR $MONASCA_KIBANA_PLUGIN_BRANCH
  503         cp -r $MONASCA_KIBANA_PLUGIN_DIR "$KIBANA_DEV_DIR/plugins"
  504         local plugin_dir="$KIBANA_DEV_DIR/plugins/monasca-kibana-plugin"
  505 
  506         yarn --cwd $KIBANA_DEV_DIR kbn bootstrap
  507         yarn --cwd $plugin_dir build
  508 
  509         local get_version_script="import json; obj = json.load(open('$plugin_dir/package.json')); print obj['version']"
  510         local monasca_kibana_plugin_version
  511         monasca_kibana_plugin_version=$(python -c "$get_version_script")
  512         local pkg="$plugin_dir/build/monasca-kibana-plugin-$monasca_kibana_plugin_version.zip"
  513         local easyPkg=$DEST/monasca-kibana-plugin.zip
  514         ln $pkg $easyPkg
  515         rm -rf $KIBANA_DEV_DIR
  516     fi
  517 }
  518 
  519 function install_kibana_plugin {
  520     if is_service_enabled kibana; then
  521         echo_summary "Install Kibana plugin"
  522         # note(trebskit) that needs to happen after kibana received
  523         # its configuration otherwise the plugin fails to be installed
  524         local pkg=file://$DEST/monasca-kibana-plugin.zip
  525         $KIBANA_DIR/bin/kibana-plugin install $pkg
  526     fi
  527 }
  528 
  529 function configure_monasca_log_persister {
  530     if is_service_enabled monasca-log-persister; then
  531         echo_summary "Configuring monasca-log-persister"
  532 
  533         sudo install -m 755 -d -o $STACK_USER $LOG_PERSISTER_DIR
  534 
  535         sudo cp -f "${PLUGIN_FILES}"/monasca-log-persister/persister.conf $LOG_PERSISTER_DIR/persister.conf
  536         sudo chown $STACK_USER $LOG_PERSISTER_DIR/persister.conf
  537         sudo chmod 0640 $LOG_PERSISTER_DIR/persister.conf
  538 
  539         sudo sed -e "
  540             s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g;
  541             s|%KAFKA_SERVICE_HOST%|$KAFKA_SERVICE_HOST|g;
  542             s|%KAFKA_SERVICE_PORT%|$KAFKA_SERVICE_PORT|g;
  543         " -i $LOG_PERSISTER_DIR/persister.conf
  544 
  545         ln -sf $LOG_PERSISTER_DIR/persister.conf $GATE_CONFIGURATION_DIR/log-persister.conf
  546     fi
  547 }
  548 
  549 function clean_monasca_log_persister {
  550     if is_service_enabled monasca-log-persister; then
  551         echo_summary "Cleaning monasca-log-persister"
  552         sudo rm -rf $LOG_PERSISTER_DIR || true
  553     fi
  554 }
  555 
  556 function start_monasca_log_persister {
  557     if is_service_enabled monasca-log-persister; then
  558         echo_summary "Starting monasca-log-persister"
  559         local logstash="$LOGSTASH_DIR/bin/logstash"
  560         run_process "monasca-log-persister" "$logstash -f $LOG_PERSISTER_DIR/persister.conf --path.data $LOGSTASH_DATA_DIR/monasca-log-persister"
  561     fi
  562 }
  563 
  564 function configure_monasca_log_transformer {
  565     if is_service_enabled monasca-log-transformer; then
  566         echo_summary "Configuring monasca-log-transformer"
  567 
  568         sudo install -m 755 -d -o $STACK_USER $LOG_TRANSFORMER_DIR
  569 
  570         sudo cp -f "${PLUGIN_FILES}"/monasca-log-transformer/transformer.conf $LOG_TRANSFORMER_DIR/transformer.conf
  571         sudo chown $STACK_USER $LOG_TRANSFORMER_DIR/transformer.conf
  572         sudo chmod 0640 $LOG_TRANSFORMER_DIR/transformer.conf
  573 
  574         sudo sed -e "
  575             s|%KAFKA_SERVICE_HOST%|$KAFKA_SERVICE_HOST|g;
  576             s|%KAFKA_SERVICE_PORT%|$KAFKA_SERVICE_PORT|g;
  577         " -i $LOG_TRANSFORMER_DIR/transformer.conf
  578 
  579         ln -sf $LOG_TRANSFORMER_DIR/transformer.conf $GATE_CONFIGURATION_DIR/log-transformer.conf
  580     fi
  581 }
  582 
  583 function clean_monasca_log_transformer {
  584     if is_service_enabled monasca-log-transformer; then
  585         echo_summary "Cleaning monasca-log-transformer"
  586         sudo rm -rf $LOG_TRANSFORMER_DIR || true
  587     fi
  588 }
  589 
  590 function start_monasca_log_transformer {
  591     if is_service_enabled monasca-log-transformer; then
  592         echo_summary "Starting monasca-log-transformer"
  593         local logstash="$LOGSTASH_DIR/bin/logstash"
  594         run_process "monasca-log-transformer" "$logstash -f $LOG_TRANSFORMER_DIR/transformer.conf --path.data $LOGSTASH_DATA_DIR/monasca-log-transformer"
  595     fi
  596 }
  597 
  598 function configure_monasca_log_metrics {
  599     if is_service_enabled monasca-log-metrics; then
  600         echo_summary "Configuring monasca-log-metrics"
  601 
  602         sudo install -m 755 -d -o $STACK_USER $LOG_METRICS_DIR
  603 
  604         sudo cp -f "${PLUGIN_FILES}"/monasca-log-metrics/log-metrics.conf $LOG_METRICS_DIR/log-metrics.conf
  605         sudo chown $STACK_USER $LOG_METRICS_DIR/log-metrics.conf
  606         sudo chmod 0640 $LOG_METRICS_DIR/log-metrics.conf
  607 
  608         sudo sed -e "
  609             s|%KAFKA_SERVICE_HOST%|$KAFKA_SERVICE_HOST|g;
  610             s|%KAFKA_SERVICE_PORT%|$KAFKA_SERVICE_PORT|g;
  611         " -i $LOG_METRICS_DIR/log-metrics.conf
  612 
  613         ln -sf $LOG_METRICS_DIR/log-metrics.conf $GATE_CONFIGURATION_DIR/log-metrics.conf
  614     fi
  615 }
  616 
  617 function clean_monasca_log_metrics {
  618     if is_service_enabled monasca-log-metrics; then
  619         echo_summary "Cleaning monasca-log-metrics"
  620         sudo rm -rf $LOG_METRICS_DIR || true
  621     fi
  622 }
  623 
  624 function start_monasca_log_metrics {
  625     if is_service_enabled monasca-log-metrics; then
  626         echo_summary "Starting monasca-log-metrics"
  627         local logstash="$LOGSTASH_DIR/bin/logstash"
  628         run_process "monasca-log-metrics" "$logstash -f $LOG_METRICS_DIR/log-metrics.conf --path.data $LOGSTASH_DATA_DIR/monasca-log-metrics"
  629     fi
  630 }
  631 
  632 function install_log_agent {
  633     if is_service_enabled monasca-log-agent; then
  634         echo_summary "Installing monasca-log-agent [logstash-output-monasca-plugin]"
  635 
  636         $LOGSTASH_DIR/bin/logstash-plugin install --version \
  637             "${LOGSTASH_OUTPUT_MONASCA_VERSION}" logstash-output-monasca_log_api
  638     fi
  639 }
  640 
  641 function configure_monasca_log_agent {
  642     if is_service_enabled monasca-log-agent; then
  643         echo_summary "Configuring monasca-log-agent"
  644 
  645         sudo install -m 755 -d -o $STACK_USER $LOG_AGENT_DIR
  646 
  647         sudo cp -f "${PLUGIN_FILES}"/monasca-log-agent/agent.conf $LOG_AGENT_DIR/agent.conf
  648         sudo chown $STACK_USER $LOG_AGENT_DIR/agent.conf
  649         sudo chmod 0640 $LOG_AGENT_DIR/agent.conf
  650 
  651         sudo sed -e "
  652             s|%MONASCA_API_URI_V2%|$MONASCA_API_URI_V2|g;
  653             s|%KEYSTONE_AUTH_URI%|$KEYSTONE_AUTH_URI_V3|g;
  654         " -i $LOG_AGENT_DIR/agent.conf
  655 
  656         ln -sf $LOG_AGENT_DIR/agent.conf $GATE_CONFIGURATION_DIR/log-agent.conf
  657 
  658     fi
  659 }
  660 
  661 function clean_monasca_log_agent {
  662     if is_service_enabled monasca-log-agent; then
  663         echo_summary "Cleaning monasca-log-agent"
  664         sudo rm -rf $LOG_AGENT_DIR || true
  665     fi
  666 }
  667 
  668 
  669 function start_monasca_log_api {
  670     if is_service_enabled monasca-log-api; then
  671         echo_summary "Starting monasca-log-api"
  672 
  673         local service_port=$MONASCA_LOG_API_SERVICE_PORT
  674         local service_protocol=$MONASCA_LOG_API_SERVICE_PROTOCOL
  675         if is_service_enabled tls-proxy; then
  676             service_port=$MONASCA_LOG_API_SERVICE_PORT_INT
  677             service_protocol="http"
  678         fi
  679         local service_uri
  680 
  681         if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then
  682             local enabled_site_file
  683             enabled_site_file=$(apache_site_config_for monasca-log-api)
  684             service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST/logs/v3.0
  685             if [ -f ${enabled_site_file} ]; then
  686                 enable_apache_site monasca-log-api
  687                 restart_apache_server
  688                 tail_log monasca-log-api /var/log/$APACHE_NAME/monasca-log-api.log
  689             fi
  690         elif [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then
  691             service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST/logs/v3.0
  692             run_process "monasca-log-api" "$MONASCA_LOG_API_BIN_DIR/uwsgi --ini $MONASCA_LOG_API_UWSGI_CONF" ""
  693         else
  694             service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST:$service_port
  695             run_process "monasca-log-api" "$MONASCA_LOG_API_BIN_DIR/gunicorn --paste $MONASCA_LOG_API_PASTE" ""
  696         fi
  697 
  698         echo "Waiting for monasca-log-api to start..."
  699         if ! wait_for_service $SERVICE_TIMEOUT $service_uri; then
  700             die $LINENO "monasca-log-api did not start"
  701         fi
  702 
  703         if is_service_enabled tls-proxy; then
  704             start_tls_proxy monasca-log-api '*' $MONASCA_LOG_API_SERVICE_PORT $MONASCA_LOG_API_SERVICE_HOST $MONASCA_LOG_API_SERVICE_PORT_INT
  705         fi
  706 
  707         restart_service memcached
  708     fi
  709 }
  710 
  711 function start_monasca_log_agent {
  712     if is_service_enabled monasca-log-agent; then
  713         echo_summary "Starting monasca-log-agent"
  714         local logstash="$LOGSTASH_DIR/bin/logstash"
  715         run_process "monasca-log-agent" "$logstash -f $LOG_AGENT_DIR/agent.conf --path.data $LOGSTASH_DATA_DIR/monasca-log-agent" "root" "root"
  716     fi
  717 }
  718 
  719 function install_nodejs {
  720     if is_service_enabled kibana; then
  721         # refresh installation
  722         apt_get install nodejs npm
  723         (
  724             npm config set registry "http://registry.npmjs.org/"; \
  725             npm config set proxy "${HTTP_PROXY}"; \
  726             npm set strict-ssl false;
  727         )
  728     fi
  729 }
  730 
  731 function clean_nodejs {
  732     if is_service_enabled kibana; then
  733         echo_summary "Cleaning Node.js"
  734         apt_get purge nodejs npm
  735     fi
  736 }
  737 
  738 function clean_gate_config_holder {
  739     sudo rm -rf $GATE_CONFIGURATION_DIR || true
  740 }
  741 
  742 function configure_kafka {
  743     echo_summary "Configuring Kafka topics"
  744     /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 \
  745         --replication-factor 1 --partitions 4 --topic log
  746     /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 \
  747         --replication-factor 1 --partitions 4 --topic transformed-log
  748 }
  749 
  750 function delete_kafka_topics {
  751     echo_summary "Deleting Kafka topics"
  752         /opt/kafka/bin/kafka-topics.sh --delete --zookeeper localhost:2181 \
  753                 --topic log || true
  754         /opt/kafka/bin/kafka-topics.sh --delete --zookeeper localhost:2181 \
  755                 --topic transformed-log || true
  756 }
  757 
  758 function create_log_management_accounts {
  759     if is_service_enabled monasca-log; then
  760         echo_summary "Enable Log Management in Keystone"
  761 
  762         # note(trebskit) following points to Kibana which is bad,
  763         # but we do not have search-api in monasca-log-api now
  764         # this code will be removed in future
  765         local log_search_url="http://$KIBANA_SERVICE_HOST:$KIBANA_SERVICE_PORT/"
  766 
  767         get_or_create_service "logs" "logs" "Monasca Log service"
  768 
  769         if $USE_OLD_LOG_API = true; then
  770             get_or_create_endpoint \
  771                 "logs" \
  772                 "$REGION_NAME" \
  773                 "$MONASCA_LOG_API_BASE_URI" \
  774                 "$MONASCA_LOG_API_BASE_URI" \
  775                 "$MONASCA_LOG_API_BASE_URI"
  776         else
  777             get_or_create_endpoint \
  778                 "logs" \
  779                 "$REGION_NAME" \
  780                 "$MONASCA_API_URI_V2" \
  781                 "$MONASCA_API_URI_V2" \
  782                 "$MONASCA_API_URI_V2"
  783 
  784         fi
  785 
  786         get_or_create_service "logs-search" "logs-search" "Monasca Log search service"
  787         get_or_create_endpoint \
  788             "logs-search" \
  789             "$REGION_NAME" \
  790             "$log_search_url" \
  791             "$log_search_url" \
  792             "$log_search_url"
  793 
  794     fi
  795 }
  796 
  797 #Restore errexit
  798 ${_ERREXIT_MON_LOG}
  799 
  800 # Restore xtrace
  801 ${_XTRACE_MON_LOG}