"Fossies" - the Fresh Open Source Software Archive

Member "storlets-7.0.0/devstack/plugin.sh" (14 Apr 2021, 16523 Bytes) of package /linux/misc/openstack/storlets-7.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Bash source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "plugin.sh": 6.0.0_vs_7.0.0.

    1 #!/bin/bash
    2 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    3 #    not use this file except in compliance with the License. You may obtain
    4 #    a copy of the License at
    5 #
    6 #         http://www.apache.org/licenses/LICENSE-2.0
    7 #
    8 #    Unless required by applicable law or agreed to in writing, software
    9 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   10 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   11 #    License for the specific language governing permissions and limitations
   12 #    under the License.
   13 
   14 # Functions to control the configuration and operation of the **Swift** service
   15 
   16 # Dependencies:
   17 #
   18 # - ``functions`` file
   19 # - ``functions-common`` file
   20 # - ``STACK_USER`` must be defined
   21 # - ``SWIFT_DATA_DIR`` or ``DATA_DIR`` must be defined
   22 # - ``lib/swift`` file
   23 # - ``lib/keystone`` file
   24 #
   25 # - install_storlets
   26 # TODO(eranr):
   27 # Add clean_storlets
   28 
   29 # Save trace setting
   30 _XTRACE_LIB_STORLETS=$(set +o | grep xtrace)
   31 set +o xtrace
   32 
   33 # Defaults
   34 # --------
   35 
   36 # General devstack install tunables
   37 ADMIN_USER=admin
   38 ADMIN_PASSWORD=admin
   39 ADMIN_PROJECT=admin
   40 
   41 SWIFT_DEFAULT_PROJECT=test
   42 SWIFT_DEFAULT_USER=tester
   43 SWIFT_DEFAULT_USER_PWD=testing
   44 SWIFT_MEMBER_USER=tester_member
   45 SWIFT_MEMBER_USER_PWD=member
   46 
   47 # Storlets install tunables
   48 STORLETS_DEFAULT_USER_DOMAIN_ID=${STORLETS_DEFAULT_USER_DOMAIN_ID:-default}
   49 STORLETS_DEFAULT_PROJECT_DOMAIN_ID=${STORLETS_DEFAULT_PROJECT_DOMAIN_ID:-default}
   50 STORLET_MANAGEMENT_USER=${STORLET_MANAGEMENT_USER:-$USER}
   51 STORLETS_DOCKER_DEVICE=${STORLETS_DOCKER_DEVICE:-/home/docker_device}
   52 STORLETS_DOCKER_BASE_IMG=${STORLETS_DOCKER_BASE_IMG:-ubuntu:20.04}
   53 STORLETS_DOCKER_BASE_IMG_NAME=${STORLETS_DOCKER_BASE_IMG_NAME:-ubuntu_20.04}
   54 STORLETS_DOCKER_SWIFT_GROUP_ID=${STORLETS_DOCKER_SWIFT_GROUP_ID:-1003}
   55 STORLETS_DOCKER_SWIFT_USER_ID=${STORLETS_DOCKER_SWIFT_USER_ID:-1003}
   56 STORLETS_SWIFT_RUNTIME_USER=${STORLETS_SWIFT_RUNTIME_USER:-$USER}
   57 STORLETS_SWIFT_RUNTIME_GROUP=${STORLETS_SWIFT_RUNTIME_GROUP:-$USER}
   58 STORLETS_MIDDLEWARE_NAME=storlet_handler
   59 STORLETS_STORLET_CONTAINER_NAME=${STORLETS_STORLET_CONTAINER_NAME:-storlet}
   60 STORLETS_DEPENDENCY_CONTAINER_NAME=${STORLETS_DEPENDENCY_CONTAINER_NAME:-dependency}
   61 STORLETS_LOG_CONTAIER_NAME=${STORLETS_LOG_CONTAIER_NAME:-log}
   62 STORLETS_GATEWAY_MODULE=${STORLETS_GATEWAY_MODULE:-docker}
   63 STORLETS_GATEWAY_CONF_FILE=${STORLETS_GATEWAY_CONF_FILE:-/etc/swift/storlet_docker_gateway.conf}
   64 STORLETS_PROXY_EXECUTION_ONLY=${STORLETS_PROXY_EXECUTION_ONLY:-false}
   65 STORLETS_SCRIPTS_DIR=${STORLETS_SCRIPTS_DIR:-"$STORLETS_DOCKER_DEVICE"/scripts}
   66 STORLETS_STORLETS_DIR=${STORLETS_STORLETS_DIR:-"$STORLETS_DOCKER_DEVICE"/storlets/scopes}
   67 STORLETS_LOGS_DIR=${STORLETS_LOGS_DIR:-"$STORLETS_DOCKER_DEVICE"/logs/scopes}
   68 STORLETS_CACHE_DIR=${STORLETS_CACHE_DIR:-"$STORLETS_DOCKER_DEVICE"/cache/scopes}
   69 STORLETS_PIPES_DIR=${STORLETS_PIPES_DIR:-"$STORLETS_DOCKER_DEVICE"/pipes/scopes}
   70 STORLETS_RESTART_CONTAINER_TIMEOUT=${STORLETS_RESTART_CONTAINER_TIMEOUT:-3}
   71 STORLETS_RUNTIME_TIMEOUT=${STORLETS_RUNTIME_TIMEOUT:-40}
   72 
   73 TMP_REGISTRY_PREFIX=/tmp/registry
   74 
   75 # Functions
   76 # ---------
   77 
   78 function _storlets_swift_start {
   79     swift-init --run-dir=${SWIFT_DATA_DIR}/run all start || true
   80 }
   81 
   82 function _storlets_swift_stop {
   83     swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true
   84 }
   85 
   86 function _storlets_swift_restart {
   87     swift-init --run-dir=${SWIFT_DATA_DIR}/run all restart || true
   88 }
   89 
   90 function _export_os_vars {
   91     export OS_IDENTITY_API_VERSION=3
   92     export OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3"
   93     export OS_REGION_NAME=RegionOne
   94 }
   95 
   96 function _export_keystone_os_vars {
   97     _export_os_vars
   98     export OS_USERNAME=$ADMIN_USER
   99     export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
  100     export OS_PASSWORD=$ADMIN_PASSWORD
  101     export OS_PROJECT_NAME=$ADMIN_USER
  102     export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
  103 }
  104 
  105 function _export_swift_os_vars {
  106     _export_os_vars
  107     export OS_USERNAME=$SWIFT_DEFAULT_USER
  108     export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
  109     export OS_PASSWORD=$SWIFT_DEFAULT_USER_PWD
  110     export OS_PROJECT_NAME=$SWIFT_DEFAULT_PROJECT
  111     export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
  112 }
  113 
  114 function configure_swift_and_keystone_for_storlets {
  115     # Add project and users to Keystone
  116     _export_keystone_os_vars
  117     project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l)
  118     if [ $project_test_created -eq 0 ]; then
  119         openstack project create $SWIFT_DEFAULT_PROJECT
  120     fi
  121     user_tester_created=$(openstack user list | grep -w $SWIFT_DEFAULT_USER | wc -l)
  122     if [ $user_tester_created -eq 0 ]; then
  123         openstack user create --project $SWIFT_DEFAULT_PROJECT --password $SWIFT_DEFAULT_USER_PWD $SWIFT_DEFAULT_USER
  124         openstack role add --user $SWIFT_DEFAULT_USER --project $SWIFT_DEFAULT_PROJECT admin
  125     fi
  126     member_user_tester_created=$(openstack user list | grep -w $SWIFT_MEMBER_USER | wc -l)
  127     if [ $member_user_tester_created -eq 0 ]; then
  128         role_member_created=$(openstack role list | grep -w _member_ | wc -l)
  129         if [ $role_member_created -eq 0 ]; then
  130             openstack role create _member_
  131         fi
  132         openstack user create --project $SWIFT_DEFAULT_PROJECT --password $SWIFT_MEMBER_USER_PWD $SWIFT_MEMBER_USER
  133         openstack role add --user $SWIFT_MEMBER_USER --project $SWIFT_DEFAULT_PROJECT _member_
  134     fi
  135 
  136     # Modify relevant Swift configuration files
  137     _generate_swift_middleware_conf
  138     _generate_storlet-docker-gateway
  139 
  140     if [ "${USE_PYTHON3}" == "False" ]; then
  141         sudo python2 devstack/swift_config.py install /tmp/swift_middleware_conf $STORLETS_SWIFT_RUNTIME_USER
  142     else
  143         sudo python3 devstack/swift_config.py install /tmp/swift_middleware_conf $STORLETS_SWIFT_RUNTIME_USER
  144     fi
  145 
  146     rm /tmp/swift_middleware_conf
  147     rm /tmp/storlet-docker-gateway.conf
  148 
  149     # Create storlet related containers and set ACLs
  150     _storlets_swift_start
  151     _export_swift_os_vars
  152     openstack object store account set --property Storlet-Enabled=True
  153     swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_STORLET_CONTAINER_NAME
  154     swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_DEPENDENCY_CONTAINER_NAME
  155     swift post $STORLETS_LOG_CONTAIER_NAME
  156 }
  157 
  158 function _install_docker {
  159     # TODO: Add other dirstors.
  160     # This one is geared towards Ubuntu
  161     # See other projects that install docker
  162     DOCKER_UNIX_SOCKET=/var/run/docker.sock
  163     DOCKER_SERVICE_TIMEOUT=5
  164 
  165     install_package socat
  166     wget http://get.docker.com -O install_docker.sh
  167     sudo chmod 777 install_docker.sh
  168     sudo bash -x install_docker.sh
  169     sudo rm install_docker.sh
  170 
  171     sudo killall docker || true
  172 
  173     # systemd env doesn't require /etc/default/docker options
  174     if [[ ! -e /etc/default/docker ]]; then
  175         sudo touch /etc/default/docker
  176         sudo ls /lib/systemd/system
  177         sudo sed -i '0,/[service]/a EnvironmentFile=-/etc/default/docker' /lib/systemd/system/docker.service
  178         sudo cat /lib/systemd/system/docker.service
  179     fi
  180     sudo cat /etc/default/docker
  181     sudo sed -r 's#^.*DOCKER_OPTS=.*$#DOCKER_OPTS="--debug -g /home/docker_device/docker --storage-opt dm.override_udev_sync_check=true"#' /etc/default/docker
  182 
  183     # Start the daemon - restart just in case the package ever auto-starts...
  184     restart_service docker
  185 
  186     echo "Waiting for docker daemon to start..."
  187     DOCKER_GROUP=$(groups | cut -d' ' -f1)
  188     CONFIGURE_CMD="while ! /bin/echo -e 'GET /version HTTP/1.0\n\n' | socat - unix-connect:$DOCKER_UNIX_SOCKET 2>/dev/null | grep -q '200 OK'; do
  189       # Set the right group on docker unix socket before retrying
  190       sudo chgrp $DOCKER_GROUP $DOCKER_UNIX_SOCKET
  191       sudo chmod g+rw $DOCKER_UNIX_SOCKET
  192       sleep 1
  193     done"
  194     if ! timeout $DOCKER_SERVICE_TIMEOUT sh -c "$CONFIGURE_CMD"; then
  195       die $LINENO "docker did not start"
  196 fi
  197 }
  198 
  199 function prepare_storlets_install {
  200     sudo mkdir -p "$STORLETS_DOCKER_DEVICE"/docker
  201     sudo chmod 777 $STORLETS_DOCKER_DEVICE
  202     _install_docker
  203 
  204     if is_ubuntu; then
  205       install_package openjdk-11-jdk-headless ant
  206     else
  207       die $LINENO "Unsupported distro"
  208     fi
  209 
  210     if [ "${USE_PYTHON3}" == "False" ]; then
  211       # TODO(takashi): Remove this when we remove py2 support
  212       install_package python2.7 python2.7-dev
  213     else
  214       install_python3
  215     fi
  216 
  217 }
  218 
  219 function _generate_jre_dockerfile {
  220     # TODO(tkajinam): Remove py2 packages when we remove its support
  221     local PYTHON_PACKAGES='python2.7 python2.7-dev python3.8 python3.8-dev'
  222     if python3_enabled; then
  223         PYTHON_PACKAGES="python2.7 python2.7-dev python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev"
  224     fi
  225 
  226     cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre11/Dockerfile
  227 FROM $STORLETS_DOCKER_BASE_IMG
  228 MAINTAINER root
  229 
  230 RUN apt-get update && \
  231     apt-get install ${PYTHON_PACKAGES} openjdk-11-jre-headless -y && \
  232     apt-get clean
  233 EOF
  234 }
  235 
  236 function create_base_jre_image {
  237     echo "Create base jre image"
  238     docker pull $STORLETS_DOCKER_BASE_IMG
  239     mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
  240     _generate_jre_dockerfile
  241     cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
  242     docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11 .
  243     cd -
  244 }
  245 
  246 function _generate_logback_xml {
  247     sudo tee /usr/local/lib/storlets/logback.xml <<EOF >/dev/null
  248 <configuration>
  249   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
  250     <file>/tmp/SDaemon.log</file>
  251     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
  252             <!-- daily rollover. Make sure the path matches the one in the file element or else
  253              the rollover logs are placed in the working directory. -->
  254             <fileNamePattern>/srv/logs/application_%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  255 
  256             <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
  257                 <maxFileSize>1MB</maxFileSize>
  258             </timeBasedFileNamingAndTriggeringPolicy>
  259             <!-- keep 30 days' worth of history -->
  260             <maxHistory>30</maxHistory>
  261     </rollingPolicy>
  262     <encoder>
  263         <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
  264     </encoder>
  265   </appender>
  266 
  267   <root level="TRACE">
  268     <appender-ref ref="FILE" />
  269   </root>
  270 </configuration>
  271 EOF
  272     sudo chmod 0744 /usr/local/lib/storlets/logback.xml
  273 }
  274 
  275 function _generate_jre_storlet_dockerfile {
  276     cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/Dockerfile
  277 FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11
  278 MAINTAINER root
  279 RUN [ "groupadd", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
  280 RUN [ "useradd", "-u" , "$STORLETS_DOCKER_SWIFT_USER_ID", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
  281 
  282 CMD ["prod", "/mnt/channels/factory_pipe", "DEBUG"]
  283 
  284 ENTRYPOINT ["/usr/local/libexec/storlets/init_container.sh"]
  285 EOF
  286 }
  287 
  288 function create_storlet_engine_image {
  289     echo "Create Storlet engine image"
  290     mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
  291     _generate_logback_xml
  292     _generate_jre_storlet_dockerfile
  293     cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
  294     docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets .
  295     cd -
  296 }
  297 
  298 function install_storlets_code {
  299     echo "Installing storlets"
  300     cd $REPO_DIR
  301     sudo ./install_libs.sh
  302     pip_install .
  303 
  304     # install test requirements because we still need some packages in the list
  305     # to prepare deployment
  306     pip_install -r ./test-requirements.txt
  307 
  308     # Also install code to library directory so that we can import them
  309     # from docker container.
  310     sudo mkdir -p -m 755 /usr/local/lib/storlets/python
  311 
  312     # NOTE(takashi): We need --no-deps to avoid enum34 installed in py 2 env,
  313     #                which causes failure in py3 execution.
  314     pip_install . -t /usr/local/lib/storlets/python --no-compile --no-deps
  315     for bin_file in storlets-daemon storlets-daemon-factory ; do
  316         sudo cp `which ${bin_file}` /usr/local/libexec/storlets/
  317     done
  318 
  319     sudo mkdir -p $STORLETS_DOCKER_DEVICE/scripts
  320     sudo chown "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" "$STORLETS_DOCKER_DEVICE"/scripts
  321     sudo chmod 0755 "$STORLETS_DOCKER_DEVICE"/scripts
  322     sudo cp scripts/restart_docker_container "$STORLETS_DOCKER_DEVICE"/scripts/
  323     sudo chmod 04755 "$STORLETS_DOCKER_DEVICE"/scripts/restart_docker_container
  324     sudo chown root:root "$STORLETS_DOCKER_DEVICE"/scripts/restart_docker_container
  325 
  326     # NOTE(takashi): We should cleanup egg-info directory here, otherwise it
  327     #                causes permission denined when installing package by tox.
  328     sudo rm -rf storlets.egg-info
  329 
  330     cd -
  331 }
  332 
  333 function _generate_swift_middleware_conf {
  334     cat <<EOF > /tmp/swift_middleware_conf
  335 [proxy-confs]
  336 proxy_server_conf_file = /etc/swift/proxy-server.conf
  337 storlet_proxy_server_conf_file = /etc/swift/storlet-proxy-server.conf
  338 
  339 [object-confs]
  340 object_server_conf_files = /etc/swift/object-server/1.conf
  341 #object_server_conf_files = /etc/swift/object-server/1.conf, /etc/swift/object-server/2.conf, /etc/swift/object-server/3.conf, /etc/swift/object-server/4.conf
  342 #object_server_conf_files = /etc/swift/object-server.conf
  343 
  344 [common-confs]
  345 storlet_middleware = $STORLETS_MIDDLEWARE_NAME
  346 storlet_container = $STORLETS_STORLET_CONTAINER_NAME
  347 storlet_dependency = $STORLETS_DEPENDENCY_CONTAINER_NAME
  348 #storlet_log = $STORLETS_LOG_CONTAIER_NAME
  349 storlet_gateway_module = $STORLETS_GATEWAY_MODULE
  350 storlet_gateway_conf = $STORLETS_GATEWAY_CONF_FILE
  351 storlet_proxy_execution = $STORLETS_PROXY_EXECUTION_ONLY
  352 EOF
  353 }
  354 
  355 function _generate_storlet-docker-gateway {
  356     cat <<EOF > /tmp/storlet-docker-gateway.conf
  357 [DEFAULT]
  358 storlet_logcontainer = $STORLETS_LOG_CONTAIER_NAME
  359 cache_dir = $STORLETS_CACHE_DIR
  360 log_dir = $STORLETS_LOGS_DIR
  361 script_dir = $STORLETS_SCRIPTS_DIR
  362 storlets_dir = $STORLETS_STORLETS_DIR
  363 pipes_dir = $STORLETS_PIPES_DIR
  364 restart_linux_container_timeout = $STORLETS_RESTART_CONTAINER_TIMEOUT
  365 storlet_timeout = $STORLETS_RUNTIME_TIMEOUT
  366 EOF
  367 }
  368 
  369 function _generate_default_tenant_dockerfile {
  370     cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile
  371 FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets
  372 MAINTAINER root
  373 EOF
  374 }
  375 
  376 function create_default_tenant_image {
  377     SWIFT_DEFAULT_PROJECT_ID=`openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | awk '{ print $2 }'`
  378     mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
  379     _generate_default_tenant_dockerfile
  380     cd ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
  381     docker build -q -t ${SWIFT_DEFAULT_PROJECT_ID:0:13} .
  382     cd -
  383 }
  384 
  385 function create_test_config_file {
  386     testfile=${REPO_DIR}/test.conf
  387     iniset ${testfile} general keystone_default_domain $STORLETS_DEFAULT_PROJECT_DOMAIN_ID
  388     iniset ${testfile} general keystone_public_url $KEYSTONE_PUBLIC_URL
  389     iniset ${testfile} general storlets_default_project_name $SWIFT_DEFAULT_PROJECT
  390     iniset ${testfile} general storlets_default_project_user_name $SWIFT_DEFAULT_USER
  391     iniset ${testfile} general storlets_default_project_user_password $SWIFT_DEFAULT_USER_PWD
  392     iniset ${testfile} general storlets_default_project_member_user $SWIFT_MEMBER_USER
  393     iniset ${testfile} general storlets_default_project_member_password $SWIFT_MEMBER_USER_PWD
  394     iniset ${testfile} general region
  395 }
  396 
  397 function install_storlets {
  398     echo "Install storlets dependencies"
  399     prepare_storlets_install
  400 
  401     echo "Install storlets code"
  402     install_storlets_code
  403 
  404     echo "Configure swift and keystone for storlets"
  405     configure_swift_and_keystone_for_storlets
  406 
  407     echo "Create Docker images"
  408     create_base_jre_image
  409     create_storlet_engine_image
  410     create_default_tenant_image
  411 
  412     echo "Create test configuration file"
  413     create_test_config_file
  414 
  415     echo "restart swift"
  416     _storlets_swift_restart
  417 }
  418 
  419 function uninstall_storlets {
  420     sudo service docker stop
  421     sudo sed -r 's#^.*DOCKER_OPTS=.*$#DOCKER_OPTS="--debug --storage-opt dm.override_udev_sync_check=true"#' /etc/default/docker
  422 
  423     echo "Cleaning all storlets runtime stuff..."
  424     sudo rm -fr ${STORLETS_DOCKER_DEVICE}
  425 }
  426 
  427 # Restore xtrace
  428 $_XTRACE_LIB_STORLETS
  429 
  430 # Tell emacs to use shell-script-mode
  431 ## Local variables:
  432 ## mode: shell-script
  433 ## End: