"Fossies" - the Fresh Open Source Software Archive

Member "ironic-16.0.3/ironic/tests/unit/drivers/modules/test_agent_base.py" (18 Jan 2021, 123690 Bytes) of package /linux/misc/openstack/ironic-16.0.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_agent_base.py": 16.0.2_vs_16.0.3.

    1 # Copyright 2015 Red Hat, Inc.
    2 # All Rights Reserved.
    3 #
    4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    5 #    not use this file except in compliance with the License. You may obtain
    6 #    a copy of the License at
    7 #
    8 #         http://www.apache.org/licenses/LICENSE-2.0
    9 #
   10 #    Unless required by applicable law or agreed to in writing, software
   11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   13 #    License for the specific language governing permissions and limitations
   14 #    under the License.
   15 
   16 import time
   17 import types
   18 from unittest import mock
   19 
   20 from oslo_config import cfg
   21 from testtools import matchers
   22 
   23 from ironic.common import boot_devices
   24 from ironic.common import exception
   25 from ironic.common import image_service
   26 from ironic.common import states
   27 from ironic.conductor import steps as conductor_steps
   28 from ironic.conductor import task_manager
   29 from ironic.conductor import utils as manager_utils
   30 from ironic.drivers import base as drivers_base
   31 from ironic.drivers.modules import agent
   32 from ironic.drivers.modules import agent_base
   33 from ironic.drivers.modules import agent_client
   34 from ironic.drivers.modules import boot_mode_utils
   35 from ironic.drivers.modules import deploy_utils
   36 from ironic.drivers.modules import fake
   37 from ironic.drivers.modules import pxe
   38 from ironic.drivers import utils as driver_utils
   39 from ironic import objects
   40 from ironic.tests.unit.db import base as db_base
   41 from ironic.tests.unit.db import utils as db_utils
   42 from ironic.tests.unit.objects import utils as object_utils
   43 
   44 CONF = cfg.CONF
   45 
   46 INSTANCE_INFO = db_utils.get_test_agent_instance_info()
   47 DRIVER_INFO = db_utils.get_test_agent_driver_info()
   48 DRIVER_INTERNAL_INFO = db_utils.get_test_agent_driver_internal_info()
   49 
   50 
   51 class AgentDeployMixinBaseTest(db_base.DbTestCase):
   52 
   53     def setUp(self):
   54         super(AgentDeployMixinBaseTest, self).setUp()
   55         for iface in drivers_base.ALL_INTERFACES:
   56             impl = 'fake'
   57             if iface == 'deploy':
   58                 impl = 'direct'
   59             if iface == 'boot':
   60                 impl = 'pxe'
   61             if iface == 'rescue':
   62                 impl = 'agent'
   63             if iface == 'network':
   64                 continue
   65             config_kwarg = {'enabled_%s_interfaces' % iface: [impl],
   66                             'default_%s_interface' % iface: impl}
   67             self.config(**config_kwarg)
   68         self.config(enabled_hardware_types=['fake-hardware'])
   69         self.deploy = agent_base.AgentDeployMixin()
   70         n = {
   71             'driver': 'fake-hardware',
   72             'instance_info': INSTANCE_INFO,
   73             'driver_info': DRIVER_INFO,
   74             'driver_internal_info': DRIVER_INTERNAL_INFO,
   75             'network_interface': 'noop'
   76         }
   77         self.node = object_utils.create_test_node(self.context, **n)
   78 
   79 
   80 class HeartbeatMixinTest(AgentDeployMixinBaseTest):
   81 
   82     def setUp(self):
   83         super(HeartbeatMixinTest, self).setUp()
   84         self.deploy = agent_base.HeartbeatMixin()
   85 
   86     @mock.patch.object(agent_base.HeartbeatMixin,
   87                        'refresh_steps', autospec=True)
   88     @mock.patch.object(agent_base.HeartbeatMixin,
   89                        'in_core_deploy_step', autospec=True)
   90     @mock.patch.object(agent_base.HeartbeatMixin,
   91                        'deploy_has_started', autospec=True)
   92     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
   93                        autospec=True)
   94     @mock.patch.object(agent_base.HeartbeatMixin,
   95                        'reboot_to_instance', autospec=True)
   96     def test_heartbeat_continue_deploy(self, rti_mock, cd_mock,
   97                                        deploy_started_mock,
   98                                        in_deploy_mock,
   99                                        refresh_steps_mock):
  100         in_deploy_mock.return_value = True
  101         deploy_started_mock.return_value = False
  102         self.node.provision_state = states.DEPLOYWAIT
  103         self.node.save()
  104         with task_manager.acquire(self.context, self.node.uuid,
  105                                   shared=True) as task:
  106             self.deploy.heartbeat(task, 'url', '3.2.0')
  107             self.assertFalse(task.shared)
  108             self.assertEqual(
  109                 'url', task.node.driver_internal_info['agent_url'])
  110             self.assertEqual(
  111                 '3.2.0',
  112                 task.node.driver_internal_info['agent_version'])
  113             cd_mock.assert_called_once_with(self.deploy, task)
  114             self.assertFalse(rti_mock.called)
  115             refresh_steps_mock.assert_called_once_with(self.deploy,
  116                                                        task, 'deploy')
  117 
  118     @mock.patch.object(agent_base.HeartbeatMixin,
  119                        'refresh_steps', autospec=True)
  120     @mock.patch.object(agent_base.HeartbeatMixin,
  121                        'in_core_deploy_step', autospec=True)
  122     @mock.patch.object(agent_base.HeartbeatMixin,
  123                        'deploy_has_started', autospec=True)
  124     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  125                        autospec=True)
  126     @mock.patch.object(agent_base.HeartbeatMixin,
  127                        'reboot_to_instance', autospec=True)
  128     def test_heartbeat_continue_deploy_second_run(self, rti_mock, cd_mock,
  129                                                   deploy_started_mock,
  130                                                   in_deploy_mock,
  131                                                   refresh_steps_mock):
  132         in_deploy_mock.return_value = True
  133         deploy_started_mock.return_value = False
  134         dii = self.node.driver_internal_info
  135         dii['agent_cached_deploy_steps'] = ['step']
  136         self.node.driver_internal_info = dii
  137         self.node.provision_state = states.DEPLOYWAIT
  138         self.node.save()
  139         with task_manager.acquire(self.context, self.node.uuid,
  140                                   shared=True) as task:
  141             self.deploy.heartbeat(task, 'url', '3.2.0')
  142             self.assertFalse(task.shared)
  143             self.assertEqual(
  144                 'url', task.node.driver_internal_info['agent_url'])
  145             self.assertEqual(
  146                 '3.2.0',
  147                 task.node.driver_internal_info['agent_version'])
  148             cd_mock.assert_called_once_with(self.deploy, task)
  149             self.assertFalse(rti_mock.called)
  150             self.assertFalse(refresh_steps_mock.called)
  151 
  152     @mock.patch.object(agent_base.HeartbeatMixin,
  153                        'in_core_deploy_step', autospec=True)
  154     @mock.patch.object(agent_base.HeartbeatMixin,
  155                        'deploy_has_started', autospec=True)
  156     @mock.patch.object(agent_base.HeartbeatMixin,
  157                        'deploy_is_done', autospec=True)
  158     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  159                        autospec=True)
  160     @mock.patch.object(agent_base.HeartbeatMixin,
  161                        'reboot_to_instance', autospec=True)
  162     def test_heartbeat_reboot_to_instance(self, rti_mock, cd_mock,
  163                                           deploy_is_done_mock,
  164                                           deploy_started_mock,
  165                                           in_deploy_mock):
  166         in_deploy_mock.return_value = True
  167         deploy_started_mock.return_value = True
  168         deploy_is_done_mock.return_value = True
  169         self.node.provision_state = states.DEPLOYWAIT
  170         self.node.save()
  171         with task_manager.acquire(self.context, self.node.uuid,
  172                                   shared=True) as task:
  173             self.deploy.heartbeat(task, 'url', '3.2.0')
  174             self.assertIsNone(task.node.last_error)
  175             self.assertFalse(task.shared)
  176             self.assertEqual(
  177                 'url', task.node.driver_internal_info['agent_url'])
  178             self.assertEqual(
  179                 '3.2.0',
  180                 task.node.driver_internal_info['agent_version'])
  181             self.assertFalse(cd_mock.called)
  182             rti_mock.assert_called_once_with(self.deploy, task)
  183 
  184     @mock.patch.object(agent_base.HeartbeatMixin,
  185                        'process_next_step', autospec=True)
  186     @mock.patch.object(agent_base.HeartbeatMixin,
  187                        'in_core_deploy_step', autospec=True)
  188     @mock.patch.object(agent_base.HeartbeatMixin,
  189                        'deploy_has_started', autospec=True)
  190     @mock.patch.object(agent_base.HeartbeatMixin,
  191                        'deploy_is_done', autospec=True)
  192     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  193                        autospec=True)
  194     @mock.patch.object(agent_base.HeartbeatMixin,
  195                        'reboot_to_instance', autospec=True)
  196     def test_heartbeat_not_in_core_deploy_step(self, rti_mock, cd_mock,
  197                                                deploy_is_done_mock,
  198                                                deploy_started_mock,
  199                                                in_deploy_mock,
  200                                                process_next_mock):
  201         # Check that heartbeats do not trigger deployment actions when not in
  202         # the deploy.deploy step.
  203         in_deploy_mock.return_value = False
  204         self.node.provision_state = states.DEPLOYWAIT
  205         self.node.save()
  206         with task_manager.acquire(self.context, self.node.uuid,
  207                                   shared=True) as task:
  208             self.deploy.heartbeat(task, 'url', '3.2.0')
  209             self.assertFalse(task.shared)
  210             self.assertEqual(
  211                 'url', task.node.driver_internal_info['agent_url'])
  212             self.assertEqual(
  213                 '3.2.0',
  214                 task.node.driver_internal_info['agent_version'])
  215             self.assertFalse(deploy_started_mock.called)
  216             self.assertFalse(deploy_is_done_mock.called)
  217             self.assertFalse(cd_mock.called)
  218             self.assertFalse(rti_mock.called)
  219             process_next_mock.assert_called_once_with(self.deploy,
  220                                                       task, 'deploy')
  221 
  222     @mock.patch.object(agent_base.HeartbeatMixin,
  223                        'refresh_steps', autospec=True)
  224     @mock.patch.object(agent_base.HeartbeatMixin,
  225                        'process_next_step', autospec=True)
  226     @mock.patch.object(agent_base.HeartbeatMixin,
  227                        'in_core_deploy_step', autospec=True)
  228     @mock.patch.object(agent_base.HeartbeatMixin,
  229                        'deploy_has_started', autospec=True)
  230     @mock.patch.object(agent_base.HeartbeatMixin,
  231                        'deploy_is_done', autospec=True)
  232     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  233                        autospec=True)
  234     @mock.patch.object(agent_base.HeartbeatMixin,
  235                        'reboot_to_instance', autospec=True)
  236     def test_heartbeat_not_in_core_deploy_step_refresh(self, rti_mock, cd_mock,
  237                                                        deploy_is_done_mock,
  238                                                        deploy_started_mock,
  239                                                        in_deploy_mock,
  240                                                        process_next_mock,
  241                                                        refresh_steps_mock):
  242         # Check loading in-band deploy steps.
  243         in_deploy_mock.return_value = False
  244         self.node.provision_state = states.DEPLOYWAIT
  245         info = self.node.driver_internal_info
  246         info.pop('agent_cached_deploy_steps', None)
  247         self.node.driver_internal_info = info
  248         self.node.save()
  249         with task_manager.acquire(self.context, self.node.uuid,
  250                                   shared=True) as task:
  251             self.deploy.heartbeat(task, 'url', '3.2.0')
  252             self.assertFalse(task.shared)
  253             self.assertEqual(
  254                 'url', task.node.driver_internal_info['agent_url'])
  255             self.assertEqual(
  256                 '3.2.0',
  257                 task.node.driver_internal_info['agent_version'])
  258             self.assertFalse(deploy_started_mock.called)
  259             self.assertFalse(deploy_is_done_mock.called)
  260             self.assertFalse(cd_mock.called)
  261             self.assertFalse(rti_mock.called)
  262             refresh_steps_mock.assert_called_once_with(self.deploy,
  263                                                        task, 'deploy')
  264             process_next_mock.assert_called_once_with(self.deploy,
  265                                                       task, 'deploy')
  266 
  267     @mock.patch.object(manager_utils,
  268                        'notify_conductor_resume_deploy', autospec=True)
  269     @mock.patch.object(agent_base.HeartbeatMixin,
  270                        'in_core_deploy_step', autospec=True)
  271     @mock.patch.object(agent_base.HeartbeatMixin,
  272                        'deploy_has_started', autospec=True)
  273     @mock.patch.object(agent_base.HeartbeatMixin,
  274                        'deploy_is_done', autospec=True)
  275     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  276                        autospec=True)
  277     @mock.patch.object(agent_base.HeartbeatMixin,
  278                        'reboot_to_instance', autospec=True)
  279     def test_heartbeat_not_in_core_deploy_step_polling(self, rti_mock, cd_mock,
  280                                                        deploy_is_done_mock,
  281                                                        deploy_started_mock,
  282                                                        in_deploy_mock,
  283                                                        in_resume_deploy_mock):
  284         # Check that heartbeats do not trigger deployment actions when not in
  285         # the deploy.deploy step.
  286         in_deploy_mock.return_value = False
  287         self.node.provision_state = states.DEPLOYWAIT
  288         info = self.node.driver_internal_info
  289         info['agent_cached_deploy_steps'] = ['step1']
  290         info['deployment_polling'] = True
  291         self.node.driver_internal_info = info
  292         self.node.save()
  293         with task_manager.acquire(self.context, self.node.uuid,
  294                                   shared=True) as task:
  295             self.deploy.heartbeat(task, 'url', '3.2.0')
  296             self.assertFalse(task.shared)
  297             self.assertEqual(
  298                 'url', task.node.driver_internal_info['agent_url'])
  299             self.assertEqual(
  300                 '3.2.0',
  301                 task.node.driver_internal_info['agent_version'])
  302             self.assertFalse(deploy_started_mock.called)
  303             self.assertFalse(deploy_is_done_mock.called)
  304             self.assertFalse(cd_mock.called)
  305             self.assertFalse(rti_mock.called)
  306             self.assertFalse(in_resume_deploy_mock.called)
  307 
  308     @mock.patch.object(agent_base.HeartbeatMixin, 'process_next_step',
  309                        autospec=True)
  310     @mock.patch.object(agent_base.HeartbeatMixin,
  311                        'in_core_deploy_step', autospec=True)
  312     @mock.patch.object(agent_base.HeartbeatMixin,
  313                        'deploy_has_started', autospec=True)
  314     @mock.patch.object(agent_base.HeartbeatMixin,
  315                        'deploy_is_done', autospec=True)
  316     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  317                        autospec=True)
  318     @mock.patch.object(agent_base.HeartbeatMixin,
  319                        'reboot_to_instance', autospec=True)
  320     def test_heartbeat_decomposed_steps(self, rti_mock, cd_mock,
  321                                         deploy_is_done_mock,
  322                                         deploy_started_mock,
  323                                         in_deploy_mock,
  324                                         next_step_mock):
  325         self.deploy.has_decomposed_deploy_steps = True
  326         # Check that heartbeats do not trigger deployment actions when the
  327         # driver has decomposed deploy steps.
  328         self.node.provision_state = states.DEPLOYWAIT
  329         self.node.save()
  330         with task_manager.acquire(self.context, self.node.uuid,
  331                                   shared=True) as task:
  332             self.deploy.heartbeat(task, 'url', '3.2.0')
  333             self.assertFalse(task.shared)
  334             self.assertEqual(
  335                 'url', task.node.driver_internal_info['agent_url'])
  336             self.assertEqual(
  337                 '3.2.0',
  338                 task.node.driver_internal_info['agent_version'])
  339             self.assertFalse(in_deploy_mock.called)
  340             self.assertFalse(deploy_started_mock.called)
  341             self.assertFalse(deploy_is_done_mock.called)
  342             self.assertFalse(cd_mock.called)
  343             self.assertFalse(rti_mock.called)
  344             self.assertTrue(next_step_mock.called)
  345 
  346     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  347                        autospec=True)
  348     @mock.patch.object(agent_base.HeartbeatMixin,
  349                        'reboot_to_instance', autospec=True)
  350     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  351                        autospec=True)
  352     def test_heartbeat_in_maintenance(self, ncrc_mock, rti_mock, cd_mock):
  353         # NOTE(pas-ha) checking only for states that are not noop
  354         for state in (states.DEPLOYWAIT, states.CLEANWAIT):
  355             for m in (ncrc_mock, rti_mock, cd_mock):
  356                 m.reset_mock()
  357             self.node.provision_state = state
  358             self.node.maintenance = True
  359             self.node.save()
  360             agent_url = 'url-%s' % state
  361             with task_manager.acquire(self.context, self.node.uuid,
  362                                       shared=True) as task:
  363                 self.deploy.heartbeat(task, agent_url, '3.2.0')
  364                 self.assertFalse(task.shared)
  365                 self.assertEqual(
  366                     agent_url,
  367                     task.node.driver_internal_info['agent_url'])
  368                 self.assertEqual(
  369                     '3.2.0',
  370                     task.node.driver_internal_info['agent_version'])
  371                 self.assertEqual(state, task.node.provision_state)
  372                 self.assertIsNone(task.node.last_error)
  373             self.assertEqual(0, ncrc_mock.call_count)
  374             self.assertEqual(0, rti_mock.call_count)
  375             self.assertEqual(0, cd_mock.call_count)
  376 
  377     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  378                        autospec=True)
  379     @mock.patch.object(agent_base.HeartbeatMixin,
  380                        'reboot_to_instance', autospec=True)
  381     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  382                        autospec=True)
  383     def test_heartbeat_in_maintenance_abort(self, ncrc_mock, rti_mock,
  384                                             cd_mock):
  385         CONF.set_override('allow_provisioning_in_maintenance', False,
  386                           group='conductor')
  387         for state, expected in [(states.DEPLOYWAIT, states.DEPLOYFAIL),
  388                                 (states.CLEANWAIT, states.CLEANFAIL),
  389                                 (states.RESCUEWAIT, states.RESCUEFAIL)]:
  390             for m in (ncrc_mock, rti_mock, cd_mock):
  391                 m.reset_mock()
  392             self.node.provision_state = state
  393             self.node.maintenance = True
  394             self.node.save()
  395             agent_url = 'url-%s' % state
  396             with task_manager.acquire(self.context, self.node.uuid,
  397                                       shared=True) as task:
  398                 self.deploy.heartbeat(task, agent_url, '3.2.0')
  399                 self.assertFalse(task.shared)
  400                 self.assertIsNone(
  401                     task.node.driver_internal_info.get('agent_url', None))
  402                 self.assertEqual(
  403                     '3.2.0',
  404                     task.node.driver_internal_info['agent_version'])
  405             self.node.refresh()
  406             self.assertEqual(expected, self.node.provision_state)
  407             self.assertIn('aborted', self.node.last_error)
  408             self.assertEqual(0, ncrc_mock.call_count)
  409             self.assertEqual(0, rti_mock.call_count)
  410             self.assertEqual(0, cd_mock.call_count)
  411 
  412     @mock.patch('time.sleep', lambda _t: None)
  413     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  414                        autospec=True)
  415     @mock.patch.object(agent_base.HeartbeatMixin,
  416                        'reboot_to_instance', autospec=True)
  417     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  418                        autospec=True)
  419     def test_heartbeat_with_reservation(self, ncrc_mock, rti_mock, cd_mock):
  420         # NOTE(pas-ha) checking only for states that are not noop
  421         for state in (states.DEPLOYWAIT, states.CLEANWAIT):
  422             for m in (ncrc_mock, rti_mock, cd_mock):
  423                 m.reset_mock()
  424             self.node.provision_state = state
  425             self.node.reservation = 'localhost'
  426             self.node.save()
  427             old_drv_info = self.node.driver_internal_info.copy()
  428             agent_url = 'url-%s' % state
  429             with task_manager.acquire(self.context, self.node.uuid,
  430                                       shared=True) as task:
  431                 self.deploy.heartbeat(task, agent_url, '3.2.0')
  432                 self.assertTrue(task.shared)
  433                 self.assertEqual(old_drv_info, task.node.driver_internal_info)
  434                 self.assertIsNone(task.node.last_error)
  435             self.assertEqual(0, ncrc_mock.call_count)
  436             self.assertEqual(0, rti_mock.call_count)
  437             self.assertEqual(0, cd_mock.call_count)
  438 
  439     @mock.patch.object(agent_base.LOG, 'error', autospec=True)
  440     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  441                        autospec=True)
  442     @mock.patch.object(agent_base.HeartbeatMixin,
  443                        'reboot_to_instance', autospec=True)
  444     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  445                        autospec=True)
  446     def test_heartbeat_noops_in_wrong_state(self, ncrc_mock, rti_mock,
  447                                             cd_mock, log_mock):
  448         allowed = {states.DEPLOYWAIT, states.CLEANWAIT, states.RESCUEWAIT,
  449                    states.DEPLOYING, states.CLEANING, states.RESCUING}
  450         for state in set(states.machine.states) - allowed:
  451             for m in (ncrc_mock, rti_mock, cd_mock, log_mock):
  452                 m.reset_mock()
  453             with task_manager.acquire(self.context, self.node.uuid,
  454                                       shared=True) as task:
  455                 task.node.provision_state = state
  456                 self.deploy.heartbeat(task, 'url', '1.0.0')
  457                 self.assertTrue(task.shared)
  458                 self.assertNotIn('agent_last_heartbeat',
  459                                  task.node.driver_internal_info)
  460             self.assertEqual(0, ncrc_mock.call_count)
  461             self.assertEqual(0, rti_mock.call_count)
  462             self.assertEqual(0, cd_mock.call_count)
  463             log_mock.assert_called_once_with(mock.ANY,
  464                                              {'node': self.node.uuid,
  465                                               'state': state})
  466 
  467     @mock.patch.object(agent_base.HeartbeatMixin, 'continue_deploy',
  468                        autospec=True)
  469     @mock.patch.object(agent_base.HeartbeatMixin,
  470                        'reboot_to_instance', autospec=True)
  471     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  472                        autospec=True)
  473     def test_heartbeat_noops_in_wrong_state2(self, ncrc_mock, rti_mock,
  474                                              cd_mock):
  475         CONF.set_override('allow_provisioning_in_maintenance', False,
  476                           group='conductor')
  477         allowed = {states.DEPLOYWAIT, states.CLEANWAIT}
  478         for state in set(states.machine.states) - allowed:
  479             for m in (ncrc_mock, rti_mock, cd_mock):
  480                 m.reset_mock()
  481             with task_manager.acquire(self.context, self.node.uuid,
  482                                       shared=True) as task:
  483                 self.node.provision_state = state
  484                 self.deploy.heartbeat(task, 'url', '1.0.0')
  485                 self.assertTrue(task.shared)
  486             self.assertEqual(0, ncrc_mock.call_count)
  487             self.assertEqual(0, rti_mock.call_count)
  488             self.assertEqual(0, cd_mock.call_count)
  489 
  490     @mock.patch.object(agent_base.HeartbeatMixin,
  491                        'in_core_deploy_step', autospec=True)
  492     @mock.patch.object(agent_base.HeartbeatMixin,
  493                        'deploy_has_started', autospec=True)
  494     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
  495     @mock.patch.object(agent_base.HeartbeatMixin, 'deploy_is_done',
  496                        autospec=True)
  497     @mock.patch.object(agent_base.LOG, 'exception', autospec=True)
  498     def test_heartbeat_deploy_done_fails(self, log_mock, done_mock,
  499                                          failed_mock, deploy_started_mock,
  500                                          in_deploy_mock):
  501         in_deploy_mock.return_value = True
  502         deploy_started_mock.return_value = True
  503         done_mock.side_effect = Exception('LlamaException')
  504         with task_manager.acquire(
  505                 self.context, self.node['uuid'], shared=False) as task:
  506             task.node.provision_state = states.DEPLOYWAIT
  507             task.node.target_provision_state = states.ACTIVE
  508             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  509             failed_mock.assert_called_once_with(
  510                 task, mock.ANY, collect_logs=True)
  511             log_mock.assert_called_once_with(
  512                 'Asynchronous exception for node %(node)s: %(err)s',
  513                 {'err': 'Failed checking if deploy is done. '
  514                  'Error: LlamaException',
  515                  'node': task.node.uuid})
  516 
  517     @mock.patch.object(agent_base.HeartbeatMixin,
  518                        'in_core_deploy_step', autospec=True)
  519     @mock.patch.object(agent_base.HeartbeatMixin,
  520                        'deploy_has_started', autospec=True)
  521     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
  522     @mock.patch.object(agent_base.HeartbeatMixin, 'deploy_is_done',
  523                        autospec=True)
  524     @mock.patch.object(agent_base.LOG, 'exception', autospec=True)
  525     def test_heartbeat_deploy_done_raises_with_event(self, log_mock, done_mock,
  526                                                      failed_mock,
  527                                                      deploy_started_mock,
  528                                                      in_deploy_mock):
  529         in_deploy_mock.return_value = True
  530         deploy_started_mock.return_value = True
  531         with task_manager.acquire(
  532                 self.context, self.node['uuid'], shared=False) as task:
  533 
  534             def driver_failure(*args, **kwargs):
  535                 # simulate driver failure that both advances the FSM
  536                 # and raises an exception
  537                 task.node.provision_state = states.DEPLOYFAIL
  538                 raise Exception('LlamaException')
  539 
  540             task.node.provision_state = states.DEPLOYWAIT
  541             task.node.target_provision_state = states.ACTIVE
  542             done_mock.side_effect = driver_failure
  543             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  544             # task.node.provision_state being set to DEPLOYFAIL
  545             # within the driver_failue, hearbeat should not call
  546             # deploy_utils.set_failed_state anymore
  547             self.assertFalse(failed_mock.called)
  548             log_mock.assert_called_once_with(
  549                 'Asynchronous exception for node %(node)s: %(err)s',
  550                 {'err': 'Failed checking if deploy is done. '
  551                  'Error: LlamaException',
  552                  'node': task.node.uuid})
  553 
  554     @mock.patch.object(objects.node.Node, 'touch_provisioning', autospec=True)
  555     @mock.patch.object(agent_base.HeartbeatMixin,
  556                        'refresh_steps', autospec=True)
  557     @mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
  558                        autospec=True)
  559     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  560                        autospec=True)
  561     def test_heartbeat_resume_clean(self, mock_notify, mock_set_steps,
  562                                     mock_refresh, mock_touch):
  563         self.node.clean_step = {}
  564         self.node.provision_state = states.CLEANWAIT
  565         self.node.save()
  566         with task_manager.acquire(
  567                 self.context, self.node.uuid, shared=False) as task:
  568             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  569 
  570         mock_touch.assert_called_once_with(mock.ANY)
  571         mock_refresh.assert_called_once_with(mock.ANY, task, 'clean')
  572         mock_notify.assert_called_once_with(task, 'clean')
  573         mock_set_steps.assert_called_once_with(task)
  574 
  575     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
  576     @mock.patch.object(objects.node.Node, 'touch_provisioning', autospec=True)
  577     @mock.patch.object(agent_base.HeartbeatMixin,
  578                        'refresh_steps', autospec=True)
  579     @mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
  580                        autospec=True)
  581     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
  582                        autospec=True)
  583     def test_heartbeat_resume_clean_fails(self, mock_notify, mock_set_steps,
  584                                           mock_refresh, mock_touch,
  585                                           mock_handler):
  586         mocks = [mock_refresh, mock_set_steps, mock_notify]
  587         self.node.clean_step = {}
  588         self.node.provision_state = states.CLEANWAIT
  589         self.node.save()
  590         for i in range(len(mocks)):
  591             before_failed_mocks = mocks[:i]
  592             failed_mock = mocks[i]
  593             after_failed_mocks = mocks[i + 1:]
  594             failed_mock.side_effect = Exception()
  595             with task_manager.acquire(
  596                     self.context, self.node.uuid, shared=False) as task:
  597                 self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  598 
  599             mock_touch.assert_called_once_with(mock.ANY)
  600             mock_handler.assert_called_once_with(task, mock.ANY, mock.ANY)
  601             for called in before_failed_mocks + [failed_mock]:
  602                 self.assertTrue(called.called)
  603             for not_called in after_failed_mocks:
  604                 self.assertFalse(not_called.called)
  605 
  606             # Reset mocks for the next interaction
  607             for m in mocks + [mock_touch, mock_handler]:
  608                 m.reset_mock()
  609             failed_mock.side_effect = None
  610 
  611     @mock.patch.object(objects.node.Node, 'touch_provisioning', autospec=True)
  612     @mock.patch.object(agent_base.HeartbeatMixin,
  613                        'continue_cleaning', autospec=True)
  614     def test_heartbeat_continue_cleaning(self, mock_continue, mock_touch):
  615         self.node.clean_step = {
  616             'priority': 10,
  617             'interface': 'deploy',
  618             'step': 'foo',
  619             'reboot_requested': False
  620         }
  621         self.node.provision_state = states.CLEANWAIT
  622         self.node.save()
  623         with task_manager.acquire(
  624                 self.context, self.node.uuid, shared=False) as task:
  625             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  626 
  627         mock_touch.assert_called_once_with(mock.ANY)
  628         mock_continue.assert_called_once_with(mock.ANY, task)
  629 
  630     @mock.patch.object(objects.node.Node, 'touch_provisioning', autospec=True)
  631     @mock.patch.object(agent_base.HeartbeatMixin,
  632                        'continue_cleaning', autospec=True)
  633     def test_heartbeat_continue_cleaning_polling(self, mock_continue,
  634                                                  mock_touch):
  635         info = self.node.driver_internal_info
  636         info['cleaning_polling'] = True
  637         self.node.driver_internal_info = info
  638         self.node.clean_step = {
  639             'priority': 10,
  640             'interface': 'deploy',
  641             'step': 'foo',
  642             'reboot_requested': False
  643         }
  644         self.node.provision_state = states.CLEANWAIT
  645         self.node.save()
  646         with task_manager.acquire(
  647                 self.context, self.node.uuid, shared=False) as task:
  648             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  649 
  650         mock_touch.assert_called_once_with(mock.ANY)
  651         self.assertFalse(mock_continue.called)
  652 
  653     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
  654     @mock.patch.object(agent_base.HeartbeatMixin,
  655                        'continue_cleaning', autospec=True)
  656     def test_heartbeat_continue_cleaning_fails(self, mock_continue,
  657                                                mock_handler):
  658         self.node.clean_step = {
  659             'priority': 10,
  660             'interface': 'deploy',
  661             'step': 'foo',
  662             'reboot_requested': False
  663         }
  664 
  665         mock_continue.side_effect = Exception()
  666 
  667         self.node.provision_state = states.CLEANWAIT
  668         self.node.save()
  669         with task_manager.acquire(
  670                 self.context, self.node.uuid, shared=False) as task:
  671             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  672 
  673         mock_continue.assert_called_once_with(mock.ANY, task)
  674         mock_handler.assert_called_once_with(task, mock.ANY, mock.ANY)
  675 
  676     @mock.patch.object(manager_utils, 'rescuing_error_handler', autospec=True)
  677     @mock.patch.object(agent_base.HeartbeatMixin, '_finalize_rescue',
  678                        autospec=True)
  679     def test_heartbeat_rescue(self, mock_finalize_rescue,
  680                               mock_rescue_err_handler):
  681         self.node.provision_state = states.RESCUEWAIT
  682         self.node.save()
  683         with task_manager.acquire(
  684                 self.context, self.node.uuid, shared=False) as task:
  685             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  686 
  687         mock_finalize_rescue.assert_called_once_with(mock.ANY, task)
  688         self.assertFalse(mock_rescue_err_handler.called)
  689 
  690     @mock.patch.object(manager_utils, 'rescuing_error_handler', autospec=True)
  691     @mock.patch.object(agent_base.HeartbeatMixin, '_finalize_rescue',
  692                        autospec=True)
  693     def test_heartbeat_rescue_fails(self, mock_finalize,
  694                                     mock_rescue_err_handler):
  695         self.node.provision_state = states.RESCUEWAIT
  696         self.node.save()
  697         mock_finalize.side_effect = Exception('some failure')
  698         with task_manager.acquire(
  699                 self.context, self.node.uuid, shared=False) as task:
  700             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '1.0.0')
  701 
  702         mock_finalize.assert_called_once_with(mock.ANY, task)
  703         mock_rescue_err_handler.assert_called_once_with(
  704             task, 'Node failed to perform '
  705             'rescue operation. Error: some failure')
  706 
  707     @mock.patch.object(agent_base.HeartbeatMixin,
  708                        'in_core_deploy_step', autospec=True)
  709     @mock.patch.object(objects.node.Node, 'touch_provisioning', autospec=True)
  710     @mock.patch.object(agent_base.HeartbeatMixin,
  711                        'deploy_has_started', autospec=True)
  712     def test_heartbeat_touch_provisioning_and_url_save(self,
  713                                                        mock_deploy_started,
  714                                                        mock_touch,
  715                                                        mock_in_deploy):
  716         mock_in_deploy.return_value = True
  717         mock_deploy_started.return_value = True
  718 
  719         self.node.provision_state = states.DEPLOYWAIT
  720         self.node.save()
  721         with task_manager.acquire(
  722                 self.context, self.node.uuid, shared=False) as task:
  723             self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '3.2.0')
  724             self.assertEqual('http://127.0.0.1:8080',
  725                              task.node.driver_internal_info['agent_url'])
  726             self.assertEqual('3.2.0',
  727                              task.node.driver_internal_info['agent_version'])
  728             self.assertIsNotNone(
  729                 task.node.driver_internal_info['agent_last_heartbeat'])
  730         mock_touch.assert_called_once_with(mock.ANY)
  731 
  732     @mock.patch.object(agent_base.LOG, 'error', autospec=True)
  733     def test_heartbeat_records_cleaning_deploying(self, log_mock):
  734         for provision_state in (states.CLEANING, states.DEPLOYING):
  735             self.node.driver_internal_info = {}
  736             self.node.provision_state = provision_state
  737             self.node.save()
  738             with task_manager.acquire(
  739                     self.context, self.node.uuid, shared=False) as task:
  740                 self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '3.2.0')
  741                 self.assertEqual('http://127.0.0.1:8080',
  742                                  task.node.driver_internal_info['agent_url'])
  743                 self.assertEqual('3.2.0',
  744                                  task.node.driver_internal_info[
  745                                      'agent_version'])
  746                 self.assertIsNotNone(
  747                     task.node.driver_internal_info['agent_last_heartbeat'])
  748                 self.assertEqual(provision_state, task.node.provision_state)
  749             self.assertFalse(log_mock.called)
  750 
  751     def test_heartbeat_records_fast_track(self):
  752         self.config(fast_track=True, group='deploy')
  753         for provision_state in [states.ENROLL, states.MANAGEABLE,
  754                                 states.AVAILABLE]:
  755             self.node.driver_internal_info = {}
  756             self.node.provision_state = provision_state
  757             self.node.save()
  758             with task_manager.acquire(
  759                     self.context, self.node.uuid, shared=False) as task:
  760                 self.deploy.heartbeat(task, 'http://127.0.0.1:8080', '3.2.0')
  761                 self.assertEqual('http://127.0.0.1:8080',
  762                                  task.node.driver_internal_info['agent_url'])
  763                 self.assertEqual('3.2.0',
  764                                  task.node.driver_internal_info[
  765                                      'agent_version'])
  766                 self.assertIsNotNone(
  767                     task.node.driver_internal_info['agent_last_heartbeat'])
  768                 self.assertEqual(provision_state, task.node.provision_state)
  769 
  770     def test_in_core_deploy_step(self):
  771         self.node.deploy_step = {
  772             'interface': 'deploy', 'step': 'deploy', 'priority': 100}
  773         info = self.node.driver_internal_info
  774         info['deploy_steps'] = [self.node.deploy_step]
  775         self.node.driver_internal_info = info
  776         self.node.save()
  777         with task_manager.acquire(
  778                 self.context, self.node.uuid, shared=False) as task:
  779             self.assertTrue(self.deploy.in_core_deploy_step(task))
  780 
  781     def test_in_core_deploy_step_in_other_step(self):
  782         self.node.deploy_step = {
  783             'interface': 'deploy', 'step': 'other-step', 'priority': 100}
  784         info = self.node.driver_internal_info
  785         info['deploy_steps'] = [self.node.deploy_step]
  786         self.node.driver_internal_info = info
  787         self.node.save()
  788         with task_manager.acquire(
  789                 self.context, self.node.uuid, shared=False) as task:
  790             self.assertFalse(self.deploy.in_core_deploy_step(task))
  791 
  792 
  793 class AgentRescueTests(AgentDeployMixinBaseTest):
  794 
  795     def setUp(self):
  796         super(AgentRescueTests, self).setUp()
  797 
  798     @mock.patch.object(agent.AgentRescue, 'clean_up',
  799                        spec_set=True, autospec=True)
  800     @mock.patch.object(agent_client.AgentClient, 'finalize_rescue',
  801                        spec=types.FunctionType)
  802     def test__finalize_rescue(self, mock_finalize_rescue,
  803                               mock_clean_up):
  804         node = self.node
  805         node.provision_state = states.RESCUEWAIT
  806         node.save()
  807         mock_finalize_rescue.return_value = {'command_status': 'SUCCEEDED'}
  808         with task_manager.acquire(self.context, self.node['uuid'],
  809                                   shared=False) as task:
  810             task.driver.network.configure_tenant_networks = mock.Mock()
  811             task.process_event = mock.Mock()
  812             self.deploy._finalize_rescue(task)
  813             mock_finalize_rescue.assert_called_once_with(task.node)
  814             task.process_event.assert_has_calls([mock.call('resume'),
  815                                                  mock.call('done')])
  816             mock_clean_up.assert_called_once_with(mock.ANY, task)
  817 
  818     @mock.patch.object(agent_client.AgentClient, 'finalize_rescue',
  819                        spec=types.FunctionType)
  820     def test__finalize_rescue_bad_command_result(self, mock_finalize_rescue):
  821         node = self.node
  822         node.provision_state = states.RESCUEWAIT
  823         node.save()
  824         mock_finalize_rescue.return_value = {'command_status': 'FAILED',
  825                                              'command_error': 'bad'}
  826         with task_manager.acquire(self.context, self.node['uuid'],
  827                                   shared=False) as task:
  828             self.assertRaises(exception.InstanceRescueFailure,
  829                               self.deploy._finalize_rescue, task)
  830             mock_finalize_rescue.assert_called_once_with(task.node)
  831 
  832     @mock.patch.object(agent_client.AgentClient, 'finalize_rescue',
  833                        spec=types.FunctionType)
  834     def test__finalize_rescue_exc(self, mock_finalize_rescue):
  835         node = self.node
  836         node.provision_state = states.RESCUEWAIT
  837         node.save()
  838         mock_finalize_rescue.side_effect = exception.IronicException("No pass")
  839         with task_manager.acquire(self.context, self.node['uuid'],
  840                                   shared=False) as task:
  841             self.assertRaises(exception.InstanceRescueFailure,
  842                               self.deploy._finalize_rescue, task)
  843             mock_finalize_rescue.assert_called_once_with(task.node)
  844 
  845     @mock.patch.object(agent_client.AgentClient, 'finalize_rescue',
  846                        spec=types.FunctionType)
  847     def test__finalize_rescue_missing_command_result(self,
  848                                                      mock_finalize_rescue):
  849         node = self.node
  850         node.provision_state = states.RESCUEWAIT
  851         node.save()
  852         mock_finalize_rescue.return_value = {}
  853         with task_manager.acquire(self.context, self.node['uuid'],
  854                                   shared=False) as task:
  855             self.assertRaises(exception.InstanceRescueFailure,
  856                               self.deploy._finalize_rescue, task)
  857             mock_finalize_rescue.assert_called_once_with(task.node)
  858 
  859     @mock.patch.object(manager_utils, 'restore_power_state_if_needed',
  860                        autospec=True)
  861     @mock.patch.object(manager_utils, 'power_on_node_if_needed',
  862                        autospec=True)
  863     @mock.patch.object(agent.AgentRescue, 'clean_up',
  864                        spec_set=True, autospec=True)
  865     @mock.patch.object(agent_client.AgentClient, 'finalize_rescue',
  866                        spec=types.FunctionType)
  867     def test__finalize_rescue_with_smartnic_port(
  868             self, mock_finalize_rescue, mock_clean_up,
  869             power_on_node_if_needed_mock, restore_power_state_mock):
  870         node = self.node
  871         node.provision_state = states.RESCUEWAIT
  872         node.save()
  873         mock_finalize_rescue.return_value = {'command_status': 'SUCCEEDED'}
  874         with task_manager.acquire(self.context, self.node['uuid'],
  875                                   shared=False) as task:
  876             task.driver.network.configure_tenant_networks = mock.Mock()
  877             task.process_event = mock.Mock()
  878             power_on_node_if_needed_mock.return_value = states.POWER_OFF
  879             self.deploy._finalize_rescue(task)
  880             mock_finalize_rescue.assert_called_once_with(task.node)
  881             task.process_event.assert_has_calls([mock.call('resume'),
  882                                                  mock.call('done')])
  883             mock_clean_up.assert_called_once_with(mock.ANY, task)
  884             power_on_node_if_needed_mock.assert_called_once_with(task)
  885             restore_power_state_mock.assert_called_once_with(
  886                 task, states.POWER_OFF)
  887 
  888 
  889 class AgentDeployMixinTest(AgentDeployMixinBaseTest):
  890     @mock.patch.object(manager_utils, 'power_on_node_if_needed', autospec=True)
  891     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
  892     @mock.patch.object(time, 'sleep', lambda seconds: None)
  893     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
  894     @mock.patch.object(fake.FakePower, 'get_power_state',
  895                        spec=types.FunctionType)
  896     @mock.patch.object(agent_client.AgentClient, 'power_off',
  897                        spec=types.FunctionType)
  898     def test_tear_down_agent(
  899             self, power_off_mock, get_power_state_mock,
  900             node_power_action_mock, collect_mock,
  901             power_on_node_if_needed_mock):
  902         cfg.CONF.set_override('deploy_logs_collect', 'always', 'agent')
  903         self.node.provision_state = states.DEPLOYING
  904         self.node.target_provision_state = states.ACTIVE
  905         self.node.save()
  906         with task_manager.acquire(self.context, self.node.uuid) as task:
  907             get_power_state_mock.side_effect = [states.POWER_ON,
  908                                                 states.POWER_OFF]
  909 
  910             power_on_node_if_needed_mock.return_value = None
  911             self.deploy.tear_down_agent(task)
  912             power_off_mock.assert_called_once_with(task.node)
  913             self.assertEqual(2, get_power_state_mock.call_count)
  914             self.assertFalse(node_power_action_mock.called)
  915             self.assertEqual(states.DEPLOYING, task.node.provision_state)
  916             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
  917             collect_mock.assert_called_once_with(task.node)
  918 
  919     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
  920     @mock.patch.object(time, 'sleep', lambda seconds: None)
  921     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
  922     @mock.patch.object(fake.FakePower, 'get_power_state',
  923                        spec=types.FunctionType)
  924     @mock.patch.object(agent_client.AgentClient, 'power_off',
  925                        spec=types.FunctionType)
  926     def test_tear_down_agent_soft_poweroff_doesnt_complete(
  927             self, power_off_mock, get_power_state_mock,
  928             node_power_action_mock, mock_collect):
  929         self.node.provision_state = states.DEPLOYING
  930         self.node.target_provision_state = states.ACTIVE
  931         self.node.save()
  932         with task_manager.acquire(self.context, self.node.uuid) as task:
  933             get_power_state_mock.return_value = states.POWER_ON
  934             self.deploy.tear_down_agent(task)
  935             power_off_mock.assert_called_once_with(task.node)
  936             self.assertEqual(7, get_power_state_mock.call_count)
  937             node_power_action_mock.assert_called_once_with(task,
  938                                                            states.POWER_OFF)
  939             self.assertEqual(states.DEPLOYING, task.node.provision_state)
  940             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
  941             self.assertFalse(mock_collect.called)
  942 
  943     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
  944     @mock.patch.object(time, 'sleep', lambda seconds: None)
  945     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
  946     @mock.patch.object(fake.FakePower, 'get_power_state',
  947                        spec=types.FunctionType)
  948     @mock.patch.object(agent_client.AgentClient, 'power_off',
  949                        spec=types.FunctionType)
  950     def test_tear_down_agent_soft_poweroff_fails(
  951             self, power_off_mock, get_power_state_mock, node_power_action_mock,
  952             mock_collect):
  953         power_off_mock.side_effect = RuntimeError("boom")
  954         self.node.provision_state = states.DEPLOYING
  955         self.node.target_provision_state = states.ACTIVE
  956         self.node.save()
  957         with task_manager.acquire(self.context, self.node.uuid) as task:
  958             get_power_state_mock.return_value = states.POWER_ON
  959             self.deploy.tear_down_agent(task)
  960             power_off_mock.assert_called_once_with(task.node)
  961             node_power_action_mock.assert_called_once_with(task,
  962                                                            states.POWER_OFF)
  963             self.assertEqual(states.DEPLOYING, task.node.provision_state)
  964             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
  965             self.assertFalse(mock_collect.called)
  966 
  967     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
  968     @mock.patch.object(time, 'sleep', lambda seconds: None)
  969     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
  970     @mock.patch.object(fake.FakePower, 'get_power_state',
  971                        spec=types.FunctionType)
  972     @mock.patch.object(agent_client.AgentClient, 'power_off',
  973                        spec=types.FunctionType)
  974     def test_tear_down_agent_soft_poweroff_race(
  975             self, power_off_mock, get_power_state_mock, node_power_action_mock,
  976             mock_collect):
  977         # Test the situation when soft power off works, but ironic doesn't
  978         # learn about it.
  979         power_off_mock.side_effect = RuntimeError("boom")
  980         self.node.provision_state = states.DEPLOYING
  981         self.node.target_provision_state = states.ACTIVE
  982         self.node.save()
  983         with task_manager.acquire(self.context, self.node.uuid) as task:
  984             get_power_state_mock.side_effect = [states.POWER_ON,
  985                                                 states.POWER_OFF]
  986             self.deploy.tear_down_agent(task)
  987             power_off_mock.assert_called_once_with(task.node)
  988             self.assertFalse(node_power_action_mock.called)
  989             self.assertEqual(states.DEPLOYING, task.node.provision_state)
  990             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
  991             self.assertFalse(mock_collect.called)
  992 
  993     @mock.patch.object(manager_utils, 'power_on_node_if_needed', autospec=True)
  994     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
  995     @mock.patch.object(time, 'sleep', lambda seconds: None)
  996     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
  997     @mock.patch.object(fake.FakePower, 'get_power_state',
  998                        spec=types.FunctionType)
  999     @mock.patch.object(agent_client.AgentClient, 'power_off',
 1000                        spec=types.FunctionType)
 1001     def test_tear_down_agent_get_power_state_fails(
 1002             self, power_off_mock, get_power_state_mock, node_power_action_mock,
 1003             mock_collect, power_on_node_if_needed_mock):
 1004         self.node.provision_state = states.DEPLOYING
 1005         self.node.target_provision_state = states.ACTIVE
 1006         self.node.save()
 1007         with task_manager.acquire(self.context, self.node.uuid) as task:
 1008             get_power_state_mock.side_effect = RuntimeError("boom")
 1009             power_on_node_if_needed_mock.return_value = None
 1010             self.deploy.tear_down_agent(task)
 1011             power_off_mock.assert_called_once_with(task.node)
 1012             self.assertEqual(7, get_power_state_mock.call_count)
 1013             node_power_action_mock.assert_called_once_with(task,
 1014                                                            states.POWER_OFF)
 1015             self.assertEqual(states.DEPLOYING, task.node.provision_state)
 1016             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1017             self.assertFalse(mock_collect.called)
 1018 
 1019     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1020     @mock.patch.object(time, 'sleep', lambda seconds: None)
 1021     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1022     @mock.patch.object(fake.FakePower, 'get_power_state',
 1023                        spec=types.FunctionType)
 1024     @mock.patch.object(agent_client.AgentClient, 'power_off',
 1025                        spec=types.FunctionType)
 1026     def test_tear_down_agent_power_off_fails(
 1027             self, power_off_mock, get_power_state_mock,
 1028             node_power_action_mock, mock_collect):
 1029         self.node.provision_state = states.DEPLOYING
 1030         self.node.target_provision_state = states.ACTIVE
 1031         self.node.save()
 1032         with task_manager.acquire(self.context, self.node.uuid) as task:
 1033             get_power_state_mock.return_value = states.POWER_ON
 1034             node_power_action_mock.side_effect = RuntimeError("boom")
 1035             self.assertRaises(exception.InstanceDeployFailure,
 1036                               self.deploy.tear_down_agent,
 1037                               task)
 1038             power_off_mock.assert_called_once_with(task.node)
 1039             self.assertEqual(7, get_power_state_mock.call_count)
 1040             node_power_action_mock.assert_called_with(task, states.POWER_OFF)
 1041             self.assertEqual(states.DEPLOYFAIL, task.node.provision_state)
 1042             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1043             mock_collect.assert_called_once_with(task.node)
 1044 
 1045     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1046     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1047     @mock.patch.object(agent_client.AgentClient, 'sync',
 1048                        spec=types.FunctionType)
 1049     def test_tear_down_agent_power_action_oob_power_off(
 1050             self, sync_mock, node_power_action_mock, mock_collect):
 1051         # Enable force power off
 1052         driver_info = self.node.driver_info
 1053         driver_info['deploy_forces_oob_reboot'] = True
 1054         self.node.driver_info = driver_info
 1055 
 1056         self.node.provision_state = states.DEPLOYING
 1057         self.node.target_provision_state = states.ACTIVE
 1058         self.node.save()
 1059         with task_manager.acquire(self.context, self.node.uuid) as task:
 1060             self.deploy.tear_down_agent(task)
 1061 
 1062             sync_mock.assert_called_once_with(task.node)
 1063             node_power_action_mock.assert_called_once_with(task,
 1064                                                            states.POWER_OFF)
 1065             self.assertEqual(states.DEPLOYING, task.node.provision_state)
 1066             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1067             self.assertFalse(mock_collect.called)
 1068 
 1069     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1070     @mock.patch.object(agent_base.LOG, 'warning', autospec=True)
 1071     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1072     @mock.patch.object(agent_client.AgentClient, 'sync',
 1073                        spec=types.FunctionType)
 1074     def test_tear_down_agent_power_action_oob_power_off_failed(
 1075             self, sync_mock, node_power_action_mock, log_mock, mock_collect):
 1076         # Enable force power off
 1077         driver_info = self.node.driver_info
 1078         driver_info['deploy_forces_oob_reboot'] = True
 1079         self.node.driver_info = driver_info
 1080 
 1081         self.node.provision_state = states.DEPLOYING
 1082         self.node.target_provision_state = states.ACTIVE
 1083         self.node.save()
 1084         with task_manager.acquire(self.context, self.node.uuid) as task:
 1085             log_mock.reset_mock()
 1086 
 1087             sync_mock.return_value = {'faultstring': 'Unknown command: blah'}
 1088             self.deploy.tear_down_agent(task)
 1089 
 1090             sync_mock.assert_called_once_with(task.node)
 1091             node_power_action_mock.assert_called_once_with(task,
 1092                                                            states.POWER_OFF)
 1093             self.assertEqual(states.DEPLOYING, task.node.provision_state)
 1094             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1095             log_error = ('The version of the IPA ramdisk used in the '
 1096                          'deployment do not support the command "sync"')
 1097             log_mock.assert_called_once_with(
 1098                 'Failed to flush the file system prior to hard rebooting the '
 1099                 'node %(node)s. Error: %(error)s',
 1100                 {'node': task.node.uuid, 'error': log_error})
 1101             self.assertFalse(mock_collect.called)
 1102 
 1103     @mock.patch.object(manager_utils, 'power_on_node_if_needed', autospec=True)
 1104     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1105     @mock.patch.object(time, 'sleep', lambda seconds: None)
 1106     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1107     @mock.patch.object(fake.FakePower, 'get_supported_power_states',
 1108                        lambda self, task: [states.REBOOT])
 1109     @mock.patch.object(agent_client.AgentClient, 'sync', autospec=True)
 1110     def test_tear_down_agent_no_power_on_support(
 1111             self, sync_mock, node_power_action_mock, collect_mock,
 1112             power_on_node_if_needed_mock):
 1113         cfg.CONF.set_override('deploy_logs_collect', 'always', 'agent')
 1114         self.node.provision_state = states.DEPLOYING
 1115         self.node.target_provision_state = states.ACTIVE
 1116         self.node.save()
 1117         with task_manager.acquire(self.context, self.node.uuid) as task:
 1118             self.deploy.tear_down_agent(task)
 1119             node_power_action_mock.assert_called_once_with(task, states.REBOOT)
 1120             self.assertEqual(states.DEPLOYING, task.node.provision_state)
 1121             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1122             collect_mock.assert_called_once_with(task.node)
 1123             self.assertFalse(power_on_node_if_needed_mock.called)
 1124             sync_mock.assert_called_once_with(self.deploy._client, task.node)
 1125 
 1126     @mock.patch.object(manager_utils, 'restore_power_state_if_needed',
 1127                        autospec=True)
 1128     @mock.patch.object(manager_utils, 'power_on_node_if_needed', autospec=True)
 1129     @mock.patch('ironic.drivers.modules.network.noop.NoopNetwork.'
 1130                 'remove_provisioning_network', spec_set=True, autospec=True)
 1131     @mock.patch('ironic.drivers.modules.network.noop.NoopNetwork.'
 1132                 'configure_tenant_networks', spec_set=True, autospec=True)
 1133     def test_switch_to_tenant_network(self, configure_tenant_net_mock,
 1134                                       remove_provisioning_net_mock,
 1135                                       power_on_node_if_needed_mock,
 1136                                       restore_power_state_mock):
 1137         power_on_node_if_needed_mock.return_value = states.POWER_OFF
 1138         self.node.provision_state = states.DEPLOYING
 1139         self.node.target_provision_state = states.ACTIVE
 1140         self.node.save()
 1141         with task_manager.acquire(self.context, self.node.uuid) as task:
 1142             self.deploy.switch_to_tenant_network(task)
 1143             remove_provisioning_net_mock.assert_called_once_with(mock.ANY,
 1144                                                                  task)
 1145             configure_tenant_net_mock.assert_called_once_with(mock.ANY, task)
 1146             power_on_node_if_needed_mock.assert_called_once_with(task)
 1147             restore_power_state_mock.assert_called_once_with(
 1148                 task, states.POWER_OFF)
 1149 
 1150     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1151     @mock.patch('ironic.drivers.modules.network.noop.NoopNetwork.'
 1152                 'remove_provisioning_network', spec_set=True, autospec=True)
 1153     @mock.patch('ironic.drivers.modules.network.noop.NoopNetwork.'
 1154                 'configure_tenant_networks', spec_set=True, autospec=True)
 1155     def test_switch_to_tenant_network_fails(self, configure_tenant_net_mock,
 1156                                             remove_provisioning_net_mock,
 1157                                             mock_collect):
 1158         self.node.provision_state = states.DEPLOYING
 1159         self.node.target_provision_state = states.ACTIVE
 1160         self.node.save()
 1161         with task_manager.acquire(self.context, self.node.uuid) as task:
 1162             configure_tenant_net_mock.side_effect = exception.NetworkError(
 1163                 "boom")
 1164             self.assertRaises(exception.InstanceDeployFailure,
 1165                               self.deploy.switch_to_tenant_network, task)
 1166             remove_provisioning_net_mock.assert_called_once_with(mock.ANY,
 1167                                                                  task)
 1168             configure_tenant_net_mock.assert_called_once_with(mock.ANY, task)
 1169             self.assertFalse(mock_collect.called)
 1170 
 1171     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1172     def test_boot_instance(self, node_power_action_mock):
 1173         self.node.provision_state = states.DEPLOYING
 1174         self.node.target_provision_state = states.ACTIVE
 1175         self.node.save()
 1176         with task_manager.acquire(self.context, self.node.uuid) as task:
 1177             self.deploy.boot_instance(task)
 1178             node_power_action_mock.assert_called_once_with(task,
 1179                                                            states.POWER_ON)
 1180 
 1181     @mock.patch.object(fake.FakePower, 'get_supported_power_states',
 1182                        lambda self, task: [states.REBOOT])
 1183     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1184     def test_boot_instance_no_power_on(self, node_power_action_mock):
 1185         self.node.provision_state = states.DEPLOYING
 1186         self.node.target_provision_state = states.ACTIVE
 1187         self.node.save()
 1188         with task_manager.acquire(self.context, self.node.uuid) as task:
 1189             self.deploy.boot_instance(task)
 1190             self.assertFalse(node_power_action_mock.called)
 1191 
 1192     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1193                        autospec=True)
 1194     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1195     @mock.patch.object(boot_mode_utils, 'get_boot_mode', autospec=True,
 1196                        return_value='whatever')
 1197     def test_configure_local_boot(self, boot_mode_mock,
 1198                                   try_set_boot_device_mock,
 1199                                   install_bootloader_mock):
 1200         install_bootloader_mock.return_value = {
 1201             'command_status': 'SUCCESS', 'command_error': None}
 1202         with task_manager.acquire(self.context, self.node['uuid'],
 1203                                   shared=False) as task:
 1204             task.node.driver_internal_info['is_whole_disk_image'] = False
 1205             self.deploy.configure_local_boot(task, root_uuid='some-root-uuid')
 1206             try_set_boot_device_mock.assert_called_once_with(
 1207                 task, boot_devices.DISK, persistent=True)
 1208             boot_mode_mock.assert_called_once_with(task.node)
 1209             install_bootloader_mock.assert_called_once_with(
 1210                 mock.ANY, task.node, root_uuid='some-root-uuid',
 1211                 efi_system_part_uuid=None, prep_boot_part_uuid=None,
 1212                 target_boot_mode='whatever', software_raid=False
 1213             )
 1214 
 1215     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1216                        autospec=True)
 1217     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1218     @mock.patch.object(boot_mode_utils, 'get_boot_mode', autospec=True,
 1219                        return_value='whatever')
 1220     def test_configure_local_boot_with_prep(self, boot_mode_mock,
 1221                                             try_set_boot_device_mock,
 1222                                             install_bootloader_mock):
 1223         install_bootloader_mock.return_value = {
 1224             'command_status': 'SUCCESS', 'command_error': None}
 1225 
 1226         with task_manager.acquire(self.context, self.node['uuid'],
 1227                                   shared=False) as task:
 1228             task.node.driver_internal_info['is_whole_disk_image'] = False
 1229             self.deploy.configure_local_boot(task, root_uuid='some-root-uuid',
 1230                                              prep_boot_part_uuid='fake-prep')
 1231             try_set_boot_device_mock.assert_called_once_with(
 1232                 task, boot_devices.DISK, persistent=True)
 1233             boot_mode_mock.assert_called_once_with(task.node)
 1234             install_bootloader_mock.assert_called_once_with(
 1235                 mock.ANY, task.node, root_uuid='some-root-uuid',
 1236                 efi_system_part_uuid=None, prep_boot_part_uuid='fake-prep',
 1237                 target_boot_mode='whatever', software_raid=False
 1238             )
 1239 
 1240     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1241                        autospec=True)
 1242     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1243     @mock.patch.object(boot_mode_utils, 'get_boot_mode', autospec=True,
 1244                        return_value='uefi')
 1245     def test_configure_local_boot_uefi(self, boot_mode_mock,
 1246                                        try_set_boot_device_mock,
 1247                                        install_bootloader_mock):
 1248         install_bootloader_mock.return_value = {
 1249             'command_status': 'SUCCESS', 'command_error': None}
 1250         with task_manager.acquire(self.context, self.node['uuid'],
 1251                                   shared=False) as task:
 1252             task.node.driver_internal_info['is_whole_disk_image'] = False
 1253             self.deploy.configure_local_boot(
 1254                 task, root_uuid='some-root-uuid',
 1255                 efi_system_part_uuid='efi-system-part-uuid')
 1256             try_set_boot_device_mock.assert_called_once_with(
 1257                 task, boot_devices.DISK, persistent=True)
 1258             boot_mode_mock.assert_called_once_with(task.node)
 1259             install_bootloader_mock.assert_called_once_with(
 1260                 mock.ANY, task.node, root_uuid='some-root-uuid',
 1261                 efi_system_part_uuid='efi-system-part-uuid',
 1262                 prep_boot_part_uuid=None,
 1263                 target_boot_mode='uefi', software_raid=False
 1264             )
 1265 
 1266     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1267     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1268                        autospec=True)
 1269     def test_configure_local_boot_whole_disk_image(
 1270             self, install_bootloader_mock, try_set_boot_device_mock):
 1271 
 1272         with task_manager.acquire(self.context, self.node['uuid'],
 1273                                   shared=False) as task:
 1274             self.deploy.configure_local_boot(task)
 1275             self.assertFalse(install_bootloader_mock.called)
 1276             try_set_boot_device_mock.assert_called_once_with(
 1277                 task, boot_devices.DISK, persistent=True)
 1278 
 1279     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1280     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1281                        autospec=True)
 1282     def test_configure_local_boot_no_root_uuid(
 1283             self, install_bootloader_mock, try_set_boot_device_mock):
 1284         with task_manager.acquire(self.context, self.node['uuid'],
 1285                                   shared=False) as task:
 1286             task.node.driver_internal_info['is_whole_disk_image'] = False
 1287             self.deploy.configure_local_boot(task)
 1288             self.assertFalse(install_bootloader_mock.called)
 1289             try_set_boot_device_mock.assert_called_once_with(
 1290                 task, boot_devices.DISK, persistent=True)
 1291 
 1292     @mock.patch.object(boot_mode_utils, 'get_boot_mode',
 1293                        autospec=True)
 1294     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1295     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1296                        autospec=True)
 1297     def test_configure_local_boot_no_root_uuid_whole_disk(
 1298             self, install_bootloader_mock, try_set_boot_device_mock,
 1299             boot_mode_mock):
 1300         with task_manager.acquire(self.context, self.node['uuid'],
 1301                                   shared=False) as task:
 1302             task.node.driver_internal_info['is_whole_disk_image'] = True
 1303             boot_mode_mock.return_value = 'uefi'
 1304             self.deploy.configure_local_boot(
 1305                 task, root_uuid=None,
 1306                 efi_system_part_uuid='efi-system-part-uuid')
 1307             install_bootloader_mock.assert_called_once_with(
 1308                 mock.ANY, task.node, root_uuid=None,
 1309                 efi_system_part_uuid='efi-system-part-uuid',
 1310                 prep_boot_part_uuid=None, target_boot_mode='uefi',
 1311                 software_raid=False)
 1312 
 1313     @mock.patch.object(image_service, 'GlanceImageService', autospec=True)
 1314     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1315     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1316                        autospec=True)
 1317     def test_configure_local_boot_on_software_raid(
 1318             self, install_bootloader_mock, try_set_boot_device_mock,
 1319             GlanceImageService_mock):
 1320         with task_manager.acquire(self.context, self.node['uuid'],
 1321                                   shared=False) as task:
 1322             task.node.driver_internal_info['is_whole_disk_image'] = True
 1323             task.node.target_raid_config = {
 1324                 "logical_disks": [
 1325                     {
 1326                         "size_gb": 100,
 1327                         "raid_level": "1",
 1328                         "controller": "software",
 1329                     },
 1330                     {
 1331                         "size_gb": 'MAX',
 1332                         "raid_level": "0",
 1333                         "controller": "software",
 1334                     }
 1335                 ]
 1336             }
 1337             self.deploy.configure_local_boot(task)
 1338             self.assertTrue(GlanceImageService_mock.called)
 1339             self.assertTrue(install_bootloader_mock.called)
 1340             try_set_boot_device_mock.assert_called_once_with(
 1341                 task, boot_devices.DISK, persistent=True)
 1342 
 1343     @mock.patch.object(image_service, 'GlanceImageService', autospec=True)
 1344     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1345     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1346                        autospec=True)
 1347     def test_configure_local_boot_on_software_raid_exception(
 1348             self, install_bootloader_mock, try_set_boot_device_mock,
 1349             GlanceImageService_mock):
 1350         GlanceImageService_mock.side_effect = Exception('Glance not found')
 1351         with task_manager.acquire(self.context, self.node['uuid'],
 1352                                   shared=False) as task:
 1353             task.node.driver_internal_info['is_whole_disk_image'] = True
 1354             root_uuid = "1efecf88-2b58-4d4e-8fbd-7bef1a40a1b0"
 1355             task.node.driver_internal_info['root_uuid_or_disk_id'] = root_uuid
 1356             task.node.target_raid_config = {
 1357                 "logical_disks": [
 1358                     {
 1359                         "size_gb": 100,
 1360                         "raid_level": "1",
 1361                         "controller": "software",
 1362                     },
 1363                     {
 1364                         "size_gb": 'MAX',
 1365                         "raid_level": "0",
 1366                         "controller": "software",
 1367                     }
 1368                 ]
 1369             }
 1370             self.deploy.configure_local_boot(task)
 1371             self.assertTrue(GlanceImageService_mock.called)
 1372             # check if the root_uuid comes from the driver_internal_info
 1373             install_bootloader_mock.assert_called_once_with(
 1374                 mock.ANY, task.node, root_uuid=root_uuid,
 1375                 efi_system_part_uuid=None, prep_boot_part_uuid=None,
 1376                 target_boot_mode='bios', software_raid=True)
 1377             try_set_boot_device_mock.assert_called_once_with(
 1378                 task, boot_devices.DISK, persistent=True)
 1379 
 1380     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1381     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1382                        autospec=True)
 1383     def test_configure_local_boot_on_non_software_raid(
 1384             self, install_bootloader_mock, try_set_boot_device_mock):
 1385         with task_manager.acquire(self.context, self.node['uuid'],
 1386                                   shared=False) as task:
 1387             task.node.driver_internal_info['is_whole_disk_image'] = False
 1388             task.node.target_raid_config = {
 1389                 "logical_disks": [
 1390                     {
 1391                         "size_gb": 100,
 1392                         "raid_level": "1",
 1393                     },
 1394                     {
 1395                         "size_gb": 'MAX',
 1396                         "raid_level": "0",
 1397                     }
 1398                 ]
 1399             }
 1400             self.deploy.configure_local_boot(task)
 1401             self.assertFalse(install_bootloader_mock.called)
 1402             try_set_boot_device_mock.assert_called_once_with(
 1403                 task, boot_devices.DISK, persistent=True)
 1404 
 1405     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1406     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1407                        autospec=True)
 1408     def test_configure_local_boot_enforce_persistent_boot_device_default(
 1409             self, install_bootloader_mock, try_set_boot_device_mock):
 1410         with task_manager.acquire(self.context, self.node['uuid'],
 1411                                   shared=False) as task:
 1412             driver_info = task.node.driver_info
 1413             driver_info['force_persistent_boot_device'] = 'Default'
 1414             task.node.driver_info = driver_info
 1415             driver_info['force_persistent_boot_device'] = 'Always'
 1416             task.node.driver_internal_info['is_whole_disk_image'] = False
 1417             self.deploy.configure_local_boot(task)
 1418             self.assertFalse(install_bootloader_mock.called)
 1419             try_set_boot_device_mock.assert_called_once_with(
 1420                 task, boot_devices.DISK, persistent=True)
 1421 
 1422     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1423     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1424                        autospec=True)
 1425     def test_configure_local_boot_enforce_persistent_boot_device_always(
 1426             self, install_bootloader_mock, try_set_boot_device_mock):
 1427         with task_manager.acquire(self.context, self.node['uuid'],
 1428                                   shared=False) as task:
 1429             driver_info = task.node.driver_info
 1430             driver_info['force_persistent_boot_device'] = 'Always'
 1431             task.node.driver_info = driver_info
 1432             task.node.driver_internal_info['is_whole_disk_image'] = False
 1433             self.deploy.configure_local_boot(task)
 1434             self.assertFalse(install_bootloader_mock.called)
 1435             try_set_boot_device_mock.assert_called_once_with(
 1436                 task, boot_devices.DISK, persistent=True)
 1437 
 1438     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1439     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1440                        autospec=True)
 1441     def test_configure_local_boot_enforce_persistent_boot_device_never(
 1442             self, install_bootloader_mock, try_set_boot_device_mock):
 1443         with task_manager.acquire(self.context, self.node['uuid'],
 1444                                   shared=False) as task:
 1445             driver_info = task.node.driver_info
 1446             driver_info['force_persistent_boot_device'] = 'Never'
 1447             task.node.driver_info = driver_info
 1448             task.node.driver_internal_info['is_whole_disk_image'] = False
 1449             self.deploy.configure_local_boot(task)
 1450             self.assertFalse(install_bootloader_mock.called)
 1451             try_set_boot_device_mock.assert_called_once_with(
 1452                 task, boot_devices.DISK, persistent=False)
 1453 
 1454     @mock.patch.object(agent_client.AgentClient, 'collect_system_logs',
 1455                        autospec=True)
 1456     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1457                        autospec=True)
 1458     @mock.patch.object(boot_mode_utils, 'get_boot_mode', autospec=True,
 1459                        return_value='whatever')
 1460     def test_configure_local_boot_boot_loader_install_fail(
 1461             self, boot_mode_mock, install_bootloader_mock,
 1462             collect_logs_mock):
 1463         install_bootloader_mock.return_value = {
 1464             'command_status': 'FAILED', 'command_error': 'boom'}
 1465         self.node.provision_state = states.DEPLOYING
 1466         self.node.target_provision_state = states.ACTIVE
 1467         self.node.save()
 1468         with task_manager.acquire(self.context, self.node['uuid'],
 1469                                   shared=False) as task:
 1470             task.node.driver_internal_info['is_whole_disk_image'] = False
 1471             self.assertRaises(exception.InstanceDeployFailure,
 1472                               self.deploy.configure_local_boot,
 1473                               task, root_uuid='some-root-uuid')
 1474             boot_mode_mock.assert_called_once_with(task.node)
 1475             install_bootloader_mock.assert_called_once_with(
 1476                 mock.ANY, task.node, root_uuid='some-root-uuid',
 1477                 efi_system_part_uuid=None, prep_boot_part_uuid=None,
 1478                 target_boot_mode='whatever', software_raid=False
 1479             )
 1480             collect_logs_mock.assert_called_once_with(mock.ANY, task.node)
 1481             self.assertEqual(states.DEPLOYFAIL, task.node.provision_state)
 1482             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1483 
 1484     @mock.patch.object(agent_client.AgentClient, 'collect_system_logs',
 1485                        autospec=True)
 1486     @mock.patch.object(deploy_utils, 'try_set_boot_device', autospec=True)
 1487     @mock.patch.object(agent_client.AgentClient, 'install_bootloader',
 1488                        autospec=True)
 1489     @mock.patch.object(boot_mode_utils, 'get_boot_mode', autospec=True,
 1490                        return_value='whatever')
 1491     def test_configure_local_boot_set_boot_device_fail(
 1492             self, boot_mode_mock, install_bootloader_mock,
 1493             try_set_boot_device_mock, collect_logs_mock):
 1494         install_bootloader_mock.return_value = {
 1495             'command_status': 'SUCCESS', 'command_error': None}
 1496         try_set_boot_device_mock.side_effect = RuntimeError('error')
 1497         self.node.provision_state = states.DEPLOYING
 1498         self.node.target_provision_state = states.ACTIVE
 1499         self.node.save()
 1500         with task_manager.acquire(self.context, self.node['uuid'],
 1501                                   shared=False) as task:
 1502             task.node.driver_internal_info['is_whole_disk_image'] = False
 1503             self.assertRaises(exception.InstanceDeployFailure,
 1504                               self.deploy.configure_local_boot,
 1505                               task, root_uuid='some-root-uuid',
 1506                               prep_boot_part_uuid=None)
 1507             boot_mode_mock.assert_called_once_with(task.node)
 1508             install_bootloader_mock.assert_called_once_with(
 1509                 mock.ANY, task.node, root_uuid='some-root-uuid',
 1510                 efi_system_part_uuid=None, prep_boot_part_uuid=None,
 1511                 target_boot_mode='whatever', software_raid=False)
 1512             try_set_boot_device_mock.assert_called_once_with(
 1513                 task, boot_devices.DISK, persistent=True)
 1514             collect_logs_mock.assert_called_once_with(mock.ANY, task.node)
 1515             self.assertEqual(states.DEPLOYFAIL, task.node.provision_state)
 1516             self.assertEqual(states.ACTIVE, task.node.target_provision_state)
 1517 
 1518     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
 1519     @mock.patch.object(pxe.PXEBoot, 'prepare_instance', autospec=True)
 1520     @mock.patch.object(deploy_utils, 'get_boot_option', autospec=True)
 1521     @mock.patch.object(agent_base.AgentDeployMixin,
 1522                        'configure_local_boot', autospec=True)
 1523     def test_prepare_instance_to_boot_netboot(self, configure_mock,
 1524                                               boot_option_mock,
 1525                                               prepare_instance_mock,
 1526                                               failed_state_mock):
 1527         boot_option_mock.return_value = 'netboot'
 1528         prepare_instance_mock.return_value = None
 1529         self.node.provision_state = states.DEPLOYING
 1530         self.node.target_provision_state = states.ACTIVE
 1531         self.node.save()
 1532         root_uuid = 'root_uuid'
 1533         efi_system_part_uuid = 'efi_sys_uuid'
 1534         with task_manager.acquire(self.context, self.node['uuid'],
 1535                                   shared=False) as task:
 1536             self.deploy.prepare_instance_to_boot(task, root_uuid,
 1537                                                  efi_system_part_uuid)
 1538             self.assertFalse(configure_mock.called)
 1539             boot_option_mock.assert_called_once_with(task.node)
 1540             prepare_instance_mock.assert_called_once_with(task.driver.boot,
 1541                                                           task)
 1542             self.assertFalse(failed_state_mock.called)
 1543 
 1544     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
 1545     @mock.patch.object(pxe.PXEBoot, 'prepare_instance', autospec=True)
 1546     @mock.patch.object(deploy_utils, 'get_boot_option', autospec=True)
 1547     @mock.patch.object(agent_base.AgentDeployMixin,
 1548                        'configure_local_boot', autospec=True)
 1549     def test_prepare_instance_to_boot_localboot(self, configure_mock,
 1550                                                 boot_option_mock,
 1551                                                 prepare_instance_mock,
 1552                                                 failed_state_mock):
 1553         boot_option_mock.return_value = 'local'
 1554         prepare_instance_mock.return_value = None
 1555         self.node.provision_state = states.DEPLOYING
 1556         self.node.target_provision_state = states.ACTIVE
 1557         self.node.save()
 1558         root_uuid = 'root_uuid'
 1559         efi_system_part_uuid = 'efi_sys_uuid'
 1560         with task_manager.acquire(self.context, self.node['uuid'],
 1561                                   shared=False) as task:
 1562             self.deploy.prepare_instance_to_boot(task, root_uuid,
 1563                                                  efi_system_part_uuid)
 1564             configure_mock.assert_called_once_with(
 1565                 self.deploy, task,
 1566                 root_uuid=root_uuid,
 1567                 efi_system_part_uuid=efi_system_part_uuid,
 1568                 prep_boot_part_uuid=None)
 1569             boot_option_mock.assert_called_once_with(task.node)
 1570             prepare_instance_mock.assert_called_once_with(task.driver.boot,
 1571                                                           task)
 1572             self.assertFalse(failed_state_mock.called)
 1573 
 1574     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
 1575     @mock.patch.object(pxe.PXEBoot, 'prepare_instance', autospec=True)
 1576     @mock.patch.object(deploy_utils, 'get_boot_option', autospec=True)
 1577     @mock.patch.object(agent_base.AgentDeployMixin,
 1578                        'configure_local_boot', autospec=True)
 1579     def test_prepare_instance_to_boot_localboot_prep_partition(
 1580             self, configure_mock, boot_option_mock,
 1581             prepare_instance_mock, failed_state_mock):
 1582         boot_option_mock.return_value = 'local'
 1583         prepare_instance_mock.return_value = None
 1584         self.node.provision_state = states.DEPLOYING
 1585         self.node.target_provision_state = states.ACTIVE
 1586         self.node.save()
 1587         root_uuid = 'root_uuid'
 1588         efi_system_part_uuid = 'efi_sys_uuid'
 1589         prep_boot_part_uuid = 'prep_boot_part_uuid'
 1590         with task_manager.acquire(self.context, self.node['uuid'],
 1591                                   shared=False) as task:
 1592             self.deploy.prepare_instance_to_boot(task, root_uuid,
 1593                                                  efi_system_part_uuid,
 1594                                                  prep_boot_part_uuid)
 1595             configure_mock.assert_called_once_with(
 1596                 self.deploy, task,
 1597                 root_uuid=root_uuid,
 1598                 efi_system_part_uuid=efi_system_part_uuid,
 1599                 prep_boot_part_uuid=prep_boot_part_uuid)
 1600             boot_option_mock.assert_called_once_with(task.node)
 1601             prepare_instance_mock.assert_called_once_with(task.driver.boot,
 1602                                                           task)
 1603             self.assertFalse(failed_state_mock.called)
 1604 
 1605     @mock.patch.object(deploy_utils, 'set_failed_state', autospec=True)
 1606     @mock.patch.object(pxe.PXEBoot, 'prepare_instance', autospec=True)
 1607     @mock.patch.object(deploy_utils, 'get_boot_option', autospec=True)
 1608     @mock.patch.object(agent_base.AgentDeployMixin,
 1609                        'configure_local_boot', autospec=True)
 1610     def test_prepare_instance_to_boot_configure_fails(self, configure_mock,
 1611                                                       boot_option_mock,
 1612                                                       prepare_mock,
 1613                                                       failed_state_mock):
 1614         boot_option_mock.return_value = 'local'
 1615         self.node.provision_state = states.DEPLOYING
 1616         self.node.target_provision_state = states.ACTIVE
 1617         self.node.save()
 1618         root_uuid = 'root_uuid'
 1619         efi_system_part_uuid = 'efi_sys_uuid'
 1620         reason = 'reason'
 1621         configure_mock.side_effect = (
 1622             exception.InstanceDeployFailure(reason=reason))
 1623         prepare_mock.side_effect = (
 1624             exception.InstanceDeployFailure(reason=reason))
 1625 
 1626         with task_manager.acquire(self.context, self.node['uuid'],
 1627                                   shared=False) as task:
 1628             self.assertRaises(exception.InstanceDeployFailure,
 1629                               self.deploy.prepare_instance_to_boot, task,
 1630                               root_uuid, efi_system_part_uuid)
 1631             configure_mock.assert_called_once_with(
 1632                 self.deploy, task,
 1633                 root_uuid=root_uuid,
 1634                 efi_system_part_uuid=efi_system_part_uuid,
 1635                 prep_boot_part_uuid=None)
 1636             boot_option_mock.assert_called_once_with(task.node)
 1637             self.assertFalse(prepare_mock.called)
 1638             self.assertFalse(failed_state_mock.called)
 1639 
 1640     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1641                        autospec=True)
 1642     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1643                        autospec=True)
 1644     def test_continue_cleaning(self, status_mock, notify_mock):
 1645         # Test a successful execute clean step on the agent
 1646         self.node.clean_step = {
 1647             'priority': 10,
 1648             'interface': 'deploy',
 1649             'step': 'erase_devices',
 1650             'reboot_requested': False
 1651         }
 1652         self.node.save()
 1653         status_mock.return_value = [{
 1654             'command_status': 'SUCCEEDED',
 1655             'command_name': 'execute_clean_step',
 1656             'command_result': {
 1657                 'clean_step': self.node.clean_step
 1658             }
 1659         }]
 1660         with task_manager.acquire(self.context, self.node['uuid'],
 1661                                   shared=False) as task:
 1662             self.deploy.continue_cleaning(task)
 1663             notify_mock.assert_called_once_with(task, 'clean')
 1664 
 1665     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1666     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1667                        autospec=True)
 1668     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1669     def test__post_step_reboot(self, mock_reboot, mock_prepare,
 1670                                mock_build_opt):
 1671         with task_manager.acquire(self.context, self.node['uuid'],
 1672                                   shared=False) as task:
 1673             i_info = task.node.driver_internal_info
 1674             i_info['agent_secret_token'] = 'magicvalue01'
 1675             task.node.driver_internal_info = i_info
 1676             agent_base._post_step_reboot(task, 'clean')
 1677             self.assertTrue(mock_build_opt.called)
 1678             self.assertTrue(mock_prepare.called)
 1679             mock_reboot.assert_called_once_with(task, states.REBOOT)
 1680             self.assertTrue(task.node.driver_internal_info['cleaning_reboot'])
 1681             self.assertNotIn('agent_secret_token',
 1682                              task.node.driver_internal_info)
 1683 
 1684     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1685     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1686                        autospec=True)
 1687     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1688     def test__post_step_reboot_deploy(self, mock_reboot, mock_prepare,
 1689                                       mock_build_opt):
 1690         with task_manager.acquire(self.context, self.node['uuid'],
 1691                                   shared=False) as task:
 1692             i_info = task.node.driver_internal_info
 1693             i_info['agent_secret_token'] = 'magicvalue01'
 1694             task.node.driver_internal_info = i_info
 1695             agent_base._post_step_reboot(task, 'deploy')
 1696             self.assertTrue(mock_build_opt.called)
 1697             self.assertTrue(mock_prepare.called)
 1698             mock_reboot.assert_called_once_with(task, states.REBOOT)
 1699             self.assertTrue(
 1700                 task.node.driver_internal_info['deployment_reboot'])
 1701             self.assertNotIn('agent_secret_token',
 1702                              task.node.driver_internal_info)
 1703 
 1704     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1705     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1706                        autospec=True)
 1707     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1708     def test__post_step_reboot_pregenerated_token(
 1709             self, mock_reboot, mock_prepare, mock_build_opt):
 1710         with task_manager.acquire(self.context, self.node['uuid'],
 1711                                   shared=False) as task:
 1712             i_info = task.node.driver_internal_info
 1713             i_info['agent_secret_token'] = 'magicvalue01'
 1714             i_info['agent_secret_token_pregenerated'] = True
 1715             task.node.driver_internal_info = i_info
 1716             agent_base._post_step_reboot(task, 'clean')
 1717             self.assertTrue(mock_build_opt.called)
 1718             self.assertTrue(mock_prepare.called)
 1719             mock_reboot.assert_called_once_with(task, states.REBOOT)
 1720             self.assertIn('agent_secret_token',
 1721                           task.node.driver_internal_info)
 1722 
 1723     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1724     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1725                        autospec=True)
 1726     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
 1727     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1728     def test__post_step_reboot_fail(self, mock_reboot, mock_handler,
 1729                                     mock_prepare, mock_build_opt):
 1730         mock_reboot.side_effect = RuntimeError("broken")
 1731 
 1732         with task_manager.acquire(self.context, self.node['uuid'],
 1733                                   shared=False) as task:
 1734             agent_base._post_step_reboot(task, 'clean')
 1735             mock_reboot.assert_called_once_with(task, states.REBOOT)
 1736             mock_handler.assert_called_once_with(task, mock.ANY,
 1737                                                  traceback=True)
 1738             self.assertNotIn('cleaning_reboot',
 1739                              task.node.driver_internal_info)
 1740 
 1741     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1742     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1743                        autospec=True)
 1744     @mock.patch.object(manager_utils, 'deploying_error_handler', autospec=True)
 1745     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1746     def test__post_step_reboot_fail_deploy(self, mock_reboot, mock_handler,
 1747                                            mock_prepare, mock_build_opt):
 1748         mock_reboot.side_effect = RuntimeError("broken")
 1749 
 1750         with task_manager.acquire(self.context, self.node['uuid'],
 1751                                   shared=False) as task:
 1752             agent_base._post_step_reboot(task, 'deploy')
 1753             mock_reboot.assert_called_once_with(task, states.REBOOT)
 1754             mock_handler.assert_called_once_with(task, mock.ANY,
 1755                                                  traceback=True)
 1756             self.assertNotIn('deployment_reboot',
 1757                              task.node.driver_internal_info)
 1758 
 1759     @mock.patch.object(deploy_utils, 'build_agent_options', autospec=True)
 1760     @mock.patch.object(pxe.PXEBoot, 'prepare_ramdisk', spec_set=True,
 1761                        autospec=True)
 1762     @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
 1763     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1764                        autospec=True)
 1765     def test_continue_cleaning_reboot(
 1766             self, status_mock, reboot_mock, mock_prepare, mock_build_opt):
 1767         # Test a successful execute clean step on the agent, with reboot
 1768         self.node.clean_step = {
 1769             'priority': 42,
 1770             'interface': 'deploy',
 1771             'step': 'reboot_me_afterwards',
 1772             'reboot_requested': True
 1773         }
 1774         self.node.save()
 1775         status_mock.return_value = [{
 1776             'command_status': 'SUCCEEDED',
 1777             'command_name': 'execute_clean_step',
 1778             'command_result': {
 1779                 'clean_step': self.node.clean_step
 1780             }
 1781         }]
 1782         with task_manager.acquire(self.context, self.node['uuid'],
 1783                                   shared=False) as task:
 1784             self.deploy.continue_cleaning(task)
 1785             reboot_mock.assert_called_once_with(task, states.REBOOT)
 1786 
 1787     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1788                        autospec=True)
 1789     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1790                        autospec=True)
 1791     def test_continue_cleaning_after_reboot(self, status_mock, notify_mock):
 1792         # Test a successful execute clean step on the agent, with reboot
 1793         self.node.clean_step = {
 1794             'priority': 42,
 1795             'interface': 'deploy',
 1796             'step': 'reboot_me_afterwards',
 1797             'reboot_requested': True
 1798         }
 1799         driver_internal_info = self.node.driver_internal_info
 1800         driver_internal_info['cleaning_reboot'] = True
 1801         self.node.driver_internal_info = driver_internal_info
 1802         self.node.save()
 1803         # Represents a freshly booted agent with no commands
 1804         status_mock.return_value = []
 1805 
 1806         with task_manager.acquire(self.context, self.node['uuid'],
 1807                                   shared=False) as task:
 1808             self.deploy.continue_cleaning(task)
 1809             notify_mock.assert_called_once_with(task, 'clean')
 1810             self.assertNotIn('cleaning_reboot',
 1811                              task.node.driver_internal_info)
 1812 
 1813     @mock.patch.object(agent_base,
 1814                        '_get_post_step_hook', autospec=True)
 1815     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1816                        autospec=True)
 1817     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1818                        autospec=True)
 1819     def test_continue_cleaning_with_hook(
 1820             self, status_mock, notify_mock, get_hook_mock):
 1821         self.node.clean_step = {
 1822             'priority': 10,
 1823             'interface': 'raid',
 1824             'step': 'create_configuration',
 1825         }
 1826         self.node.save()
 1827         command_status = {
 1828             'command_status': 'SUCCEEDED',
 1829             'command_name': 'execute_clean_step',
 1830             'command_result': {'clean_step': self.node.clean_step}}
 1831         status_mock.return_value = [command_status]
 1832         hook_mock = mock.MagicMock(spec=types.FunctionType, __name__='foo')
 1833         get_hook_mock.return_value = hook_mock
 1834         with task_manager.acquire(self.context, self.node.uuid,
 1835                                   shared=False) as task:
 1836             self.deploy.continue_cleaning(task)
 1837 
 1838             get_hook_mock.assert_called_once_with(task.node, 'clean')
 1839             hook_mock.assert_called_once_with(task, command_status)
 1840             notify_mock.assert_called_once_with(task, 'clean')
 1841 
 1842     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1843     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1844                        autospec=True)
 1845     @mock.patch.object(agent_base,
 1846                        '_get_post_step_hook', autospec=True)
 1847     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
 1848     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1849                        autospec=True)
 1850     def test_continue_cleaning_with_hook_fails(
 1851             self, status_mock, error_handler_mock, get_hook_mock,
 1852             notify_mock, collect_logs_mock):
 1853         self.node.clean_step = {
 1854             'priority': 10,
 1855             'interface': 'raid',
 1856             'step': 'create_configuration',
 1857         }
 1858         self.node.save()
 1859         command_status = {
 1860             'command_status': 'SUCCEEDED',
 1861             'command_name': 'execute_clean_step',
 1862             'command_result': {'clean_step': self.node.clean_step}}
 1863         status_mock.return_value = [command_status]
 1864         hook_mock = mock.MagicMock(spec=types.FunctionType, __name__='foo')
 1865         hook_mock.side_effect = RuntimeError('error')
 1866         get_hook_mock.return_value = hook_mock
 1867         with task_manager.acquire(self.context, self.node.uuid,
 1868                                   shared=False) as task:
 1869             self.deploy.continue_cleaning(task)
 1870 
 1871             get_hook_mock.assert_called_once_with(task.node, 'clean')
 1872             hook_mock.assert_called_once_with(task, command_status)
 1873             error_handler_mock.assert_called_once_with(task, mock.ANY,
 1874                                                        traceback=True)
 1875             self.assertFalse(notify_mock.called)
 1876             collect_logs_mock.assert_called_once_with(task.node,
 1877                                                       label='cleaning')
 1878 
 1879     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1880                        autospec=True)
 1881     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1882                        autospec=True)
 1883     def test_continue_cleaning_old_command(self, status_mock, notify_mock):
 1884         # Test when a second execute_clean_step happens to the agent, but
 1885         # the new step hasn't started yet.
 1886         self.node.clean_step = {
 1887             'priority': 10,
 1888             'interface': 'deploy',
 1889             'step': 'erase_devices',
 1890             'reboot_requested': False
 1891         }
 1892         self.node.save()
 1893         status_mock.return_value = [{
 1894             'command_status': 'SUCCEEDED',
 1895             'command_name': 'execute_clean_step',
 1896             'command_result': {
 1897                 'priority': 20,
 1898                 'interface': 'deploy',
 1899                 'step': 'update_firmware',
 1900                 'reboot_requested': False
 1901             }
 1902         }]
 1903         with task_manager.acquire(self.context, self.node['uuid'],
 1904                                   shared=False) as task:
 1905             self.deploy.continue_cleaning(task)
 1906             self.assertFalse(notify_mock.called)
 1907 
 1908     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1909                        autospec=True)
 1910     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1911                        autospec=True)
 1912     def test_continue_cleaning_running(self, status_mock, notify_mock):
 1913         # Test that no action is taken while a clean step is executing
 1914         status_mock.return_value = [{
 1915             'command_status': 'RUNNING',
 1916             'command_name': 'execute_clean_step',
 1917             'command_result': None
 1918         }]
 1919         with task_manager.acquire(self.context, self.node['uuid'],
 1920                                   shared=False) as task:
 1921             self.deploy.continue_cleaning(task)
 1922             self.assertFalse(notify_mock.called)
 1923 
 1924     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1925                        autospec=True)
 1926     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1927                        autospec=True)
 1928     def test_continue_cleaning_no_step_running(self, status_mock, notify_mock):
 1929         status_mock.return_value = [{
 1930             'command_status': 'SUCCEEDED',
 1931             'command_name': 'get_clean_steps',
 1932             'command_result': []
 1933         }]
 1934         with task_manager.acquire(self.context, self.node['uuid'],
 1935                                   shared=False) as task:
 1936             self.deploy.continue_cleaning(task)
 1937             notify_mock.assert_called_once_with(task, 'clean')
 1938 
 1939     @mock.patch.object(driver_utils, 'collect_ramdisk_logs', autospec=True)
 1940     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
 1941     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1942                        autospec=True)
 1943     def test_continue_cleaning_fail(self, status_mock, error_mock,
 1944                                     collect_logs_mock):
 1945         # Test that a failure puts the node in CLEANFAIL
 1946         status_mock.return_value = [{
 1947             'command_status': 'FAILED',
 1948             'command_name': 'execute_clean_step',
 1949             'command_result': {}
 1950         }]
 1951         with task_manager.acquire(self.context, self.node['uuid'],
 1952                                   shared=False) as task:
 1953             self.deploy.continue_cleaning(task)
 1954             error_mock.assert_called_once_with(task, mock.ANY, traceback=False)
 1955             collect_logs_mock.assert_called_once_with(task.node,
 1956                                                       label='cleaning')
 1957 
 1958     @mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
 1959                        autospec=True)
 1960     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 1961                        autospec=True)
 1962     @mock.patch.object(agent_base.AgentDeployMixin,
 1963                        'refresh_steps', autospec=True)
 1964     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 1965                        autospec=True)
 1966     def _test_continue_cleaning_clean_version_mismatch(
 1967             self, status_mock, refresh_steps_mock, notify_mock, steps_mock,
 1968             manual=False):
 1969         status_mock.return_value = [{
 1970             'command_status': 'CLEAN_VERSION_MISMATCH',
 1971             'command_name': 'execute_clean_step',
 1972         }]
 1973         tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
 1974         self.node.provision_state = states.CLEANWAIT
 1975         self.node.target_provision_state = tgt_prov_state
 1976         self.node.save()
 1977         with task_manager.acquire(self.context, self.node.uuid,
 1978                                   shared=False) as task:
 1979             self.deploy.continue_cleaning(task)
 1980             notify_mock.assert_called_once_with(task, 'clean')
 1981             refresh_steps_mock.assert_called_once_with(mock.ANY, task, 'clean')
 1982             if manual:
 1983                 self.assertFalse(
 1984                     task.node.driver_internal_info['skip_current_clean_step'])
 1985                 self.assertFalse(steps_mock.called)
 1986             else:
 1987                 steps_mock.assert_called_once_with(task)
 1988                 self.assertNotIn('skip_current_clean_step',
 1989                                  task.node.driver_internal_info)
 1990 
 1991     def test_continue_cleaning_automated_clean_version_mismatch(self):
 1992         self._test_continue_cleaning_clean_version_mismatch()
 1993 
 1994     def test_continue_cleaning_manual_clean_version_mismatch(self):
 1995         self._test_continue_cleaning_clean_version_mismatch(manual=True)
 1996 
 1997     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
 1998     @mock.patch.object(conductor_steps, 'set_node_cleaning_steps',
 1999                        autospec=True)
 2000     @mock.patch.object(manager_utils, 'notify_conductor_resume_operation',
 2001                        autospec=True)
 2002     @mock.patch.object(agent_base.AgentDeployMixin,
 2003                        'refresh_steps', autospec=True)
 2004     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 2005                        autospec=True)
 2006     def test_continue_cleaning_clean_version_mismatch_fail(
 2007             self, status_mock, refresh_steps_mock, notify_mock, steps_mock,
 2008             error_mock, manual=False):
 2009         status_mock.return_value = [{
 2010             'command_status': 'CLEAN_VERSION_MISMATCH',
 2011             'command_name': 'execute_clean_step',
 2012             'command_result': {'hardware_manager_version': {'Generic': '1'}}
 2013         }]
 2014         refresh_steps_mock.side_effect = exception.NodeCleaningFailure("boo")
 2015         tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
 2016         self.node.provision_state = states.CLEANWAIT
 2017         self.node.target_provision_state = tgt_prov_state
 2018         self.node.save()
 2019         with task_manager.acquire(self.context, self.node.uuid,
 2020                                   shared=False) as task:
 2021             self.deploy.continue_cleaning(task)
 2022 
 2023             status_mock.assert_called_once_with(mock.ANY, task.node)
 2024             refresh_steps_mock.assert_called_once_with(mock.ANY, task, 'clean')
 2025             error_mock.assert_called_once_with(task, mock.ANY, traceback=True)
 2026             self.assertFalse(notify_mock.called)
 2027             self.assertFalse(steps_mock.called)
 2028 
 2029     @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
 2030     @mock.patch.object(agent_client.AgentClient, 'get_commands_status',
 2031                        autospec=True)
 2032     def test_continue_cleaning_unknown(self, status_mock, error_mock):
 2033         # Test that unknown commands are treated as failures
 2034         status_mock.return_value = [{
 2035             'command_status': 'UNKNOWN',
 2036             'command_name': 'execute_clean_step',
 2037             'command_result': {}
 2038         }]
 2039         with task_manager.acquire(self.context, self.node['uuid'],
 2040                                   shared=False) as task:
 2041             self.deploy.continue_cleaning(task)
 2042             error_mock.assert_called_once_with(task, mock.ANY, traceback=False)
 2043 
 2044     def _test_clean_step_hook(self):
 2045         """Helper method for unit tests related to clean step hooks."""
 2046         some_function_mock = mock.MagicMock()
 2047 
 2048         @agent_base.post_clean_step_hook(
 2049             interface='raid', step='delete_configuration')
 2050         @agent_base.post_clean_step_hook(
 2051             interface='raid', step='create_configuration')
 2052         def hook_method():
 2053             some_function_mock('some-arguments')
 2054 
 2055         return hook_method
 2056 
 2057     @mock.patch.object(agent_base, '_POST_STEP_HOOKS',
 2058                        {'clean': {}, 'deploy': {}})
 2059     def test_post_clean_step_hook(self):
 2060         # This unit test makes sure that hook methods are registered
 2061         # properly and entries are made in
 2062         # agent_base.POST_CLEAN_STEP_HOOKS
 2063         hook_method = self._test_clean_step_hook()
 2064         hooks = agent_base._POST_STEP_HOOKS['clean']
 2065         self.assertEqual(hook_method, hooks['raid']['create_configuration'])
 2066         self.assertEqual(hook_method, hooks['raid']['delete_configuration'])
 2067 
 2068     @mock.patch.object(agent_base, '_POST_STEP_HOOKS',
 2069                        {'clean': {}, 'deploy': {}})
 2070     def test__get_post_step_hook(self):
 2071         # Check if agent_base._get_post_step_hook can get
 2072         # clean step for which hook is registered.
 2073         hook_method = self._test_clean_step_hook()
 2074         self.node.clean_step = {'step': 'create_configuration',
 2075                                 'interface': 'raid'}
 2076         self.node.save()
 2077         hook_returned = agent_base._get_post_step_hook(self.node, 'clean')
 2078         self.assertEqual(hook_method, hook_returned)
 2079 
 2080     @mock.patch.object(agent_base, '_POST_STEP_HOOKS',
 2081                        {'clean': {}, 'deploy': {}})
 2082     def test__get_post_step_hook_no_hook_registered(self):
 2083         # Make sure agent_base._get_post_step_hook returns
 2084         # None when no clean step hook is registered for the clean step.
 2085         self._test_clean_step_hook()
 2086         self.node.clean_step = {'step': 'some-clean-step',
 2087                                 'interface': 'some-other-interface'}
 2088         self.node.save()
 2089         hook_returned = agent_base._get_post_step_hook(self.node, 'clean')
 2090         self.assertIsNone(hook_returned)
 2091 
 2092 
 2093 class TestRefreshCleanSteps(AgentDeployMixinBaseTest):
 2094 
 2095     def setUp(self):
 2096         super(TestRefreshCleanSteps, self).setUp()
 2097         self.node.driver_internal_info['agent_url'] = 'http://127.0.0.1:9999'
 2098         self.ports = [object_utils.create_test_port(self.context,
 2099                                                     node_id=self.node.id)]
 2100 
 2101         self.clean_steps = {
 2102             'hardware_manager_version': '1',
 2103             'clean_steps': {
 2104                 'GenericHardwareManager': [
 2105                     {'interface': 'deploy',
 2106                      'step': 'erase_devices',
 2107                      'priority': 20},
 2108                 ],
 2109                 'SpecificHardwareManager': [
 2110                     {'interface': 'deploy',
 2111                      'step': 'update_firmware',
 2112                      'priority': 30},
 2113                     {'interface': 'raid',
 2114                      'step': 'create_configuration',
 2115                      'priority': 10},
 2116                 ]
 2117             }
 2118         }
 2119         # NOTE(dtantsur): deploy steps are structurally identical to clean
 2120         # steps, reusing self.clean_steps for simplicity
 2121         self.deploy_steps = {
 2122             'hardware_manager_version': '1',
 2123             'deploy_steps': self.clean_steps['clean_steps'],
 2124         }
 2125 
 2126     @mock.patch.object(agent_client.AgentClient, 'get_clean_steps',
 2127                        autospec=True)
 2128     def test_refresh_steps(self, client_mock):
 2129         client_mock.return_value = {
 2130             'command_result': self.clean_steps}
 2131 
 2132         with task_manager.acquire(
 2133                 self.context, self.node.uuid, shared=False) as task:
 2134             self.deploy.refresh_steps(task, 'clean')
 2135 
 2136             client_mock.assert_called_once_with(mock.ANY, task.node,
 2137                                                 task.ports)
 2138             self.assertEqual('1', task.node.driver_internal_info[
 2139                 'hardware_manager_version'])
 2140             self.assertIn('agent_cached_clean_steps_refreshed',
 2141                           task.node.driver_internal_info)
 2142             steps = task.node.driver_internal_info['agent_cached_clean_steps']
 2143             # Since steps are returned in dicts, they have non-deterministic
 2144             # ordering
 2145             self.assertEqual(2, len(steps))
 2146             self.assertIn(self.clean_steps['clean_steps'][
 2147                 'GenericHardwareManager'][0], steps['deploy'])
 2148             self.assertIn(self.clean_steps['clean_steps'][
 2149                 'SpecificHardwareManager'][0], steps['deploy'])
 2150             self.assertEqual([self.clean_steps['clean_steps'][
 2151                 'SpecificHardwareManager'][1]], steps['raid'])
 2152 
 2153     @mock.patch.object(agent_client.AgentClient, 'get_deploy_steps',
 2154                        autospec=True)
 2155     def test_refresh_steps_deploy(self, client_mock):
 2156         client_mock.return_value = {
 2157             'command_result': self.deploy_steps}
 2158 
 2159         with task_manager.acquire(
 2160                 self.context, self.node.uuid, shared=False) as task:
 2161             self.deploy.refresh_steps(task, 'deploy')
 2162 
 2163             client_mock.assert_called_once_with(mock.ANY, task.node,
 2164                                                 task.ports)
 2165             self.assertEqual('1', task.node.driver_internal_info[
 2166                 'hardware_manager_version'])
 2167             self.assertIn('agent_cached_deploy_steps_refreshed',
 2168                           task.node.driver_internal_info)
 2169             steps = task.node.driver_internal_info['agent_cached_deploy_steps']
 2170             self.assertEqual({'deploy', 'raid'}, set(steps))
 2171             # Since steps are returned in dicts, they have non-deterministic
 2172             # ordering
 2173             self.assertIn(self.clean_steps['clean_steps'][
 2174                 'GenericHardwareManager'][0], steps['deploy'])
 2175             self.assertIn(self.clean_steps['clean_steps'][
 2176                 'SpecificHardwareManager'][0], steps['deploy'])
 2177             self.assertEqual([self.clean_steps['clean_steps'][
 2178                 'SpecificHardwareManager'][1]], steps['raid'])
 2179 
 2180     @mock.patch.object(agent_base.LOG, 'warning', autospec=True)
 2181     @mock.patch.object(agent_client.AgentClient, 'get_deploy_steps',
 2182                        autospec=True)
 2183     def test_refresh_steps_busy(self, client_mock, log_mock):
 2184         client_mock.side_effect = exception.AgentAPIError(
 2185             node="node", status="500", error='agent is busy')
 2186 
 2187         with task_manager.acquire(
 2188                 self.context, self.node.uuid, shared=False) as task:
 2189             log_mock.reset_mock()
 2190             self.deploy.refresh_steps(task, 'deploy')
 2191 
 2192             client_mock.assert_called_once_with(mock.ANY, task.node,
 2193                                                 task.ports)
 2194             self.assertNotIn('agent_cached_deploy_steps_refreshed',
 2195                              task.node.driver_internal_info)
 2196             self.assertIsNone(task.node.driver_internal_info.get(
 2197                 'agent_cached_deploy_steps'))
 2198             log_mock.assert_not_called()
 2199 
 2200     @mock.patch.object(agent_client.AgentClient, 'get_clean_steps',
 2201                        autospec=True)
 2202     def test_refresh_steps_missing_steps(self, client_mock):
 2203         del self.clean_steps['clean_steps']
 2204         client_mock.return_value = {
 2205             'command_result': self.clean_steps}
 2206 
 2207         with task_manager.acquire(
 2208                 self.context, self.node.uuid, shared=False) as task:
 2209             self.assertRaisesRegex(exception.NodeCleaningFailure,
 2210                                    'invalid result',
 2211                                    self.deploy.refresh_steps,
 2212                                    task, 'clean')
 2213             client_mock.assert_called_once_with(mock.ANY, task.node,
 2214                                                 task.ports)
 2215 
 2216     @mock.patch.object(agent_client.AgentClient, 'get_clean_steps',
 2217                        autospec=True)
 2218     def test_refresh_steps_missing_interface(self, client_mock):
 2219         step = self.clean_steps['clean_steps']['SpecificHardwareManager'][1]
 2220         del step['interface']
 2221         client_mock.return_value = {
 2222             'command_result': self.clean_steps}
 2223 
 2224         with task_manager.acquire(
 2225                 self.context, self.node.uuid, shared=False) as task:
 2226             self.assertRaisesRegex(exception.NodeCleaningFailure,
 2227                                    'invalid clean step',
 2228                                    self.deploy.refresh_steps,
 2229                                    task, 'clean')
 2230             client_mock.assert_called_once_with(mock.ANY, task.node,
 2231                                                 task.ports)
 2232 
 2233 
 2234 class FakeAgentDeploy(agent_base.AgentDeployMixin, fake.FakeDeploy):
 2235     pass
 2236 
 2237 
 2238 class StepMethodsTestCase(db_base.DbTestCase):
 2239 
 2240     def setUp(self):
 2241         super(StepMethodsTestCase, self).setUp()
 2242 
 2243         self.clean_steps = {
 2244             'deploy': [
 2245                 {'interface': 'deploy',
 2246                  'step': 'erase_devices',
 2247                  'priority': 20},
 2248                 {'interface': 'deploy',
 2249                  'step': 'update_firmware',
 2250                  'priority': 30}
 2251             ],
 2252             'raid': [
 2253                 {'interface': 'raid',
 2254                  'step': 'create_configuration',
 2255                  'priority': 10}
 2256             ]
 2257         }
 2258         n = {'boot_interface': 'pxe',
 2259              'deploy_interface': 'direct',
 2260              'driver_internal_info': {
 2261                  'agent_cached_clean_steps': self.clean_steps}}
 2262         self.node = object_utils.create_test_node(self.context, **n)
 2263         self.ports = [object_utils.create_test_port(self.context,
 2264                                                     node_id=self.node.id)]
 2265         self.deploy = FakeAgentDeploy()
 2266 
 2267     def test_agent_get_steps(self):
 2268         with task_manager.acquire(
 2269                 self.context, self.node.uuid, shared=False) as task:
 2270             response = agent_base.get_steps(task, 'clean')
 2271 
 2272             # Since steps are returned in dicts, they have non-deterministic
 2273             # ordering
 2274             self.assertThat(response, matchers.HasLength(3))
 2275             self.assertIn(self.clean_steps['deploy'][0], response)
 2276             self.assertIn(self.clean_steps['deploy'][1], response)
 2277             self.assertIn(self.clean_steps['raid'][0], response)
 2278 
 2279     def test_agent_get_steps_deploy(self):
 2280         with task_manager.acquire(
 2281                 self.context, self.node.uuid, shared=False) as task:
 2282             task.node.driver_internal_info = {
 2283                 'agent_cached_deploy_steps': self.clean_steps
 2284             }
 2285             response = agent_base.get_steps(task, 'deploy')
 2286 
 2287             # Since steps are returned in dicts, they have non-deterministic
 2288             # ordering
 2289             self.assertThat(response, matchers.HasLength(3))
 2290             self.assertIn(self.clean_steps['deploy'][0], response)
 2291             self.assertIn(self.clean_steps['deploy'][1], response)
 2292             self.assertIn(self.clean_steps['raid'][0], response)
 2293 
 2294     def test_get_steps_custom_interface(self):
 2295         with task_manager.acquire(
 2296                 self.context, self.node.uuid, shared=False) as task:
 2297             response = agent_base.get_steps(task, 'clean', interface='raid')
 2298             self.assertThat(response, matchers.HasLength(1))
 2299             self.assertEqual(self.clean_steps['raid'], response)
 2300 
 2301     def test_get_steps_override_priorities(self):
 2302         with task_manager.acquire(
 2303                 self.context, self.node.uuid, shared=False) as task:
 2304             new_priorities = {'create_configuration': 42}
 2305             response = agent_base.get_steps(
 2306                 task, 'clean', interface='raid',
 2307                 override_priorities=new_priorities)
 2308             self.assertEqual(42, response[0]['priority'])
 2309 
 2310     def test_get_steps_override_priorities_none(self):
 2311         with task_manager.acquire(
 2312                 self.context, self.node.uuid, shared=False) as task:
 2313             # this is simulating the default value of a configuration option
 2314             new_priorities = {'create_configuration': None}
 2315             response = agent_base.get_steps(
 2316                 task, 'clean', interface='raid',
 2317                 override_priorities=new_priorities)
 2318             self.assertEqual(10, response[0]['priority'])
 2319 
 2320     def test_get_steps_missing_steps(self):
 2321         info = self.node.driver_internal_info
 2322         del info['agent_cached_clean_steps']
 2323         self.node.driver_internal_info = info
 2324         self.node.save()
 2325         with task_manager.acquire(
 2326                 self.context, self.node.uuid, shared=False) as task:
 2327             self.assertEqual([], agent_base.get_steps(task, 'clean'))
 2328 
 2329     def test_find_step(self):
 2330         with task_manager.acquire(
 2331                 self.context, self.node.uuid, shared=False) as task:
 2332             step = agent_base.find_step(task, 'clean', 'deploy',
 2333                                         'erase_devices')
 2334             self.assertEqual(self.clean_steps['deploy'][0], step)
 2335 
 2336     def test_find_step_not_found(self):
 2337         with task_manager.acquire(
 2338                 self.context, self.node.uuid, shared=False) as task:
 2339             self.assertIsNone(agent_base.find_step(
 2340                 task, 'clean', 'non-deploy', 'erase_devices'))
 2341             self.assertIsNone(agent_base.find_step(
 2342                 task, 'clean', 'deploy', 'something_else'))
 2343             self.assertIsNone(agent_base.find_step(
 2344                 task, 'deploy', 'deploy', 'erase_devices'))
 2345 
 2346     def test_get_deploy_steps(self):
 2347         with task_manager.acquire(
 2348                 self.context, self.node.uuid, shared=False) as task:
 2349             task.node.driver_internal_info = {
 2350                 'agent_cached_deploy_steps': self.clean_steps
 2351             }
 2352             steps = self.deploy.get_deploy_steps(task)
 2353             # 2 in-band steps + 3 out-of-band
 2354             expected = [
 2355                 {'step': 'deploy', 'priority': 100, 'argsinfo': None,
 2356                  'interface': 'deploy'},
 2357                 {'step': 'tear_down_agent', 'priority': 40, 'argsinfo': None,
 2358                  'interface': 'deploy'},
 2359                 {'step': 'switch_to_tenant_network', 'priority': 30,
 2360                  'argsinfo': None, 'interface': 'deploy'},
 2361                 {'step': 'boot_instance', 'priority': 20, 'argsinfo': None,
 2362                  'interface': 'deploy'},
 2363             ] + self.clean_steps['deploy']
 2364             self.assertCountEqual(expected, steps)
 2365 
 2366     def test_get_deploy_steps_only_oob(self):
 2367         with task_manager.acquire(
 2368                 self.context, self.node.uuid, shared=False) as task:
 2369             steps = self.deploy.get_deploy_steps(task)
 2370             # three base out-of-band steps
 2371             expected = [
 2372                 {'step': 'deploy', 'priority': 100, 'argsinfo': None,
 2373                  'interface': 'deploy'},
 2374                 {'step': 'tear_down_agent', 'priority': 40, 'argsinfo': None,
 2375                  'interface': 'deploy'},
 2376                 {'step': 'switch_to_tenant_network', 'priority': 30,
 2377                  'argsinfo': None, 'interface': 'deploy'},
 2378                 {'step': 'boot_instance', 'priority': 20, 'argsinfo': None,
 2379                  'interface': 'deploy'},
 2380             ]
 2381             self.assertCountEqual(expected, steps)
 2382 
 2383     @mock.patch('ironic.objects.Port.list_by_node_id',
 2384                 spec_set=types.FunctionType)
 2385     @mock.patch.object(agent_client.AgentClient, 'execute_clean_step',
 2386                        autospec=True)
 2387     def test_execute_clean_step(self, client_mock, list_ports_mock):
 2388         client_mock.return_value = {
 2389             'command_status': 'SUCCEEDED'}
 2390         list_ports_mock.return_value = self.ports
 2391 
 2392         with task_manager.acquire(
 2393                 self.context, self.node.uuid, shared=False) as task:
 2394             response = agent_base.execute_step(
 2395                 task, self.clean_steps['deploy'][0], 'clean')
 2396             self.assertEqual(states.CLEANWAIT, response)
 2397 
 2398     @mock.patch('ironic.objects.Port.list_by_node_id',
 2399                 spec_set=types.FunctionType)
 2400     @mock.patch.object(agent_client.AgentClient, 'execute_deploy_step',
 2401                        autospec=True)
 2402     def test_execute_deploy_step(self, client_mock, list_ports_mock):
 2403         client_mock.return_value = {
 2404             'command_status': 'SUCCEEDED'}
 2405         list_ports_mock.return_value = self.ports
 2406 
 2407         with task_manager.acquire(
 2408                 self.context, self.node.uuid, shared=False) as task:
 2409             response = agent_base.execute_step(
 2410                 task, self.clean_steps['deploy'][0], 'deploy')
 2411             self.assertEqual(states.DEPLOYWAIT, response)
 2412 
 2413     @mock.patch('ironic.objects.Port.list_by_node_id',
 2414                 spec_set=types.FunctionType)
 2415     @mock.patch.object(agent_client.AgentClient, 'execute_clean_step',
 2416                        autospec=True)
 2417     def test_execute_clean_step_running(self, client_mock, list_ports_mock):
 2418         client_mock.return_value = {
 2419             'command_status': 'RUNNING'}
 2420         list_ports_mock.return_value = self.ports
 2421 
 2422         with task_manager.acquire(
 2423                 self.context, self.node.uuid, shared=False) as task:
 2424             response = agent_base.execute_step(
 2425                 task, self.clean_steps['deploy'][0], 'clean')
 2426             self.assertEqual(states.CLEANWAIT, response)
 2427 
 2428     @mock.patch('ironic.objects.Port.list_by_node_id',
 2429                 spec_set=types.FunctionType)
 2430     @mock.patch.object(agent_client.AgentClient, 'execute_clean_step',
 2431                        autospec=True)
 2432     def test_execute_clean_step_version_mismatch(
 2433             self, client_mock, list_ports_mock):
 2434         client_mock.return_value = {
 2435             'command_status': 'RUNNING'}
 2436         list_ports_mock.return_value = self.ports
 2437 
 2438         with task_manager.acquire(
 2439                 self.context, self.node.uuid, shared=False) as task:
 2440             response = agent_base.execute_step(
 2441                 task, self.clean_steps['deploy'][0], 'clean')
 2442             self.assertEqual(states.CLEANWAIT, response)