"Fossies" - the Fresh Open Source Software Archive

Member "swift-2.21.0/test/unit/cli/test_relinker.py" (25 Mar 2019, 6959 Bytes) of package /linux/misc/openstack/swift-2.21.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_relinker.py": 2.19.1_vs_2.21.0.

    1 # Licensed under the Apache License, Version 2.0 (the "License");
    2 # you may not use this file except in compliance with the License.
    3 # You may obtain a copy of the License at
    4 #
    5 #    http://www.apache.org/licenses/LICENSE-2.0
    6 #
    7 # Unless required by applicable law or agreed to in writing, software
    8 # distributed under the License is distributed on an "AS IS" BASIS,
    9 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
   10 # implied.
   11 # See the License for the specific language governing permissions and
   12 # limitations under the License.
   13 
   14 import binascii
   15 import os
   16 import shutil
   17 import struct
   18 import tempfile
   19 import unittest
   20 
   21 from swift.cli import relinker
   22 from swift.common import exceptions, ring, utils
   23 from swift.common import storage_policy
   24 from swift.common.storage_policy import (
   25     StoragePolicy, StoragePolicyCollection, POLICIES, ECStoragePolicy)
   26 
   27 from swift.obj.diskfile import write_metadata
   28 
   29 from test.unit import FakeLogger, skip_if_no_xattrs, DEFAULT_TEST_EC_TYPE, \
   30     patch_policies
   31 
   32 
   33 class TestRelinker(unittest.TestCase):
   34     def setUp(self):
   35         skip_if_no_xattrs()
   36         self.logger = FakeLogger()
   37         self.testdir = tempfile.mkdtemp()
   38         self.devices = os.path.join(self.testdir, 'node')
   39         shutil.rmtree(self.testdir, ignore_errors=1)
   40         os.mkdir(self.testdir)
   41         os.mkdir(self.devices)
   42 
   43         self.rb = ring.RingBuilder(8, 6.0, 1)
   44 
   45         for i in range(6):
   46             ip = "127.0.0.%s" % i
   47             self.rb.add_dev({'id': i, 'region': 0, 'zone': 0, 'weight': 1,
   48                              'ip': ip, 'port': 10000, 'device': 'sda1'})
   49         self.rb.rebalance(seed=1)
   50 
   51         self.existing_device = 'sda1'
   52         os.mkdir(os.path.join(self.devices, self.existing_device))
   53         self.objects = os.path.join(self.devices, self.existing_device,
   54                                     'objects')
   55         os.mkdir(self.objects)
   56         self._hash = utils.hash_path('a/c/o')
   57         digest = binascii.unhexlify(self._hash)
   58         part = struct.unpack_from('>I', digest)[0] >> 24
   59         self.next_part = struct.unpack_from('>I', digest)[0] >> 23
   60         self.objdir = os.path.join(
   61             self.objects, str(part), self._hash[-3:], self._hash)
   62         os.makedirs(self.objdir)
   63         self.object_fname = "1278553064.00000.data"
   64         self.objname = os.path.join(self.objdir, self.object_fname)
   65         with open(self.objname, "wb") as dummy:
   66             dummy.write(b"Hello World!")
   67             write_metadata(dummy, {'name': '/a/c/o', 'Content-Length': '12'})
   68 
   69         test_policies = [StoragePolicy(0, 'platin', True)]
   70         storage_policy._POLICIES = StoragePolicyCollection(test_policies)
   71 
   72         self.expected_dir = os.path.join(
   73             self.objects, str(self.next_part), self._hash[-3:], self._hash)
   74         self.expected_file = os.path.join(self.expected_dir, self.object_fname)
   75 
   76     def _save_ring(self):
   77         rd = self.rb.get_ring()
   78         for policy in POLICIES:
   79             rd.save(os.path.join(
   80                 self.testdir, '%s.ring.gz' % policy.ring_name))
   81             # Enforce ring reloading in relinker
   82             policy.object_ring = None
   83 
   84     def tearDown(self):
   85         shutil.rmtree(self.testdir, ignore_errors=1)
   86         storage_policy.reload_storage_policies()
   87 
   88     def test_relink(self):
   89         self.rb.prepare_increase_partition_power()
   90         self._save_ring()
   91         relinker.relink(self.testdir, self.devices, True)
   92 
   93         self.assertTrue(os.path.isdir(self.expected_dir))
   94         self.assertTrue(os.path.isfile(self.expected_file))
   95 
   96         stat_old = os.stat(os.path.join(self.objdir, self.object_fname))
   97         stat_new = os.stat(self.expected_file)
   98         self.assertEqual(stat_old.st_ino, stat_new.st_ino)
   99 
  100     def _common_test_cleanup(self, relink=True):
  101         # Create a ring that has prev_part_power set
  102         self.rb.prepare_increase_partition_power()
  103         self.rb.increase_partition_power()
  104         self._save_ring()
  105 
  106         os.makedirs(self.expected_dir)
  107 
  108         if relink:
  109             # Create a hardlink to the original object name. This is expected
  110             # after a normal relinker run
  111             os.link(os.path.join(self.objdir, self.object_fname),
  112                     self.expected_file)
  113 
  114     def test_cleanup(self):
  115         self._common_test_cleanup()
  116         self.assertEqual(0, relinker.cleanup(self.testdir, self.devices, True))
  117 
  118         # Old objectname should be removed, new should still exist
  119         self.assertTrue(os.path.isdir(self.expected_dir))
  120         self.assertTrue(os.path.isfile(self.expected_file))
  121         self.assertFalse(os.path.isfile(
  122             os.path.join(self.objdir, self.object_fname)))
  123 
  124     def test_cleanup_not_yet_relinked(self):
  125         self._common_test_cleanup(relink=False)
  126         self.assertEqual(1, relinker.cleanup(self.testdir, self.devices, True))
  127 
  128         self.assertTrue(os.path.isfile(
  129             os.path.join(self.objdir, self.object_fname)))
  130 
  131     def test_cleanup_deleted(self):
  132         self._common_test_cleanup()
  133 
  134         # Pretend the object got deleted inbetween and there is a tombstone
  135         fname_ts = self.expected_file[:-4] + "ts"
  136         os.rename(self.expected_file, fname_ts)
  137 
  138         self.assertEqual(0, relinker.cleanup(self.testdir, self.devices, True))
  139 
  140     def test_cleanup_doesnotexist(self):
  141         self._common_test_cleanup()
  142 
  143         # Pretend the file in the new place got deleted inbetween
  144         os.remove(self.expected_file)
  145 
  146         self.assertEqual(
  147             1, relinker.cleanup(self.testdir, self.devices, True, self.logger))
  148         self.assertEqual(self.logger.get_lines_for_level('warning'),
  149                          ['Error cleaning up %s: %s' % (self.objname,
  150                           repr(exceptions.DiskFileNotExist()))])
  151 
  152     @patch_policies(
  153         [ECStoragePolicy(
  154          0, name='platin', is_default=True, ec_type=DEFAULT_TEST_EC_TYPE,
  155          ec_ndata=4, ec_nparity=2)])
  156     def test_cleanup_non_durable_fragment(self):
  157         self._common_test_cleanup()
  158 
  159         # Switch the policy type so that actually all fragments are non-durable
  160         # and raise a DiskFileNotExist in EC in this test. However, if the
  161         # counterpart exists in the new location, this is ok - it will be fixed
  162         # by the reconstructor later on
  163         self.assertEqual(
  164             0, relinker.cleanup(self.testdir, self.devices, True,
  165                                 self.logger))
  166         self.assertEqual(self.logger.get_lines_for_level('warning'), [])
  167 
  168     def test_cleanup_quarantined(self):
  169         self._common_test_cleanup()
  170         # Pretend the object in the new place got corrupted
  171         with open(self.expected_file, "wb") as obj:
  172             obj.write(b'trash')
  173 
  174         self.assertEqual(
  175             1, relinker.cleanup(self.testdir, self.devices, True, self.logger))
  176 
  177         self.assertIn('failed audit and was quarantined',
  178                       self.logger.get_lines_for_level('warning')[0])