"Fossies" - the Fresh Open Source Software Archive

Member "cloudkitty-9.0.0/cloudkitty/tests/storage/v2/test_storage_unit.py" (10 Apr 2019, 12273 Bytes) of package /linux/misc/openstack/cloudkitty-9.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file.

    1 # Copyright 2018 Objectif Libre
    2 #
    3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    4 #    not use this file except in compliance with the License. You may obtain
    5 #    a copy of the License at
    6 #
    7 #         http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 #    Unless required by applicable law or agreed to in writing, software
   10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   12 #    License for the specific language governing permissions and limitations
   13 #    under the License.
   14 #
   15 # @author: Luka Peschke
   16 #
   17 import datetime
   18 
   19 import mock
   20 import testscenarios
   21 
   22 from cloudkitty import storage
   23 from cloudkitty.tests import samples
   24 from cloudkitty.tests.storage.v2 import influx_utils
   25 from cloudkitty.tests import TestCase
   26 from cloudkitty.tests import utils as test_utils
   27 
   28 
   29 class StorageUnitTest(TestCase):
   30 
   31     storage_scenarios = [
   32         ('influx', dict(storage_backend='influxdb'))]
   33 
   34     @classmethod
   35     def generate_scenarios(cls):
   36         cls.scenarios = testscenarios.multiply_scenarios(
   37             cls.scenarios,
   38             cls.storage_scenarios)
   39 
   40     @mock.patch('cloudkitty.storage.v2.influx.InfluxClient',
   41                 new=influx_utils.FakeInfluxClient)
   42     @mock.patch('cloudkitty.utils.load_conf', new=test_utils.load_conf)
   43     def setUp(self):
   44         super(StorageUnitTest, self).setUp()
   45         self._project_id = samples.TENANT
   46         self._other_project_id = samples.OTHER_TENANT
   47         self.conf.set_override('backend', self.storage_backend, 'storage')
   48         self.conf.set_override('version', '2', 'storage')
   49         self.storage = storage.get_storage(conf=test_utils.load_conf())
   50         self.storage.init()
   51         self.data = []
   52         self.init_data()
   53 
   54     def init_data(self):
   55         project_ids = [self._project_id, self._other_project_id]
   56         for i in range(3):
   57             start_delta = 3600 * i
   58             end_delta = start_delta + 3600
   59             start = datetime.datetime(2018, 1, 1) \
   60                 + datetime.timedelta(seconds=start_delta)
   61             end = datetime.datetime(2018, 1, 1) \
   62                 + datetime.timedelta(seconds=end_delta)
   63             data = test_utils.generate_v2_storage_data(
   64                 project_ids=project_ids,
   65                 start=start,
   66                 end=end)
   67             self.data.append(data)
   68             self.storage.push([data])
   69 
   70     @staticmethod
   71     def _expected_total_qty_len(data, project_id=None, types=None):
   72         total = 0
   73         qty = 0
   74         length = 0
   75         for data_part in data:
   76             for mtype, usage_part in data_part['usage'].items():
   77                 if types is not None and mtype not in types:
   78                     continue
   79                 for item in usage_part:
   80                     if project_id is None or \
   81                        project_id == item['groupby']['project_id']:
   82                         total += item['rating']['price']
   83                         qty += item['vol']['qty']
   84                         length += 1
   85 
   86         return round(float(total), 5), round(float(qty), 5), length
   87 
   88     def _compare_get_total_result_with_expected(self,
   89                                                 expected_qty,
   90                                                 expected_total,
   91                                                 expected_total_len,
   92                                                 total):
   93         self.assertEqual(len(total['results']), expected_total_len)
   94         self.assertEqual(total['total'], expected_total_len)
   95 
   96         returned_total = round(sum(r['rate'] for r in total['results']), 5)
   97         self.assertLessEqual(abs(expected_total - returned_total), 0.00001)
   98 
   99         returned_qty = round(sum(r['qty'] for r in total['results']), 5)
  100         self.assertLessEqual(abs(expected_qty - returned_qty), 0.00001)
  101 
  102     def test_get_total_all_scopes_all_periods(self):
  103         expected_total, expected_qty, _ = self._expected_total_qty_len(
  104             self.data)
  105 
  106         begin = datetime.datetime(2018, 1, 1)
  107         end = datetime.datetime(2018, 1, 1, 4)
  108 
  109         self._compare_get_total_result_with_expected(
  110             expected_qty,
  111             expected_total,
  112             1,
  113             self.storage.total(begin=begin, end=end))
  114 
  115     def test_get_total_one_scope_all_periods(self):
  116         expected_total, expected_qty, _ = self._expected_total_qty_len(
  117             self.data, self._project_id)
  118 
  119         begin = datetime.datetime(2018, 1, 1)
  120         end = datetime.datetime(2018, 1, 1, 4)
  121 
  122         group_filters = {'project_id': self._project_id}
  123         self._compare_get_total_result_with_expected(
  124             expected_qty,
  125             expected_total,
  126             1,
  127             self.storage.total(begin=begin,
  128                                end=end,
  129                                group_filters=group_filters),
  130         )
  131 
  132     def test_get_total_all_scopes_one_period(self):
  133         expected_total, expected_qty, _ = self._expected_total_qty_len(
  134             [self.data[0]])
  135 
  136         begin = datetime.datetime(2018, 1, 1)
  137         end = datetime.datetime(2018, 1, 1, 1)
  138 
  139         self._compare_get_total_result_with_expected(
  140             expected_qty,
  141             expected_total,
  142             1,
  143             self.storage.total(begin=begin, end=end))
  144 
  145     def test_get_total_one_scope_one_period(self):
  146         expected_total, expected_qty, _ = self._expected_total_qty_len(
  147             [self.data[0]], self._project_id)
  148         expected_total, expected_qty, _ = self._expected_total_qty_len(
  149             [self.data[0]], self._project_id)
  150 
  151         begin = datetime.datetime(2018, 1, 1)
  152         end = datetime.datetime(2018, 1, 1, 1)
  153 
  154         group_filters = {'project_id': self._project_id}
  155         self._compare_get_total_result_with_expected(
  156             expected_qty,
  157             expected_total,
  158             1,
  159             self.storage.total(begin=begin,
  160                                end=end,
  161                                group_filters=group_filters),
  162         )
  163 
  164     def test_get_total_all_scopes_all_periods_groupby_project_id(self):
  165         expected_total_first, expected_qty_first, _ = \
  166             self._expected_total_qty_len(self.data, self._project_id)
  167         expected_total_second, expected_qty_second, _ = \
  168             self._expected_total_qty_len(self.data, self._other_project_id)
  169 
  170         begin = datetime.datetime(2018, 1, 1)
  171         end = datetime.datetime(2018, 1, 1, 4)
  172         total = self.storage.total(begin=begin, end=end,
  173                                    groupby=['project_id'])
  174         self.assertEqual(len(total['results']), 2)
  175         self.assertEqual(total['total'], 2)
  176 
  177         for t in total['results']:
  178             self.assertIn('project_id', t.keys())
  179 
  180         total['results'].sort(key=lambda x: x['project_id'], reverse=True)
  181 
  182         self.assertLessEqual(
  183             abs(round(total['results'][0]['rate'], 5) - expected_total_first),
  184             0.00001,
  185         )
  186         self.assertLessEqual(
  187             abs(round(total['results'][1]['rate'], 5) - expected_total_second),
  188             0.00001,
  189         )
  190         self.assertLessEqual(
  191             abs(round(total['results'][0]['qty'], 5) - expected_qty_first),
  192             0.00001,
  193         )
  194         self.assertLessEqual(
  195             abs(round(total['results'][1]['qty'], 5) - expected_qty_second),
  196             0.00001,
  197         )
  198 
  199     def test_get_total_all_scopes_one_period_groupby_project_id(self):
  200         expected_total_first, expected_qty_first, _ = \
  201             self._expected_total_qty_len([self.data[0]], self._project_id)
  202         expected_total_second, expected_qty_second, _ = \
  203             self._expected_total_qty_len([self.data[0]],
  204                                          self._other_project_id)
  205 
  206         begin = datetime.datetime(2018, 1, 1)
  207         end = datetime.datetime(2018, 1, 1, 1)
  208         total = self.storage.total(begin=begin, end=end,
  209                                    groupby=['project_id'])
  210         self.assertEqual(len(total), 2)
  211 
  212         for t in total['results']:
  213             self.assertIn('project_id', t.keys())
  214 
  215         total['results'].sort(key=lambda x: x['project_id'], reverse=True)
  216 
  217         self.assertLessEqual(
  218             abs(round(total['results'][0]['rate'], 5) - expected_total_first),
  219             0.00001,
  220         )
  221         self.assertLessEqual(
  222             abs(round(total['results'][1]['rate'], 5) - expected_total_second),
  223             0.00001,
  224         )
  225         self.assertLessEqual(
  226             abs(round(total['results'][0]['qty'], 5) - expected_qty_first),
  227             0.00001,
  228         )
  229         self.assertLessEqual(
  230             abs(round(total['results'][1]['qty'], 5) - expected_qty_second),
  231             0.00001,
  232         )
  233 
  234     def test_get_total_all_scopes_all_periods_groupby_type_paginate(self):
  235         expected_total, expected_qty, _ = \
  236             self._expected_total_qty_len(self.data)
  237 
  238         begin = datetime.datetime(2018, 1, 1)
  239         end = datetime.datetime(2018, 1, 1, 4)
  240 
  241         total = {'total': 0, 'results': []}
  242         for offset in range(0, 7, 2):
  243             chunk = self.storage.total(
  244                 begin=begin,
  245                 end=end,
  246                 offset=offset,
  247                 limit=2,
  248                 groupby=['type'])
  249             # there are seven metric types
  250             self.assertEqual(chunk['total'], 7)
  251             # last chunk, shorter
  252             if offset == 6:
  253                 self.assertEqual(len(chunk['results']), 1)
  254             else:
  255                 self.assertEqual(len(chunk['results']), 2)
  256             total['results'] += chunk['results']
  257             total['total'] += len(chunk['results'])
  258 
  259         unpaginated_total = self.storage.total(
  260             begin=begin, end=end, groupby=['type'])
  261         self.assertEqual(total, unpaginated_total)
  262 
  263         self._compare_get_total_result_with_expected(
  264             expected_qty,
  265             expected_total,
  266             7,
  267             total)
  268 
  269     def test_retrieve_all_scopes_all_types(self):
  270         expected_total, expected_qty, expected_length = \
  271             self._expected_total_qty_len(self.data)
  272 
  273         begin = datetime.datetime(2018, 1, 1)
  274         end = datetime.datetime(2018, 1, 1, 4)
  275 
  276         frames = self.storage.retrieve(begin=begin, end=end)
  277         self.assertEqual(frames['total'], expected_length)
  278 
  279         retrieved_length = 0
  280         for data_part in frames['dataframes']:
  281             for usage_part in data_part['usage'].values():
  282                 retrieved_length += len(usage_part)
  283 
  284         self.assertEqual(expected_length, retrieved_length)
  285 
  286     def test_retrieve_all_scopes_one_type(self):
  287         expected_total, expected_qty, expected_length = \
  288             self._expected_total_qty_len(self.data, types=['image.size'])
  289 
  290         begin = datetime.datetime(2018, 1, 1)
  291         end = datetime.datetime(2018, 1, 1, 4)
  292 
  293         frames = self.storage.retrieve(begin=begin, end=end,
  294                                        metric_types=['image.size'])
  295         self.assertEqual(frames['total'], expected_length)
  296 
  297         retrieved_length = 0
  298         for data_part in frames['dataframes']:
  299             for usage_part in data_part['usage'].values():
  300                 retrieved_length += len(usage_part)
  301 
  302         self.assertEqual(expected_length, retrieved_length)
  303 
  304     def test_retrieve_one_scope_two_types_one_period(self):
  305         expected_total, expected_qty, expected_length = \
  306             self._expected_total_qty_len([self.data[0]], self._project_id,
  307                                          types=['image.size', 'instance'])
  308 
  309         begin = datetime.datetime(2018, 1, 1)
  310         end = datetime.datetime(2018, 1, 1, 1)
  311 
  312         group_filters = {'project_id': self._project_id}
  313         frames = self.storage.retrieve(begin=begin, end=end,
  314                                        group_filters=group_filters,
  315                                        metric_types=['image.size', 'instance'])
  316         self.assertEqual(frames['total'], expected_length)
  317 
  318         retrieved_length = 0
  319         for data_part in frames['dataframes']:
  320             for usage_part in data_part['usage'].values():
  321                 retrieved_length += len(usage_part)
  322 
  323         self.assertEqual(expected_length, retrieved_length)
  324 
  325 
  326 StorageUnitTest.generate_scenarios()