"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "test/unit/common/test_db.py" between
swift-2.19.1.tar.gz and swift-2.21.0.tar.gz

About: OpenStack swift is software for creating redundant, scalable object storage using clusters of commodity servers to store terabytes or even petabytes of accessible data (now supporting storage policies).
The "Stein" series (latest release).

test_db.py  (swift-2.19.1):test_db.py  (swift-2.21.0)
skipping to change at line 26 skipping to change at line 26
"""Tests for swift.common.db""" """Tests for swift.common.db"""
import os import os
import sys import sys
import unittest import unittest
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree, copy from shutil import rmtree, copy
from uuid import uuid4 from uuid import uuid4
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
import base64
import json import json
import sqlite3 import sqlite3
import itertools import itertools
import time import time
import random import random
from mock import patch, MagicMock from mock import patch, MagicMock
from eventlet.timeout import Timeout from eventlet.timeout import Timeout
from six.moves import range from six.moves import range
import six
import swift.common.db import swift.common.db
from swift.common.constraints import \ from swift.common.constraints import \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE
from swift.common.db import chexor, dict_factory, get_db_connection, \ from swift.common.db import chexor, dict_factory, get_db_connection, \
DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \
GreenDBConnection, PICKLE_PROTOCOL, zero_like GreenDBConnection, PICKLE_PROTOCOL, zero_like
from swift.common.utils import normalize_timestamp, mkdirs, Timestamp from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException from swift.common.swob import HTTPException
skipping to change at line 267 skipping to change at line 270
if not conn.execute( if not conn.execute(
'SELECT 1 FROM test WHERE name = ?', 'SELECT 1 FROM test WHERE name = ?',
(rec['name'],)).fetchall(): (rec['name'],)).fetchall():
conn.execute(''' conn.execute('''
INSERT INTO test (name, created_at, deleted) INSERT INTO test (name, created_at, deleted)
VALUES (?, ?, ?)''', ( VALUES (?, ?, ?)''', (
rec['name'], rec['created_at'], rec['deleted'])) rec['name'], rec['created_at'], rec['deleted']))
conn.commit() conn.commit()
def _commit_puts_load(self, item_list, entry): def _commit_puts_load(self, item_list, entry):
(name, timestamp, deleted) = pickle.loads(entry.decode('base64')) (name, timestamp, deleted) = entry
item_list.append({ item_list.append({
'name': name, 'name': name,
'created_at': timestamp, 'created_at': timestamp,
'deleted': deleted, 'deleted': deleted,
}) })
def _load_item(self, name, timestamp, deleted): def _load_item(self, name, timestamp, deleted):
if self.db_file == ':memory:': if self.db_file == ':memory:':
record = { record = {
'name': name, 'name': name,
'created_at': timestamp, 'created_at': timestamp,
'deleted': deleted, 'deleted': deleted,
} }
self.merge_items([record]) self.merge_items([record])
return return
with open(self.pending_file, 'a+b') as fp: with open(self.pending_file, 'a+b') as fp:
fp.write(':') fp.write(b':')
fp.write(pickle.dumps( fp.write(base64.b64encode(pickle.dumps(
(name, timestamp, deleted), (name, timestamp, deleted),
protocol=PICKLE_PROTOCOL).encode('base64')) protocol=PICKLE_PROTOCOL)))
fp.flush() fp.flush()
def put_test(self, name, timestamp): def put_test(self, name, timestamp):
self._load_item(name, timestamp, 0) self._load_item(name, timestamp, 0)
def delete_test(self, name, timestamp): def delete_test(self, name, timestamp):
self._load_item(name, timestamp, 1) self._load_item(name, timestamp, 1)
def _delete_db(self, conn, timestamp): def _delete_db(self, conn, timestamp):
conn.execute(""" conn.execute("""
skipping to change at line 512 skipping to change at line 515
self.assertEqual(info[k], v, self.assertEqual(info[k], v,
'mismatch for %s, %s != %s' % ( 'mismatch for %s, %s != %s' % (
k, info[k], v)) k, info[k], v))
def test_get_raw_metadata(self): def test_get_raw_metadata(self):
broker = self.broker_class(':memory:', account='test', container='c') broker = self.broker_class(':memory:', account='test', container='c')
broker.initialize(Timestamp(0).internal, broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy)) storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {}) self.assertEqual(broker.metadata, {})
self.assertEqual(broker.get_raw_metadata(), '') self.assertEqual(broker.get_raw_metadata(), '')
key = u'test\u062a'.encode('utf-8') # This is not obvious. The actual JSON in the database is the same:
# '{"test\\u062a": ["value\\u062a", "0000000001.00000"]}'
# The only difference is what reading it produces on py2 and py3.
# We use native strings for metadata keys (see native_str_keys()),
# so keys are different.
if six.PY2:
key = u'test\u062a'.encode('utf-8')
else:
key = u'test\u062a'
value = u'value\u062a' value = u'value\u062a'
metadata = { metadata = {
key: [value, Timestamp(1).internal] key: [value, Timestamp(1).internal]
} }
broker.update_metadata(metadata) broker.update_metadata(metadata)
self.assertEqual(broker.metadata, metadata) self.assertEqual(broker.metadata, metadata)
self.assertEqual(broker.get_raw_metadata(), self.assertEqual(broker.get_raw_metadata(),
json.dumps(metadata)) json.dumps(metadata))
def test_put_timestamp(self): def test_put_timestamp(self):
skipping to change at line 670 skipping to change at line 681
# We only wrote 1 byte, so we should end with the 1st step or 1 MB. # We only wrote 1 byte, so we should end with the 1st step or 1 MB.
self.assertEqual(test_size[0], 1024 * 1024) self.assertEqual(test_size[0], 1024 * 1024)
def test_initialize(self): def test_initialize(self):
self.assertRaises(AttributeError, self.assertRaises(AttributeError,
DatabaseBroker(':memory:').initialize, DatabaseBroker(':memory:').initialize,
normalize_timestamp('1')) normalize_timestamp('1'))
stub_dict = {} stub_dict = {}
def stub(*args, **kwargs): def stub(*args, **kwargs):
for key in stub_dict.keys(): stub_dict.clear()
del stub_dict[key]
stub_dict['args'] = args stub_dict['args'] = args
for key, value in kwargs.items(): stub_dict.update(kwargs)
stub_dict[key] = value
broker = DatabaseBroker(':memory:') broker = DatabaseBroker(':memory:')
broker._initialize = stub broker._initialize = stub
broker.initialize(normalize_timestamp('1')) broker.initialize(normalize_timestamp('1'))
self.assertTrue(hasattr(stub_dict['args'][0], 'execute')) self.assertTrue(hasattr(stub_dict['args'][0], 'execute'))
self.assertEqual(stub_dict['args'][1], '0000000001.00000') self.assertEqual(stub_dict['args'][1], '0000000001.00000')
with broker.get() as conn: with broker.get() as conn:
conn.execute('SELECT * FROM outgoing_sync') conn.execute('SELECT * FROM outgoing_sync')
conn.execute('SELECT * FROM incoming_sync') conn.execute('SELECT * FROM incoming_sync')
broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker = DatabaseBroker(os.path.join(self.testdir, '1.db'))
broker._initialize = stub broker._initialize = stub
skipping to change at line 1407 skipping to change at line 1416
with open(broker.pending_file, 'wb'): with open(broker.pending_file, 'wb'):
pass pass
# merge given list # merge given list
with patch.object(broker, 'merge_items') as mock_merge_items: with patch.object(broker, 'merge_items') as mock_merge_items:
broker._commit_puts(['test']) broker._commit_puts(['test'])
mock_merge_items.assert_called_once_with(['test']) mock_merge_items.assert_called_once_with(['test'])
# load file and merge # load file and merge
with open(broker.pending_file, 'wb') as fd: with open(broker.pending_file, 'wb') as fd:
fd.write(':1:2:99') for v in (1, 2, 99):
fd.write(b':' + base64.b64encode(pickle.dumps(
v, protocol=PICKLE_PROTOCOL)))
with patch.object(broker, 'merge_items') as mock_merge_items: with patch.object(broker, 'merge_items') as mock_merge_items:
broker._commit_puts_load = lambda l, e: l.append(e) broker._commit_puts_load = lambda l, e: l.append(e)
broker._commit_puts() broker._commit_puts()
mock_merge_items.assert_called_once_with(['1', '2', '99']) mock_merge_items.assert_called_once_with([1, 2, 99])
self.assertEqual(0, os.path.getsize(broker.pending_file)) self.assertEqual(0, os.path.getsize(broker.pending_file))
# load file and merge with given list # load file and merge with given list
with open(broker.pending_file, 'wb') as fd: with open(broker.pending_file, 'wb') as fd:
fd.write(':bad') fd.write(b':' + base64.b64encode(pickle.dumps(
b'bad', protocol=PICKLE_PROTOCOL)))
with patch.object(broker, 'merge_items') as mock_merge_items: with patch.object(broker, 'merge_items') as mock_merge_items:
broker._commit_puts_load = lambda l, e: l.append(e) broker._commit_puts_load = lambda l, e: l.append(e)
broker._commit_puts(['not']) broker._commit_puts([b'not'])
mock_merge_items.assert_called_once_with(['not', 'bad']) mock_merge_items.assert_called_once_with([b'not', b'bad'])
self.assertEqual(0, os.path.getsize(broker.pending_file)) self.assertEqual(0, os.path.getsize(broker.pending_file))
# skip_commits True - no merge # skip_commits True - no merge
db_file = os.path.join(self.testdir, '2.db') db_file = os.path.join(self.testdir, '2.db')
broker = DatabaseBroker(db_file, skip_commits=True) broker = DatabaseBroker(db_file, skip_commits=True)
broker._initialize = MagicMock() broker._initialize = MagicMock()
broker.initialize(Timestamp.now()) broker.initialize(Timestamp.now())
with open(broker.pending_file, 'wb') as fd: with open(broker.pending_file, 'wb') as fd:
fd.write(':ignored') fd.write(b':ignored')
with patch.object(broker, 'merge_items') as mock_merge_items: with patch.object(broker, 'merge_items') as mock_merge_items:
with self.assertRaises(DatabaseConnectionError) as cm: with self.assertRaises(DatabaseConnectionError) as cm:
broker._commit_puts(['hmmm']) broker._commit_puts([b'hmmm'])
mock_merge_items.assert_not_called() mock_merge_items.assert_not_called()
self.assertIn('commits not accepted', str(cm.exception)) self.assertIn('commits not accepted', str(cm.exception))
with open(broker.pending_file, 'rb') as fd: with open(broker.pending_file, 'rb') as fd:
self.assertEqual(':ignored', fd.read()) self.assertEqual(b':ignored', fd.read())
def test_put_record(self): def test_put_record(self):
db_file = os.path.join(self.testdir, '1.db') db_file = os.path.join(self.testdir, '1.db')
broker = DatabaseBroker(db_file) broker = DatabaseBroker(db_file)
broker._initialize = MagicMock() broker._initialize = MagicMock()
broker.initialize(Timestamp.now()) broker.initialize(Timestamp.now())
# pending file created and record written # pending file created and record written
broker.make_tuple_for_pickle = lambda x: x.upper() broker.make_tuple_for_pickle = lambda x: x.upper()
with patch.object(broker, '_commit_puts') as mock_commit_puts: with patch.object(broker, '_commit_puts') as mock_commit_puts:
broker.put_record('pinky') broker.put_record('pinky')
mock_commit_puts.assert_not_called() mock_commit_puts.assert_not_called()
with open(broker.pending_file, 'rb') as fd: with open(broker.pending_file, 'rb') as fd:
pending = fd.read() pending = fd.read()
items = pending.split(':') items = pending.split(b':')
self.assertEqual(['PINKY'], self.assertEqual(['PINKY'],
[pickle.loads(i.decode('base64')) for i in items[1:]]) [pickle.loads(base64.b64decode(i))
for i in items[1:]])
# record appended # record appended
with patch.object(broker, '_commit_puts') as mock_commit_puts: with patch.object(broker, '_commit_puts') as mock_commit_puts:
broker.put_record('perky') broker.put_record('perky')
mock_commit_puts.assert_not_called() mock_commit_puts.assert_not_called()
with open(broker.pending_file, 'rb') as fd: with open(broker.pending_file, 'rb') as fd:
pending = fd.read() pending = fd.read()
items = pending.split(':') items = pending.split(b':')
self.assertEqual(['PINKY', 'PERKY'], self.assertEqual(['PINKY', 'PERKY'],
[pickle.loads(i.decode('base64')) for i in items[1:]]) [pickle.loads(base64.b64decode(i))
for i in items[1:]])
# pending file above cap # pending file above cap
cap = swift.common.db.PENDING_CAP cap = swift.common.db.PENDING_CAP
while os.path.getsize(broker.pending_file) < cap: while os.path.getsize(broker.pending_file) < cap:
with open(broker.pending_file, 'ab') as fd: with open(broker.pending_file, 'ab') as fd:
fd.write('x' * 100000) fd.write(b'x' * 100000)
with patch.object(broker, '_commit_puts') as mock_commit_puts: with patch.object(broker, '_commit_puts') as mock_commit_puts:
broker.put_record('direct') broker.put_record('direct')
mock_commit_puts.called_once_with(['direct']) mock_commit_puts.called_once_with(['direct'])
# records shouldn't be put to brokers with skip_commits True because # records shouldn't be put to brokers with skip_commits True because
# they cannot be accepted if the pending file is full # they cannot be accepted if the pending file is full
broker.skip_commits = True broker.skip_commits = True
with open(broker.pending_file, 'wb'): with open(broker.pending_file, 'wb'):
# empty the pending file # empty the pending file
pass pass
 End of changes. 20 change blocks. 
22 lines changed or deleted 36 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)