"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "swift/common/middleware/s3api/controllers/multi_delete.py" between
swift-2.19.1.tar.gz and swift-2.21.0.tar.gz

About: OpenStack swift is software for creating redundant, scalable object storage using clusters of commodity servers to store terabytes or even petabytes of accessible data (now supporting storage policies).
The "Stein" series (latest release).

multi_delete.py  (swift-2.19.1):multi_delete.py  (swift-2.21.0)
skipping to change at line 16 skipping to change at line 16
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. # implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from swift.common.utils import public import copy
import json
from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
from swift.common.utils import public, StreamingPile
from swift.common.middleware.s3api.controllers.base import Controller, \ from swift.common.middleware.s3api.controllers.base import Controller, \
bucket_operation bucket_operation
from swift.common.middleware.s3api.etree import Element, SubElement, \ from swift.common.middleware.s3api.etree import Element, SubElement, \
fromstring, tostring, XMLSyntaxError, DocumentInvalid fromstring, tostring, XMLSyntaxError, DocumentInvalid
from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \ from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \ NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \
AccessDenied, MissingRequestBodyError AccessDenied, MissingRequestBodyError
MAX_MULTI_DELETE_BODY_SIZE = 61365
class MultiObjectDeleteController(Controller): class MultiObjectDeleteController(Controller):
""" """
Handles Delete Multiple Objects, which is logged as a MULTI_OBJECT_DELETE Handles Delete Multiple Objects, which is logged as a MULTI_OBJECT_DELETE
operation in the S3 server log. operation in the S3 server log.
""" """
def _gen_error_body(self, error, elem, delete_list): def _gen_error_body(self, error, elem, delete_list):
for key, version in delete_list: for key, version in delete_list:
if version is not None: if version is not None:
# TODO: delete the specific version of the object # TODO: delete the specific version of the object
raise S3NotImplemented() raise S3NotImplemented()
skipping to change at line 63 skipping to change at line 65
for obj in elem.iterchildren('Object'): for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text key = obj.find('./Key').text
if not key: if not key:
raise UserKeyMustBeSpecified() raise UserKeyMustBeSpecified()
version = obj.find('./VersionId') version = obj.find('./VersionId')
if version is not None: if version is not None:
version = version.text version = version.text
yield key, version yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try: try:
xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE) xml = req.xml(max_body_size)
if not xml: if not xml:
raise MissingRequestBodyError() raise MissingRequestBodyError()
req.check_md5(xml) req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger) elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet') quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true': if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True self.quiet = True
else: else:
skipping to change at line 97 skipping to change at line 107
elem = Element('DeleteResult') elem = Element('DeleteResult')
# check bucket existence # check bucket existence
try: try:
req.get_response(self.app, 'HEAD') req.get_response(self.app, 'HEAD')
except AccessDenied as error: except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list) body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body) return HTTPOk(body=body)
for key, version in delete_list: if any(version is not None for _key, version in delete_list):
if version is not None: # TODO: support deleting specific versions of objects
# TODO: delete the specific version of the object raise S3NotImplemented()
raise S3NotImplemented()
def do_delete(base_req, key, version):
req = copy.copy(base_req)
req.environ = copy.copy(base_req.environ)
req.object_name = key req.object_name = key
try: try:
query = req.gen_multipart_manifest_delete_query(self.app) query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query) resp = req.get_response(self.app, method='DELETE', query=query,
headers={'Accept': 'application/json'})
# Have to read the response to actually do the SLO delete
if query:
try:
delete_result = json.loads(resp.body)
if delete_result['Errors']:
# NB: bulk includes 404s in "Number Not Found",
# not "Errors"
msg_parts = [delete_result['Response Status']]
msg_parts.extend(
'%s: %s' % (obj, status)
for obj, status in delete_result['Errors'])
return key, {'code': 'SLODeleteError',
'message': '\n'.join(msg_parts)}
# else, all good
except (ValueError, TypeError, KeyError):
# Logs get all the gory details
self.logger.exception(
'Could not parse SLO delete response: %r',
resp.body)
# Client gets something more generic
return key, {'code': 'SLODeleteError',
'message': 'Unexpected swift response'}
except NoSuchKey: except NoSuchKey:
pass pass
except ErrorResponse as e: except ErrorResponse as e:
error = SubElement(elem, 'Error') return key, {'code': e.__class__.__name__, 'message': e._msg}
SubElement(error, 'Key').text = key return key, None
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg with StreamingPile(self.conf.multi_delete_concurrency) as pile:
continue for key, err in pile.asyncstarmap(do_delete, (
(req, key, version) for key, version in delete_list)):
if not self.quiet: if err:
deleted = SubElement(elem, 'Deleted') error = SubElement(elem, 'Error')
SubElement(deleted, 'Key').text = key SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = err['code']
SubElement(error, 'Message').text = err['message']
elif not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem) body = tostring(elem)
return HTTPOk(body=body) return HTTPOk(body=body)
 End of changes. 7 change blocks. 
19 lines changed or deleted 59 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)