¡@

OpenStack Study: swift.py

OpenStack Index

**** CubicPower OpenStack Study ****

# Licensed under the Apache License, Version 2.0 (the "License"); you may

# not use this file except in compliance with the License. You may obtain

# a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT

# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

# License for the specific language governing permissions and limitations

# under the License.

"""Storage backend for SWIFT"""

from __future__ import absolute_import

import hashlib

import httplib

import math

from oslo.config import cfg

import six.moves.urllib.parse as urlparse

from glance.common import auth

from glance.common import exception

from glance.openstack.common import excutils

import glance.openstack.common.log as logging

import glance.store

import glance.store.base

import glance.store.location

LOG = logging.getLogger(__name__)

DEFAULT_CONTAINER = 'glance'

DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB

DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M

ONE_MB = 1000 * 1024

swift_opts = [

cfg.BoolOpt('swift_enable_snet', default=False,

help=_('Whether to use ServiceNET to communicate with the '

'Swift storage servers.')),

cfg.StrOpt('swift_store_auth_address',

help=_('The address where the Swift authentication service '

'is listening.')),

cfg.StrOpt('swift_store_user', secret=True,

help=_('The user to authenticate against the Swift '

'authentication service.')),

cfg.StrOpt('swift_store_key', secret=True,

help=_('Auth key for the user authenticating against the '

'Swift authentication service.')),

cfg.StrOpt('swift_store_auth_version', default='2',

help=_('Version of the authentication service to use. '

'Valid versions are 2 for keystone and 1 for swauth '

'and rackspace.')),

cfg.BoolOpt('swift_store_auth_insecure', default=False,

help=_('If True, swiftclient won\'t check for a valid SSL '

'certificate when authenticating.')),

cfg.StrOpt('swift_store_region',

help=_('The region of the swift endpoint to be used for '

'single tenant. This setting is only necessary if the '

'tenant has multiple swift endpoints.')),

cfg.StrOpt('swift_store_endpoint_type', default='publicURL',

help=_('A string giving the endpoint type of the swift '

'service to use (publicURL, adminURL or internalURL). '

'This setting is only used if swift_store_auth_version '

'is 2.')),

cfg.StrOpt('swift_store_service_type', default='object-store',

help=_('A string giving the service type of the swift service '

'to use. This setting is only used if '

'swift_store_auth_version is 2.')),

cfg.StrOpt('swift_store_container',

default=DEFAULT_CONTAINER,

help=_('Container within the account that the account should '

'use for storing images in Swift.')),

cfg.IntOpt('swift_store_large_object_size',

default=DEFAULT_LARGE_OBJECT_SIZE,

help=_('The size, in MB, that Glance will start chunking image '

'files and do a large object manifest in Swift.')),

cfg.IntOpt('swift_store_large_object_chunk_size',

default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE,

help=_('The amount of data written to a temporary disk buffer '

'during the process of chunking the image file.')),

cfg.BoolOpt('swift_store_create_container_on_put', default=False,

help=_('A boolean value that determines if we create the '

'container if it does not exist.')),

cfg.BoolOpt('swift_store_multi_tenant', default=False,

help=_('If set to True, enables multi-tenant storage '

'mode which causes Glance images to be stored in '

'tenant specific Swift accounts.')),

cfg.ListOpt('swift_store_admin_tenants', default=[],

help=_('A list of tenants that will be granted read/write '

'access on all Swift containers created by Glance in '

'multi-tenant mode.')),

cfg.BoolOpt('swift_store_ssl_compression', default=True,

help=_('If set to False, disables SSL layer compression of '

'https swift requests. Setting to False may improve '

'performance for images which are already in a '

'compressed format, eg qcow2.')),

cfg.IntOpt('swift_store_retry_get_count', default=0,

help=_('The number of times a Swift download will be retried '

'before the request fails.'))

]

CONF = cfg.CONF

CONF.register_opts(swift_opts)

**** CubicPower OpenStack Study ****

def swift_retry_iter(resp_iter, length, store, location):

length = length if length else (resp_iter.len

if hasattr(resp_iter, 'len') else 0)

retries = 0

bytes_read = 0

while retries <= CONF.swift_store_retry_get_count:

try:

for chunk in resp_iter:

yield chunk

bytes_read += len(chunk)

except swiftclient.ClientException as e:

LOG.warn(_("Swift exception raised %s") % e)

if bytes_read != length:

if retries == CONF.swift_store_retry_get_count:

# terminate silently and let higher level decide

LOG.error(_("Stopping Swift retries after %d "

"attempts") % retries)

break

else:

retries += 1

LOG.info(_("Retrying Swift connection "

"(%(retries)d/%(max_retries)d) with "

"range=%(start)d-%(end)d") %

{'retries': retries,

'max_retries': CONF.swift_store_retry_get_count,

'start': bytes_read,

'end': length})

(resp_headers, resp_iter) = store._get_object(location, None,

bytes_read)

else:

break

**** CubicPower OpenStack Study ****

class StoreLocation(glance.store.location.StoreLocation):

"""

Class describing a Swift URI. A Swift URI can look like any of

the following:

swift://user:pass@authurl.com/container/obj-id

swift://account:user:pass@authurl.com/container/obj-id

swift+http://user:pass@authurl.com/container/obj-id

swift+https://user:pass@authurl.com/container/obj-id

When using multi-tenant a URI might look like this (a storage URL):

swift+https://example.com/container/obj-id

The swift+http:// URIs indicate there is an HTTP authentication URL.

The

**** CubicPower OpenStack Study ****

def process_specs(self):

self.scheme = self.specs.get('scheme', 'swift+https')

self.user = self.specs.get('user')

self.key = self.specs.get('key')

self.auth_or_store_url = self.specs.get('auth_or_store_url')

self.container = self.specs.get('container')

self.obj = self.specs.get('obj')

**** CubicPower OpenStack Study ****

def _get_credstring(self):

if self.user and self.key:

return '%s:%s@' % (urlparse.quote(self.user),

urlparse.quote(self.key))

return ''

**** CubicPower OpenStack Study ****

def get_uri(self):

auth_or_store_url = self.auth_or_store_url

if auth_or_store_url.startswith('http://'):

auth_or_store_url = auth_or_store_url[len('http://'):]

elif auth_or_store_url.startswith('https://'):

auth_or_store_url = auth_or_store_url[len('https://'):]

credstring = self._get_credstring()

auth_or_store_url = auth_or_store_url.strip('/')

container = self.container.strip('/')

obj = self.obj.strip('/')

return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url,

container, obj)

**** CubicPower OpenStack Study ****

def parse_uri(self, uri):

"""

Parse URLs. This method fixes an issue where credentials specified

in the URL are interpreted differently in Python 2.6.1+ than prior

versions of Python. It also deals with the peculiarity that new-style

Swift URIs have where a username can contain a ':', like so:

swift://account:user:pass@authurl.com/container/obj

"""

# Make sure that URIs that contain multiple schemes, such as:

# swift://user:pass@http://authurl.com/v1/container/obj

# are immediately rejected.

if uri.count('://') != 1:

reason = _("URI cannot contain more than one occurrence "

"of a scheme. If you have specified a URI like "

"swift://user:pass@http://authurl.com/v1/container/obj"

", you need to change it to use the "

"swift+http:// scheme, like so: "

"swift+http://user:pass@authurl.com/v1/container/obj")

LOG.debug(_("Invalid store URI: %(reason)s"), {'reason': reason})

raise exception.BadStoreUri(message=reason)

pieces = urlparse.urlparse(uri)

assert pieces.scheme in ('swift', 'swift+http', 'swift+https')

self.scheme = pieces.scheme

netloc = pieces.netloc

path = pieces.path.lstrip('/')

if netloc != '':

# > Python 2.6.1

if '@' in netloc:

creds, netloc = netloc.split('@')

else:

creds = None

else:

# Python 2.6.1 compat

# see lp659445 and Python issue7904

if '@' in path:

creds, path = path.split('@')

else:

creds = None

netloc = path[0:path.find('/')].strip('/')

path = path[path.find('/'):].strip('/')

if creds:

cred_parts = creds.split(':')

if len(cred_parts) != 2:

reason = (_("Badly formed credentials in Swift URI."))

LOG.debug(reason)

raise exception.BadStoreUri()

user, key = cred_parts

self.user = urlparse.unquote(user)

self.key = urlparse.unquote(key)

else:

self.user = None

self.key = None

path_parts = path.split('/')

try:

self.obj = path_parts.pop()

self.container = path_parts.pop()

if not netloc.startswith('http'):

# push hostname back into the remaining to build full authurl

path_parts.insert(0, netloc)

self.auth_or_store_url = '/'.join(path_parts)

except IndexError:

reason = _("Badly formed Swift URI.")

LOG.debug(reason)

raise exception.BadStoreUri()

@property

**** CubicPower OpenStack Study ****

def swift_url(self):

"""

Creates a fully-qualified auth url that the Swift client library can

use. The scheme for the auth_url is determined using the scheme

included in the `location` field.

HTTPS is assumed, unless 'swift+http' is specified.

"""

if self.auth_or_store_url.startswith('http'):

return self.auth_or_store_url

else:

if self.scheme in ('swift+https', 'swift'):

auth_scheme = 'https://'

else:

auth_scheme = 'http://'

return ''.join([auth_scheme, self.auth_or_store_url])

def Store(context=None, loc=None):

if (CONF.swift_store_multi_tenant and

(loc is None or loc.store_location.user is None)):

return MultiTenantStore(context, loc)

return SingleTenantStore(context, loc)

**** CubicPower OpenStack Study ****

def Store(context=None, loc=None):

if (CONF.swift_store_multi_tenant and

(loc is None or loc.store_location.user is None)):

return MultiTenantStore(context, loc)

return SingleTenantStore(context, loc)

**** CubicPower OpenStack Study ****

class BaseStore(glance.store.base.Store):

CHUNKSIZE = 65536

**** CubicPower OpenStack Study ****

def get_schemes(self):

return ('swift+https', 'swift', 'swift+http')

**** CubicPower OpenStack Study ****

def configure(self):

_obj_size = self._option_get('swift_store_large_object_size')

self.large_object_size = _obj_size * ONE_MB

_chunk_size = self._option_get('swift_store_large_object_chunk_size')

self.large_object_chunk_size = _chunk_size * ONE_MB

self.admin_tenants = CONF.swift_store_admin_tenants

self.region = CONF.swift_store_region

self.service_type = CONF.swift_store_service_type

self.endpoint_type = CONF.swift_store_endpoint_type

self.snet = CONF.swift_enable_snet

self.insecure = CONF.swift_store_auth_insecure

self.ssl_compression = CONF.swift_store_ssl_compression

**** CubicPower OpenStack Study ****

def _get_object(self, location, connection=None, start=None):

if not connection:

connection = self.get_connection(location)

headers = {}

if start is not None:

bytes_range = 'bytes=%d-' % start

headers = {'Range': bytes_range}

try:

resp_headers, resp_body = connection.get_object(

container=location.container, obj=location.obj,

resp_chunk_size=self.CHUNKSIZE, headers=headers)

except swiftclient.ClientException as e:

if e.http_status == httplib.NOT_FOUND:

msg = _("Swift could not find object %s.") % location.obj

LOG.warn(msg)

raise exception.NotFound(msg)

else:

raise

return (resp_headers, resp_body)

**** CubicPower OpenStack Study ****

def get(self, location, connection=None):

location = location.store_location

(resp_headers, resp_body) = self._get_object(location, connection)

class ResponseIndexable(glance.store.Indexable):

def another(self):

try:

return self.wrapped.next()

except StopIteration:

return ''

length = int(resp_headers.get('content-length', 0))

if CONF.swift_store_retry_get_count > 0:

resp_body = swift_retry_iter(resp_body, length, self, location)

return (ResponseIndexable(resp_body, length), length)

**** CubicPower OpenStack Study ****

def get_size(self, location, connection=None):

location = location.store_location

if not connection:

connection = self.get_connection(location)

try:

resp_headers = connection.head_object(

container=location.container, obj=location.obj)

return int(resp_headers.get('content-length', 0))

except Exception:

return 0

**** CubicPower OpenStack Study ****

def _option_get(self, param):

result = getattr(CONF, param)

if not result:

reason = (_("Could not find %(param)s in configuration "

"options.") % {'param': param})

LOG.error(reason)

raise exception.BadStoreConfiguration(store_name="swift",

reason=reason)

return result

**** CubicPower OpenStack Study ****

def _delete_stale_chunks(self, connection, container, chunk_list):

for chunk in chunk_list:

LOG.debug(_("Deleting chunk %s") % chunk)

try:

connection.delete_object(container, chunk)

except Exception:

msg = _("Failed to delete orphaned chunk "

"%(container)s/%(chunk)s")

LOG.exception(msg % {'container': container,

'chunk': chunk})

**** CubicPower OpenStack Study ****

def add(self, image_id, image_file, image_size, connection=None):

location = self.create_location(image_id)

if not connection:

connection = self.get_connection(location)

self._create_container_if_missing(location.container, connection)

LOG.debug(_("Adding image object '%(obj_name)s' "

"to Swift") % dict(obj_name=location.obj))

try:

if image_size > 0 and image_size < self.large_object_size:

# Image size is known, and is less than large_object_size.

# Send to Swift with regular PUT.

obj_etag = connection.put_object(location.container,

location.obj, image_file,

content_length=image_size)

else:

# Write the image into Swift in chunks.

chunk_id = 1

if image_size > 0:

total_chunks = str(int(

math.ceil(float(image_size) /

float(self.large_object_chunk_size))))

else:

# image_size == 0 is when we don't know the size

# of the image. This can occur with older clients

# that don't inspect the payload size.

LOG.debug(_("Cannot determine image size. Adding as a "

"segmented object to Swift."))

total_chunks = '?'

checksum = hashlib.md5()

written_chunks = []

combined_chunks_size = 0

while True:

chunk_size = self.large_object_chunk_size

if image_size == 0:

content_length = None

else:

left = image_size - combined_chunks_size

if left == 0:

break

if chunk_size > left:

chunk_size = left

content_length = chunk_size

chunk_name = "%s-%05d" % (location.obj, chunk_id)

reader = ChunkReader(image_file, checksum, chunk_size)

try:

chunk_etag = connection.put_object(

location.container, chunk_name, reader,

content_length=content_length)

written_chunks.append(chunk_name)

except Exception:

# Delete orphaned segments from swift backend

with excutils.save_and_reraise_exception():

LOG.exception(_("Error during chunked upload to "

"backend, deleting stale chunks"))

self._delete_stale_chunks(connection,

location.container,

written_chunks)

bytes_read = reader.bytes_read

msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"

"%(total_chunks)s) of length %(bytes_read)d "

"to Swift returning MD5 of content: "

"%(chunk_etag)s") %

{'chunk_name': chunk_name,

'chunk_id': chunk_id,

'total_chunks': total_chunks,

'bytes_read': bytes_read,

'chunk_etag': chunk_etag})

LOG.debug(msg)

if bytes_read == 0:

# Delete the last chunk, because it's of zero size.

# This will happen if size == 0.

LOG.debug(_("Deleting final zero-length chunk"))

connection.delete_object(location.container,

chunk_name)

break

chunk_id += 1

combined_chunks_size += bytes_read

# In the case we have been given an unknown image size,

# set the size to the total size of the combined chunks.

if image_size == 0:

image_size = combined_chunks_size

# Now we write the object manifest and return the

# manifest's etag...

manifest = "%s/%s-" % (location.container, location.obj)

headers = {'ETag': hashlib.md5("").hexdigest(),

'X-Object-Manifest': manifest}

# The ETag returned for the manifest is actually the

# MD5 hash of the concatenated checksums of the strings

# of each chunk...so we ignore this result in favour of

# the MD5 of the entire image file contents, so that

# users can verify the image file contents accordingly

connection.put_object(location.container, location.obj,

None, headers=headers)

obj_etag = checksum.hexdigest()

# NOTE: We return the user and key here! Have to because

# location is used by the API server to return the actual

# image data. We *really* should consider NOT returning

# the location attribute from GET /images/ and

# GET /images/details

return (location.get_uri(), image_size, obj_etag, {})

except swiftclient.ClientException as e:

if e.http_status == httplib.CONFLICT:

raise exception.Duplicate(_("Swift already has an image at "

"this location"))

msg = (_("Failed to add object to Swift.\n"

"Got error from Swift: %(e)s") % {'e': e})

LOG.error(msg)

raise glance.store.BackendException(msg)

**** CubicPower OpenStack Study ****

def delete(self, location, connection=None):

location = location.store_location

if not connection:

connection = self.get_connection(location)

try:

# We request the manifest for the object. If one exists,

# that means the object was uploaded in chunks/segments,

# and we need to delete all the chunks as well as the

# manifest.

manifest = None

try:

headers = connection.head_object(

location.container, location.obj)

manifest = headers.get('x-object-manifest')

except swiftclient.ClientException as e:

if e.http_status != httplib.NOT_FOUND:

raise

if manifest:

# Delete all the chunks before the object manifest itself

obj_container, obj_prefix = manifest.split('/', 1)

segments = connection.get_container(

obj_container, prefix=obj_prefix)[1]

for segment in segments:

# TODO(jaypipes): This would be an easy area to parallelize

# since we're simply sending off parallelizable requests

# to Swift to delete stuff. It's not like we're going to

# be hogging up network or file I/O here...

connection.delete_object(obj_container,

segment['name'])

# Delete object (or, in segmented case, the manifest)

connection.delete_object(location.container, location.obj)

except swiftclient.ClientException as e:

if e.http_status == httplib.NOT_FOUND:

msg = _("Swift could not find image at URI.")

raise exception.NotFound(msg)

else:

raise

**** CubicPower OpenStack Study ****

def _create_container_if_missing(self, container, connection):

"""

Creates a missing container in Swift if the

``swift_store_create_container_on_put`` option is set.

:param container: Name of container to create

:param connection: Connection to swift service

"""

try:

connection.head_container(container)

except swiftclient.ClientException as e:

if e.http_status == httplib.NOT_FOUND:

if CONF.swift_store_create_container_on_put:

try:

connection.put_container(container)

except swiftclient.ClientException as e:

msg = (_("Failed to add container to Swift.\n"

"Got error from Swift: %(e)s") % {'e': e})

raise glance.store.BackendException(msg)

else:

msg = (_("The container %(container)s does not exist in "

"Swift. Please set the "

"swift_store_create_container_on_put option"

"to add container to Swift automatically.") %

{'container': container})

raise glance.store.BackendException(msg)

else:

raise

**** CubicPower OpenStack Study ****

def get_connection(self):

raise NotImplemented()

**** CubicPower OpenStack Study ****

def create_location(self):

raise NotImplemented()

**** CubicPower OpenStack Study ****

class SingleTenantStore(BaseStore):

EXAMPLE_URL = "swift://:@//"

**** CubicPower OpenStack Study ****

def configure(self):

super(SingleTenantStore, self).configure()

self.auth_version = self._option_get('swift_store_auth_version')

**** CubicPower OpenStack Study ****

def configure_add(self):

self.auth_address = self._option_get('swift_store_auth_address')

if self.auth_address.startswith('http://'):

self.scheme = 'swift+http'

else:

self.scheme = 'swift+https'

self.container = CONF.swift_store_container

self.user = self._option_get('swift_store_user')

self.key = self._option_get('swift_store_key')

**** CubicPower OpenStack Study ****

def create_location(self, image_id):

specs = {'scheme': self.scheme,

'container': self.container,

'obj': str(image_id),

'auth_or_store_url': self.auth_address,

'user': self.user,

'key': self.key}

return StoreLocation(specs)

**** CubicPower OpenStack Study ****

def get_connection(self, location):

if not location.user:

reason = (_("Location is missing user:password information."))

LOG.debug(reason)

raise exception.BadStoreUri(message=reason)

auth_url = location.swift_url

if not auth_url.endswith('/'):

auth_url += '/'

if self.auth_version == '2':

try:

tenant_name, user = location.user.split(':')

except ValueError:

reason = (_("Badly formed tenant:user '%(user)s' in "

"Swift URI") % {'user': location.user})

LOG.debug(reason)

raise exception.BadStoreUri()

os_options['region_name'] = self.region

os_options['endpoint_type'] = self.endpoint_type

os_options['service_type'] = self.service_type

return swiftclient.Connection(

auth_url, user, location.key, insecure=self.insecure,

tenant_name=tenant_name, snet=self.snet,

auth_version=self.auth_version, os_options=os_options,

ssl_compression=self.ssl_compression)

**** CubicPower OpenStack Study ****

class MultiTenantStore(BaseStore):

EXAMPLE_URL = "swift:////"

**** CubicPower OpenStack Study ****

def configure_add(self):

self.container = CONF.swift_store_container

if self.context is None:

reason = _("Multi-tenant Swift storage requires a context.")

raise exception.BadStoreConfiguration(store_name="swift",

reason=reason)

if self.context.service_catalog is None:

reason = _("Multi-tenant Swift storage requires "

"a service catalog.")

raise exception.BadStoreConfiguration(store_name="swift",

reason=reason)

self.storage_url = auth.get_endpoint(

self.context.service_catalog, service_type=self.service_type,

endpoint_region=self.region, endpoint_type=self.endpoint_type)

if self.storage_url.startswith('http://'):

self.scheme = 'swift+http'

else:

self.scheme = 'swift+https'

**** CubicPower OpenStack Study ****

def delete(self, location, connection=None):

if not connection:

connection = self.get_connection(location.store_location)

super(MultiTenantStore, self).delete(location, connection)

connection.delete_container(location.store_location.container)

**** CubicPower OpenStack Study ****

def set_acls(self, location, public=False, read_tenants=None,

write_tenants=None, connection=None):

location = location.store_location

if not connection:

connection = self.get_connection(location)

if read_tenants is None:

read_tenants = []

if write_tenants is None:

write_tenants = []

headers = {}

if public:

headers['X-Container-Read'] = ".r:*,.rlistings"

elif read_tenants:

headers['X-Container-Read'] = ','.join('%s:*' % i

for i in read_tenants)

else:

headers['X-Container-Read'] = ''

write_tenants.extend(self.admin_tenants)

if write_tenants:

headers['X-Container-Write'] = ','.join('%s:*' % i

for i in write_tenants)

else:

headers['X-Container-Write'] = ''

try:

connection.post_container(location.container, headers=headers)

except swiftclient.ClientException as e:

if e.http_status == httplib.NOT_FOUND:

msg = _("Swift could not find image at URI.")

raise exception.NotFound(msg)

else:

raise

**** CubicPower OpenStack Study ****

def create_location(self, image_id):

specs = {'scheme': self.scheme,

'container': self.container + '_' + str(image_id),

'obj': str(image_id),

'auth_or_store_url': self.storage_url}

return StoreLocation(specs)

**** CubicPower OpenStack Study ****

def get_connection(self, location):

return swiftclient.Connection(

None, self.context.user, None,

preauthurl=location.swift_url,

preauthtoken=self.context.auth_tok,

tenant_name=self.context.tenant,

auth_version='2', snet=self.snet, insecure=self.insecure,

ssl_compression=self.ssl_compression)

**** CubicPower OpenStack Study ****

class ChunkReader(object):

**** CubicPower OpenStack Study ****

def __init__(self, fd, checksum, total):

self.fd = fd

self.checksum = checksum

self.total = total

self.bytes_read = 0

**** CubicPower OpenStack Study ****

def read(self, i):

left = self.total - self.bytes_read

if i > left:

i = left

result = self.fd.read(i)

self.bytes_read += len(result)

self.checksum.update(result)

return result