Package cloudfiles :: Module connection
[frames] | no frames]

Source Code for Module cloudfiles.connection

  1  """ 
  2  connection operations 
  3   
  4  Connection instances are used to communicate with the remote service at 
  5  the account level creating, listing and deleting Containers, and returning 
  6  Container instances. 
  7   
  8  See COPYING for license information. 
  9  """ 
 10   
 11  import  socket 
 12  from    urllib    import quote 
 13  from    httplib   import HTTPSConnection, HTTPConnection, HTTPException 
 14  from    container import Container, ContainerResults 
 15  from    utils     import parse_url 
 16  from    errors    import ResponseError, NoSuchContainer, ContainerNotEmpty, \ 
 17                           InvalidContainerName, CDNNotEnabled 
 18  from    Queue     import Queue, Empty, Full 
 19  from    time      import time 
 20  import  consts 
 21  from    authentication import Authentication 
 22  from    fjson     import json_loads 
 23   
 24  # Because HTTPResponse objects *have* to have read() called on them  
 25  # before they can be used again ... 
 26  # pylint: disable-msg=W0612 
 27   
28 -class Connection(object):
29 """ 30 Manages the connection to the storage system and serves as a factory 31 for Container instances. 32 33 @undocumented: cdn_connect 34 @undocumented: http_connect 35 @undocumented: cdn_request 36 @undocumented: make_request 37 @undocumented: _check_container_name 38 """
39 - def __init__(self, username=None, api_key=None, **kwargs):
40 """ 41 Accepts keyword arguments for Mosso username and api key. 42 Optionally, you can omit these keywords and supply an 43 Authentication object using the auth keyword. Setting the argument 44 servicenet to True will make use of Rackspace servicenet network. 45 46 @type username: str 47 @param username: a Mosso username 48 @type api_key: str 49 @param api_key: a Mosso API key 50 @ivar servicenet: Use Rackspace servicenet to access Cloud Files. 51 @type cdn_log_retention: bool 52 """ 53 self.cdn_enabled = False 54 self.cdn_args = None 55 self.connection_args = None 56 self.cdn_connection = None 57 self.connection = None 58 self.token = None 59 self.debuglevel = int(kwargs.get('debuglevel', 0)) 60 self.servicenet = kwargs.get('servicenet', False) 61 socket.setdefaulttimeout = int(kwargs.get('timeout', 5)) 62 self.auth = kwargs.has_key('auth') and kwargs['auth'] or None 63 64 if not self.auth: 65 authurl = kwargs.get('authurl', consts.default_authurl) 66 if username and api_key and authurl: 67 self.auth = Authentication(username, api_key, authurl) 68 else: 69 raise TypeError("Incorrect or invalid arguments supplied") 70 71 self._authenticate()
72
73 - def _authenticate(self):
74 """ 75 Authenticate and setup this instance with the values returned. 76 """ 77 (url, self.cdn_url, self.token) = self.auth.authenticate() 78 url = self._set_storage_url(url) 79 self.connection_args = parse_url(url) 80 self.conn_class = self.connection_args[3] and HTTPSConnection or \ 81 HTTPConnection 82 self.http_connect() 83 if self.cdn_url: 84 self.cdn_connect()
85
86 - def _set_storage_url(self, url):
87 if self.servicenet: 88 return "https://snet-%s" % url.replace("https://", "") 89 return url
90
91 - def cdn_connect(self):
92 """ 93 Setup the http connection instance for the CDN service. 94 """ 95 (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url) 96 conn_class = is_ssl and HTTPSConnection or HTTPConnection 97 self.cdn_connection = conn_class(host, port) 98 self.cdn_enabled = True
99
100 - def http_connect(self):
101 """ 102 Setup the http connection instance. 103 """ 104 (host, port, self.uri, is_ssl) = self.connection_args 105 self.connection = self.conn_class(host, port=port) 106 self.connection.set_debuglevel(self.debuglevel)
107
108 - def cdn_request(self, method, path=[], data='', hdrs=None):
109 """ 110 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 111 metadata dicts, performs an http request against the CDN service. 112 """ 113 if not self.cdn_enabled: 114 raise CDNNotEnabled() 115 116 path = '/%s/%s' % \ 117 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) 118 headers = {'Content-Length': len(data), 'User-Agent': consts.user_agent, 119 'X-Auth-Token': self.token} 120 if isinstance(hdrs, dict): 121 headers.update(hdrs) 122 123 # Send the request 124 self.cdn_connection.request(method, path, data, headers) 125 126 def retry_request(): 127 '''Re-connect and re-try a failed request once''' 128 self.cdn_connect() 129 self.cdn_connection.request(method, path, data, headers) 130 return self.cdn_connection.getresponse()
131 132 try: 133 response = self.cdn_connection.getresponse() 134 except HTTPException: 135 response = retry_request() 136 137 if response.status == 401: 138 self._authenticate() 139 response = retry_request() 140 141 return response
142 143
144 - def make_request(self, method, path=[], data='', hdrs=None, parms=None):
145 """ 146 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 147 metadata dicts, and an optional dictionary of query parameters, 148 performs an http request. 149 """ 150 path = '/%s/%s' % \ 151 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) 152 153 if isinstance(parms, dict) and parms: 154 query_args = \ 155 ['%s=%s' % (quote(x),quote(str(y))) for (x,y) in parms.items()] 156 path = '%s?%s' % (path, '&'.join(query_args)) 157 158 headers = {'Content-Length': len(data), 'User-Agent': consts.user_agent, 159 'X-Auth-Token': self.token} 160 isinstance(hdrs, dict) and headers.update(hdrs) 161 162 def retry_request(): 163 '''Re-connect and re-try a failed request once''' 164 self.http_connect() 165 self.connection.request(method, path, data, headers) 166 return self.connection.getresponse()
167 168 try: 169 self.connection.request(method, path, data, headers) 170 response = self.connection.getresponse() 171 except HTTPException: 172 response = retry_request() 173 174 if response.status == 401: 175 self._authenticate() 176 response = retry_request() 177 178 return response 179
180 - def get_info(self):
181 """ 182 Return tuple for number of containers and total bytes in the account 183 184 >>> connection.get_info() 185 (5, 2309749) 186 187 @rtype: tuple 188 @return: a tuple containing the number of containers and total bytes 189 used by the account 190 """ 191 response = self.make_request('HEAD') 192 count = size = None 193 for hdr in response.getheaders(): 194 if hdr[0].lower() == 'x-account-container-count': 195 try: 196 count = int(hdr[1]) 197 except ValueError: 198 count = 0 199 if hdr[0].lower() == 'x-account-bytes-used': 200 try: 201 size = int(hdr[1]) 202 except ValueError: 203 size = 0 204 buff = response.read() 205 if (response.status < 200) or (response.status > 299): 206 raise ResponseError(response.status, response.reason) 207 return (count, size)
208
209 - def _check_container_name(self, container_name):
210 if not container_name or \ 211 '/' in container_name or \ 212 len(container_name) > consts.container_name_limit: 213 raise InvalidContainerName(container_name)
214
215 - def create_container(self, container_name):
216 """ 217 Given a container name, returns a L{Container} item, creating a new 218 Container if one does not already exist. 219 220 >>> connection.create_container('new_container') 221 <cloudfiles.container.Container object at 0xb77d628c> 222 223 @param container_name: name of the container to create 224 @type container_name: str 225 @rtype: L{Container} 226 @return: an object representing the newly created container 227 """ 228 self._check_container_name(container_name) 229 230 response = self.make_request('PUT', [container_name]) 231 buff = response.read() 232 if (response.status < 200) or (response.status > 299): 233 raise ResponseError(response.status, response.reason) 234 return Container(self, container_name)
235
236 - def delete_container(self, container_name):
237 """ 238 Given a container name, delete it. 239 240 >>> connection.delete_container('old_container') 241 242 @param container_name: name of the container to delete 243 @type container_name: str 244 """ 245 if isinstance(container_name, Container): 246 container_name = container_name.name 247 self._check_container_name(container_name) 248 249 response = self.make_request('DELETE', [container_name]) 250 buff = response.read() 251 252 if (response.status == 409): 253 raise ContainerNotEmpty(container_name) 254 elif (response.status < 200) or (response.status > 299): 255 raise ResponseError(response.status, response.reason) 256 257 if self.cdn_enabled: 258 response = self.cdn_request('POST', [container_name], 259 hdrs={'X-CDN-Enabled': 'False'})
260
261 - def get_all_containers(self, limit=None, marker=None, **parms):
262 """ 263 Returns a Container item result set. 264 265 >>> connection.get_all_containers() 266 ContainerResults: 4 containers 267 >>> print ', '.join([container.name for container in 268 connection.get_all_containers()]) 269 new_container, old_container, pictures, music 270 271 @rtype: L{ContainerResults} 272 @return: an iterable set of objects representing all containers on the 273 account 274 @param limit: number of results to return, up to 10,000 275 @type limit: int 276 @param marker: return only results whose name is greater than "marker" 277 @type marker: str 278 """ 279 if limit: 280 parms['limit'] = limit 281 if marker: 282 parms['marker'] = marker 283 return ContainerResults(self, self.list_containers_info(**parms))
284
285 - def get_container(self, container_name):
286 """ 287 Return a single Container item for the given Container. 288 289 >>> connection.get_container('old_container') 290 <cloudfiles.container.Container object at 0xb77d628c> 291 >>> container = connection.get_container('old_container') 292 >>> container.size_used 293 23074 294 295 @param container_name: name of the container to create 296 @type container_name: str 297 @rtype: L{Container} 298 @return: an object representing the container 299 """ 300 self._check_container_name(container_name) 301 302 response = self.make_request('HEAD', [container_name]) 303 count = size = None 304 for hdr in response.getheaders(): 305 if hdr[0].lower() == 'x-container-object-count': 306 try: 307 count = int(hdr[1]) 308 except ValueError: 309 count = 0 310 if hdr[0].lower() == 'x-container-bytes-used': 311 try: 312 size = int(hdr[1]) 313 except ValueError: 314 size = 0 315 buff = response.read() 316 if response.status == 404: 317 raise NoSuchContainer(container_name) 318 if (response.status < 200) or (response.status > 299): 319 raise ResponseError(response.status, response.reason) 320 return Container(self, container_name, count, size)
321
322 - def list_public_containers(self):
323 """ 324 Returns a list of containers that have been published to the CDN. 325 326 >>> connection.list_public_containers() 327 ['container1', 'container2', 'container3'] 328 329 @rtype: list(str) 330 @return: a list of all CDN-enabled container names as strings 331 """ 332 response = self.cdn_request('GET', ['']) 333 if (response.status < 200) or (response.status > 299): 334 buff = response.read() 335 raise ResponseError(response.status, response.reason) 336 return response.read().splitlines()
337
338 - def list_containers_info(self, limit=None, marker=None, **parms):
339 """ 340 Returns a list of Containers, including object count and size. 341 342 >>> connection.list_containers_info() 343 [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'}, 344 {u'count': 12, u'bytes': 23074, u'name': u'old_container'}, 345 {u'count': 0, u'bytes': 0, u'name': u'container1'}, 346 {u'count': 0, u'bytes': 0, u'name': u'container2'}, 347 {u'count': 0, u'bytes': 0, u'name': u'container3'}, 348 {u'count': 3, u'bytes': 2306, u'name': u'test'}] 349 350 @rtype: list({"name":"...", "count":..., "bytes":...}) 351 @return: a list of all container info as dictionaries with the 352 keys "name", "count", and "bytes" 353 @param limit: number of results to return, up to 10,000 354 @type limit: int 355 @param marker: return only results whose name is greater than "marker" 356 @type marker: str 357 """ 358 if limit: 359 parms['limit'] = limit 360 if marker: 361 parms['marker'] = marker 362 parms['format'] = 'json' 363 response = self.make_request('GET', [''], parms=parms) 364 if (response.status < 200) or (response.status > 299): 365 buff = response.read() 366 raise ResponseError(response.status, response.reason) 367 return json_loads(response.read())
368
369 - def list_containers(self, limit=None, marker=None, **parms):
370 """ 371 Returns a list of Containers. 372 373 >>> connection.list_containers() 374 ['new_container', 375 'old_container', 376 'container1', 377 'container2', 378 'container3', 379 'test'] 380 381 @rtype: list(str) 382 @return: a list of all containers names as strings 383 @param limit: number of results to return, up to 10,000 384 @type limit: int 385 @param marker: return only results whose name is greater than "marker" 386 @type marker: str 387 """ 388 if limit: 389 parms['limit'] = limit 390 if marker: 391 parms['marker'] = marker 392 response = self.make_request('GET', [''], parms=parms) 393 if (response.status < 200) or (response.status > 299): 394 buff = response.read() 395 raise ResponseError(response.status, response.reason) 396 return response.read().splitlines()
397
398 - def __getitem__(self, key):
399 """ 400 Container objects can be grabbed from a connection using index 401 syntax. 402 403 >>> container = conn['old_container'] 404 >>> container.size_used 405 23074 406 407 @rtype: L{Container} 408 @return: an object representing the container 409 """ 410 return self.get_container(key)
411
412 -class ConnectionPool(Queue):
413 """ 414 A thread-safe connection pool object. 415 416 This component isn't required when using the cloudfiles library, but it may 417 be useful when building threaded applications. 418 """
419 - def __init__(self, username=None, api_key=None, **kwargs):
420 auth = kwargs.get('auth', None) 421 self.timeout = kwargs.get('timeout', 5) 422 self.connargs = {'username': username, 'api_key': api_key} 423 poolsize = kwargs.get('poolsize', 10) 424 Queue.__init__(self, poolsize)
425
426 - def get(self):
427 """ 428 Return a cloudfiles connection object. 429 430 @rtype: L{Connection} 431 @return: a cloudfiles connection object 432 """ 433 try: 434 (create, connobj) = Queue.get(self, block=0) 435 except Empty: 436 connobj = Connection(**self.connargs) 437 return connobj
438
439 - def put(self, connobj):
440 """ 441 Place a cloudfiles connection object back into the pool. 442 443 @param connobj: a cloudfiles connection object 444 @type connobj: L{Connection} 445 """ 446 try: 447 Queue.put(self, (time(), connobj), block=0) 448 except Full: 449 del connobj
450 451 # vim:set ai sw=4 ts=4 tw=0 expandtab: 452