1 """
2 connection operations
3
4 Connection instances are used to communicate with the remote service at
5 the account level creating, listing and deleting Containers, and returning
6 Container instances.
7
8 See COPYING for license information.
9 """
10
11 import socket
12 import os
13 from urllib import urlencode
14 from httplib import HTTPSConnection, HTTPConnection, HTTPException
15 from container import Container, ContainerResults
16 from utils import unicode_quote, parse_url, THTTPConnection, THTTPSConnection
17 from errors import ResponseError, NoSuchContainer, ContainerNotEmpty, \
18 InvalidContainerName, CDNNotEnabled, ContainerExists
19 from Queue import Queue, Empty, Full
20 from time import time
21 import consts
22 from authentication import Authentication
23 from fjson import json_loads
24 from sys import version_info
25
26
27
28
29
31 """
32 Manages the connection to the storage system and serves as a factory
33 for Container instances.
34
35 @undocumented: cdn_connect
36 @undocumented: http_connect
37 @undocumented: cdn_request
38 @undocumented: make_request
39 @undocumented: _check_container_name
40 """
41
42 - def __init__(self, username=None, api_key=None, timeout=5, **kwargs):
43 """
44 Accepts keyword arguments for Mosso username and api key.
45 Optionally, you can omit these keywords and supply an
46 Authentication object using the auth keyword. Setting the argument
47 servicenet to True will make use of Rackspace servicenet network.
48
49 @type username: str
50 @param username: a Mosso username
51 @type api_key: str
52 @param api_key: a Mosso API key
53 @type servicenet: bool
54 @param servicenet: Use Rackspace servicenet to access Cloud Files.
55 @type cdn_log_retention: bool
56 @param cdn_log_retention: set logs retention for this cdn enabled
57 container.
58 """
59 self.cdn_enabled = False
60 self.cdn_args = None
61 self.connection_args = None
62 self.cdn_connection = None
63 self.connection = None
64 self.token = None
65 self.debuglevel = int(kwargs.get('debuglevel', 0))
66 self.servicenet = kwargs.get('servicenet', False)
67 self.user_agent = kwargs.get('useragent', consts.user_agent)
68 self.timeout = timeout
69
70
71
72 if not 'servicenet' in kwargs \
73 and 'RACKSPACE_SERVICENET' in os.environ:
74 self.servicenet = True
75
76 self.auth = 'auth' in kwargs and kwargs['auth'] or None
77
78 if not self.auth:
79 authurl = kwargs.get('authurl', consts.us_authurl)
80 if username and api_key and authurl:
81 self.auth = Authentication(username, api_key, authurl=authurl,
82 useragent=self.user_agent, timeout=self.timeout)
83 else:
84 raise TypeError("Incorrect or invalid arguments supplied")
85 self._authenticate()
87 """
88 Authenticate and setup this instance with the values returned.
89 """
90 (url, self.cdn_url, self.token) = self.auth.authenticate()
91 url = self._set_storage_url(url)
92 self.connection_args = parse_url(url)
93
94 if version_info[0] <= 2 and version_info[1] < 6:
95 self.conn_class = self.connection_args[3] and THTTPSConnection or \
96 THTTPConnection
97 else:
98 self.conn_class = self.connection_args[3] and HTTPSConnection or \
99 HTTPConnection
100 self.http_connect()
101 if self.cdn_url:
102 self.cdn_connect()
103
105 if self.servicenet:
106 return "https://snet-%s" % url.replace("https://", "")
107 return url
108
110 """
111 Setup the http connection instance for the CDN service.
112 """
113 (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
114 self.cdn_connection = self.conn_class(host, port, timeout=self.timeout)
115 self.cdn_enabled = True
116
118 """
119 Setup the http connection instance.
120 """
121 (host, port, self.uri, is_ssl) = self.connection_args
122 self.connection = self.conn_class(host, port=port, \
123 timeout=self.timeout)
124 self.connection.set_debuglevel(self.debuglevel)
125
126 - def cdn_request(self, method, path=[], data='', hdrs=None):
127 """
128 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
129 metadata dicts, performs an http request against the CDN service.
130 """
131 if not self.cdn_enabled:
132 raise CDNNotEnabled()
133
134 path = '/%s/%s' % \
135 (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
136 headers = {'Content-Length': str(len(data)),
137 'User-Agent': self.user_agent,
138 'X-Auth-Token': self.token}
139 if isinstance(hdrs, dict):
140 headers.update(hdrs)
141
142 def retry_request():
143 '''Re-connect and re-try a failed request once'''
144 self.cdn_connect()
145 self.cdn_connection.request(method, path, data, headers)
146 return self.cdn_connection.getresponse()
147
148 try:
149 self.cdn_connection.request(method, path, data, headers)
150 response = self.cdn_connection.getresponse()
151 except (socket.error, IOError, HTTPException):
152 response = retry_request()
153 if response.status == 401:
154 self._authenticate()
155 headers['X-Auth-Token'] = self.token
156 response = retry_request()
157
158 return response
159
160 - def make_request(self, method, path=[], data='', hdrs=None, parms=None):
161 """
162 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
163 metadata dicts, and an optional dictionary of query parameters,
164 performs an http request.
165 """
166 path = '/%s/%s' % \
167 (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
168
169 if isinstance(parms, dict) and parms:
170 path = '%s?%s' % (path, urlencode(parms))
171
172 headers = {'Content-Length': str(len(data)),
173 'User-Agent': self.user_agent,
174 'X-Auth-Token': self.token}
175 isinstance(hdrs, dict) and headers.update(hdrs)
176
177 def retry_request():
178 '''Re-connect and re-try a failed request once'''
179 self.http_connect()
180 self.connection.request(method, path, data, headers)
181 return self.connection.getresponse()
182
183 try:
184 self.connection.request(method, path, data, headers)
185 response = self.connection.getresponse()
186 except (socket.error, IOError, HTTPException):
187 response = retry_request()
188 if response.status == 401:
189 self._authenticate()
190 headers['X-Auth-Token'] = self.token
191 response = retry_request()
192
193 return response
194
196 """
197 Return tuple for number of containers, total bytes in the account and account metadata
198
199 >>> connection.get_info()
200 (5, 2309749)
201
202 @rtype: tuple
203 @return: a tuple containing the number of containers, total bytes
204 used by the account and a dictionary containing account metadata
205 """
206 response = self.make_request('HEAD')
207 count = size = None
208 metadata = {}
209 for hdr in response.getheaders():
210 if hdr[0].lower() == 'x-account-container-count':
211 try:
212 count = int(hdr[1])
213 except ValueError:
214 count = 0
215 if hdr[0].lower() == 'x-account-bytes-used':
216 try:
217 size = int(hdr[1])
218 except ValueError:
219 size = 0
220 if hdr[0].lower().startswith('x-account-meta-'):
221 metadata[hdr[0].lower()[15:]] = hdr[1]
222 buff = response.read()
223 if (response.status < 200) or (response.status > 299):
224 raise ResponseError(response.status, response.reason)
225 return (count, size, metadata)
226
240
242 if not container_name or \
243 '/' in container_name or \
244 len(container_name) > consts.container_name_limit:
245 raise InvalidContainerName(container_name)
246
248 """
249 Given a container name, returns a L{Container} item, creating a new
250 Container if one does not already exist.
251
252 >>> connection.create_container('new_container')
253 <cloudfiles.container.Container object at 0xb77d628c>
254
255 @param container_name: name of the container to create
256 @type container_name: str
257 @param error_on_existing: raise ContainerExists if container already
258 exists
259 @type error_on_existing: bool
260 @rtype: L{Container}
261 @return: an object representing the newly created container
262 """
263 self._check_container_name(container_name)
264
265 response = self.make_request('PUT', [container_name])
266 buff = response.read()
267 if (response.status < 200) or (response.status > 299):
268 raise ResponseError(response.status, response.reason)
269 if error_on_existing and (response.status == 202):
270 raise ContainerExists(container_name)
271 return Container(self, container_name)
272
274 """
275 Given a container name, delete it.
276
277 >>> connection.delete_container('old_container')
278
279 @param container_name: name of the container to delete
280 @type container_name: str
281 """
282 if isinstance(container_name, Container):
283 container_name = container_name.name
284 self._check_container_name(container_name)
285
286 response = self.make_request('DELETE', [container_name])
287 response.read()
288
289 if (response.status == 409):
290 raise ContainerNotEmpty(container_name)
291 elif (response.status == 404):
292 raise NoSuchContainer
293 elif (response.status < 200) or (response.status > 299):
294 raise ResponseError(response.status, response.reason)
295
296 if self.cdn_enabled:
297 response = self.cdn_request('POST', [container_name],
298 hdrs={'X-CDN-Enabled': 'False'})
299
301 """
302 Returns a Container item result set.
303
304 >>> connection.get_all_containers()
305 ContainerResults: 4 containers
306 >>> print ', '.join([container.name for container in
307 connection.get_all_containers()])
308 new_container, old_container, pictures, music
309
310 @rtype: L{ContainerResults}
311 @return: an iterable set of objects representing all containers on the
312 account
313 @param limit: number of results to return, up to 10,000
314 @type limit: int
315 @param marker: return only results whose name is greater than "marker"
316 @type marker: str
317 """
318 if limit:
319 parms['limit'] = limit
320 if marker:
321 parms['marker'] = marker
322 return ContainerResults(self, self.list_containers_info(**parms))
323
325 """
326 Return a single Container item for the given Container.
327
328 >>> connection.get_container('old_container')
329 <cloudfiles.container.Container object at 0xb77d628c>
330 >>> container = connection.get_container('old_container')
331 >>> container.size_used
332 23074
333
334 @param container_name: name of the container to create
335 @type container_name: str
336 @rtype: L{Container}
337 @return: an object representing the container
338 """
339 self._check_container_name(container_name)
340
341 response = self.make_request('HEAD', [container_name])
342 count = size = None
343 metadata = {}
344 for hdr in response.getheaders():
345 if hdr[0].lower() == 'x-container-object-count':
346 try:
347 count = int(hdr[1])
348 except ValueError:
349 count = 0
350 if hdr[0].lower() == 'x-container-bytes-used':
351 try:
352 size = int(hdr[1])
353 except ValueError:
354 size = 0
355 if hdr[0].lower().startswith('x-container-meta-'):
356 metadata[hdr[0].lower()[17:]] = hdr[1]
357 buff = response.read()
358 if response.status == 404:
359 raise NoSuchContainer(container_name)
360 if (response.status < 200) or (response.status > 299):
361 raise ResponseError(response.status, response.reason)
362 return Container(self, container_name, count, size, metadata)
363
365 """
366 Returns a list of containers that have been published to the CDN.
367
368 >>> connection.list_public_containers()
369 ['container1', 'container2', 'container3']
370
371 @rtype: list(str)
372 @return: a list of all CDN-enabled container names as strings
373 """
374 response = self.cdn_request('GET', [''])
375 if (response.status < 200) or (response.status > 299):
376 buff = response.read()
377 raise ResponseError(response.status, response.reason)
378 return response.read().splitlines()
379
381 """
382 Returns a list of Containers, including object count and size.
383
384 >>> connection.list_containers_info()
385 [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'},
386 {u'count': 12, u'bytes': 23074, u'name': u'old_container'},
387 {u'count': 0, u'bytes': 0, u'name': u'container1'},
388 {u'count': 0, u'bytes': 0, u'name': u'container2'},
389 {u'count': 0, u'bytes': 0, u'name': u'container3'},
390 {u'count': 3, u'bytes': 2306, u'name': u'test'}]
391
392 @rtype: list({"name":"...", "count":..., "bytes":...})
393 @return: a list of all container info as dictionaries with the
394 keys "name", "count", and "bytes"
395 @param limit: number of results to return, up to 10,000
396 @type limit: int
397 @param marker: return only results whose name is greater than "marker"
398 @type marker: str
399 """
400 if limit:
401 parms['limit'] = limit
402 if marker:
403 parms['marker'] = marker
404 parms['format'] = 'json'
405 response = self.make_request('GET', [''], parms=parms)
406 if (response.status < 200) or (response.status > 299):
407 buff = response.read()
408 raise ResponseError(response.status, response.reason)
409 return json_loads(response.read())
410
412 """
413 Returns a list of Containers.
414
415 >>> connection.list_containers()
416 ['new_container',
417 'old_container',
418 'container1',
419 'container2',
420 'container3',
421 'test']
422
423 @rtype: list(str)
424 @return: a list of all containers names as strings
425 @param limit: number of results to return, up to 10,000
426 @type limit: int
427 @param marker: return only results whose name is greater than "marker"
428 @type marker: str
429 """
430 if limit:
431 parms['limit'] = limit
432 if marker:
433 parms['marker'] = marker
434 response = self.make_request('GET', [''], parms=parms)
435 if (response.status < 200) or (response.status > 299):
436 buff = response.read()
437 raise ResponseError(response.status, response.reason)
438 return response.read().splitlines()
439
441 """
442 Container objects can be grabbed from a connection using index
443 syntax.
444
445 >>> container = conn['old_container']
446 >>> container.size_used
447 23074
448
449 @rtype: L{Container}
450 @return: an object representing the container
451 """
452 return self.get_container(key)
453
454
456 """
457 A thread-safe connection pool object.
458
459 This component isn't required when using the cloudfiles library, but it may
460 be useful when building threaded applications.
461 """
462
463 - def __init__(self, username=None, api_key=None, **kwargs):
464 poolsize = kwargs.pop('poolsize', 10)
465 self.connargs = {'username': username, 'api_key': api_key}
466 self.connargs.update(kwargs)
467 Queue.__init__(self, poolsize)
468
470 """
471 Return a cloudfiles connection object.
472
473 @rtype: L{Connection}
474 @return: a cloudfiles connection object
475 """
476 try:
477 (create, connobj) = Queue.get(self, block=0)
478 except Empty:
479 connobj = Connection(**self.connargs)
480 return connobj
481
482 - def put(self, connobj):
483 """
484 Place a cloudfiles connection object back into the pool.
485
486 @param connobj: a cloudfiles connection object
487 @type connobj: L{Connection}
488 """
489 try:
490 Queue.put(self, (time(), connobj), block=0)
491 except Full:
492 del connobj
493
494