Source Code for Module httxlib.httxnetlocation

  1  #!/usr/bin/env python 
  2  # -*- coding: latin-1; py-indent-offset:4 -*- 
  3  ################################################################################ 
  4  #  
  5  # This file is part of HttxLib 
  6  # 
  7  # HttxLib is an HTTP(s) Python library suited multithreaded/multidomain 
  8  # applications 
  9  # 
 10  # Copyright (C) 2010-2011 Daniel Rodriguez (aka Daniel Rodriksson) 
 11  # Copyright (C) 2011 Sensible Odds Ltd 
 12  # 
 13  # You can learn more and contact the author at: 
 14  # 
 15  #    http://code.google.com/p/httxlib/ 
 16  # 
 17  # HttxLib is free software: you can redistribute it and/or modify 
 18  # it under the terms of the GNU General Public License as published by 
 19  # the Free Software Foundation, either version 3 of the License, or 
 20  # (at your option) any later version. 
 21  # 
 22  # HttxLib is distributed in the hope that it will be useful, 
 23  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 24  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 25  # GNU General Public License for more details. 
 26  # 
 27  # You should have received a copy of the GNU General Public License 
 28  # along with HttxLib. If not, see <http://www.gnu.org/licenses/>. 
 29  # 
 30  ################################################################################ 
 31  ''' 
 32  Net Location connecting object L{HttxNetLocation} implementation 
 33  ''' 
 34   
 35  from collections import deque 
 36   
 37  from httxbase import HttxBase 
 38  from httxconnection import HttxConnection 
 39  from httxutil import tclock 
 40   
 41   
 42 -class HttxNetLocation(HttxBase): 
 43      ''' 
 44      Net Location Connecting object. The HttxNetLocation is responsible for creating 
 45      and managing a set of L{HttxConnection} connection objects that will  
 46      perform the actual connection 
 47   
 48      L{HttxConnection} connections will be created on demand and re-used if possible. 
 49      Active connections will be kept in a cache, whilst non-active will be kept 
 50      in a double queue (next in use to be popped from the right, and after usage 
 51      they wll be appended to the left 
 52   
 53      Due to the threading nature, a in-operation cache may hold connections during 
 54      manipulation 
 55   
 56      @ivar url: url used to set the net location to which connections will 
 57                 connect 
 58      @type url: str 
 59      @ivar httxconnque: The double queue holding non-active connections 
 60      @type options: collections.deque 
 61      @ivar httxconnache: Cache of connections with a pending request/response 
 62      @type options: dict 
 63      @ivar inopcache: Temporary in-operation cache for connections during request/response 
 64      @type options: set 
 65      ''' 
 66   
 67 -    def __init__(self, url, **kwargs): 
 68          ''' 
 69          Constructor. It delegates construction to the base class 
 70          L{HttxBase} and initializes the member variables 
 71   
 72          @param url: url to open a connection to 
 73          @type url: str 
 74          @param kwargs: keywords arguments passed to L{HttxBase} 
 75          @see: L{HttxOptions} 
 76          ''' 
 77          HttxBase.__init__(self, **kwargs) 
 78   
 79          self.url = url 
 80   
 81          # For connections currently with no pending network activity 
 82          self.httxconnque = deque() 
 83          # For connections with pending network activity 
 84          self.httxconncache = dict() 
 85          # Temporary cache after creation and before usage 
 86          self.inopcache = set() 
 87   
 88   
 89 -    def __deepcopy__(self, memo): 
 90          ''' 
 91          Deepcopy support. 
 92   
 93          @param memo: standard __deepcopy__ parameter to avoid circular references 
 94          @type memo: dict 
 95          @return: a cloned object 
 96          @rtype: L{HttxNetLocation} 
 97          @see L{clone} 
 98          ''' 
 99          return self.clone() 
100       
101   
102 -    def clone(self, options=None, connections=True): 
103          ''' 
104          Clone the object using the supplied options or a new set of options if 
105          given. 
106   
107          An equivalente set of L{HttxConnection} objects will be replicated 
108   
109          A new set of options will separate the clone object from the original 
110          object, since they will no longer share cookies, user/password/realm 
111          combinations or https certificates 
112   
113          @param options: options for the cloned object 
114          @type options: L{HttxOptions} 
115          @param connections: whether to clone the existing connections 
116          @type connections: bool 
117          @return: a cloned object 
118          @rtype: L{HttxNetLocation} 
119          ''' 
120          if not options: 
121              options = self.options.clone() 
122          clone = self.__class__(self.url, options=options) 
123   
124          with self.lock: 
125              if connections: 
126                  for conniterable in (self.httxconnque, self.inopcache, self.httxconncache.itervalues()): 
127                      for httxconn in conniterable: 
128                          clone.httxconnque.appendleft(httxconn.clone(options)) 
129   
130          return clone 
131   
132   
133 -    def request(self, httxreq): 
134          ''' 
135          Send the L{HttxRequest} httxreq to the specified server inside the request 
136          It does get a connection or create one and relay the request down to it, taking 
137          into account the HTTP keepalive timeout 
138           
139          @param httxreq: Request or url to be executed 
140          @type httxreq: L{HttxRequest} or url (string) 
141          @return: sock 
142          @rtype: opaque type for the caller (a Python sock) 
143          ''' 
144          with self.lock: 
145              try: 
146                  httxconn = self.httxconnque.pop() 
147                  # Check if the HTTP keepalive timeout has been exceeded 
148                  if (tclock() - httxconn.timestamp) >= self.options.keepalive: 
149                      # Simulate that no connection was available 
150                      raise IndexError 
151                  # The connection is no longer in any container, it will be discarded 
152              except IndexError: 
153                  # use self.url and not the request url ... to be able to proxy connections 
154                  httxconn = HttxConnection(self.url, options=self.options) 
155   
156              # keep a copy of this dangling connecion in a set to avoid missing it if 
157              # we are cloning an object and some threads find themselves issuing requests 
158              self.inopcache.add(httxconn) 
159   
160          try: 
161              sock = httxconn.request(httxreq) 
162          except: 
163              # let it be reused 
164              httxconn.reset() 
165              with self.lock: 
166                  # no longer in operation 
167                  self.inopcache.discard(httxconn) 
168                  # back to conn queue 
169                  self.httxconnque.appendleft(httxconn) 
170              # let the exception propagate 
171              raise 
172   
173          # Remove the cache from the in-operation cache and place it in the 
174          # cache for connections with pending network activity 
175          with self.lock: 
176              self.inopcache.discard(httxconn) 
177              self.httxconncache[sock] = httxconn 
178   
179          return sock 
180   
181   
182 -    def getresponse(self, sock): 
183          ''' 
184          Recover a L{HttxResponse} using the connection that is in the cache 
185          indexed by sock and calling its getresponse 
186           
187          @param sock: The opaque type returned by L{request} 
188          @type sock: opaque (a Python sock) 
189          @return: response 
190          @rtype: L{HttxResponse} (compatible with httplib HTTPResponse) 
191          ''' 
192          with self.lock: 
193              httxconn = self.httxconncache.pop(sock) 
194              # keep a copy of this dangling connecion in a set to avoid missing it if 
195              # we are cloning an object and some threads find themselves issuing requests 
196              self.inopcache.add(httxconn) 
197   
198          try: 
199              response = httxconn.getresponse(sock) 
200          except: 
201              httxconn.reset() 
202              with self.lock: 
203                  self.inopcache.discard(httxconn) 
204                  self.httxconnque.appendleft(httxconn) 
205   
206              # Let the exception propagate 
207              raise 
208   
209          # Put it back into the connection queue unless not ready 
210          with self.lock: 
211              self.inopcache.discard(httxconn) 
212   
213              if not response.isactive(): 
214                  self.httxconnque.appendleft(httxconn) 
215              else: 
216                  # Redirecting or authenticating and therefore still with 
217                  # pending network activity. Back to the active conn cache 
218                  self.httxconncache[response.sock] = httxconn 
219   
220          return response 
221