Package restkit :: Module client
[hide private]
[frames] | no frames]

Source Code for Module restkit.client

  1  # -*- coding: utf-8 - 
  2  # 
  3  # This file is part of restkit released under the MIT license. 
  4  # See the NOTICE for more information. 
  5  import base64 
  6  import errno 
  7  import logging 
  8  import os 
  9  import time 
 10  import socket 
 11  import ssl 
 12  import traceback 
 13  import types 
 14  import urlparse 
 15   
 16  try: 
 17      from http_parser.http import HttpStream 
 18      from http_parser.reader import SocketReader 
 19  except ImportError: 
 20      raise ImportError("""http-parser isn't installed. 
 21   
 22          pip install http-parser""") 
 23   
 24  from restkit import __version__ 
 25   
 26  from restkit.conn import Connection 
 27  from restkit.errors import RequestError, RequestTimeout, RedirectLimit, \ 
 28  NoMoreData, ProxyError 
 29  from restkit.session import get_session 
 30  from restkit.util import parse_netloc, rewrite_location 
 31  from restkit.wrappers import Request, Response 
 32   
 33  MAX_CLIENT_TIMEOUT=300 
 34  MAX_CLIENT_CONNECTIONS = 5 
 35  MAX_CLIENT_TRIES =3 
 36  CLIENT_WAIT_TRIES = 0.3 
 37  MAX_FOLLOW_REDIRECTS = 5 
 38  USER_AGENT = "restkit/%s" % __version__ 
 39   
 40  log = logging.getLogger(__name__) 
 41   
42 -class Client(object):
43 44 """ A client handle a connection at a time. A client is threadsafe, 45 but an handled shouldn't be shared between threads. All connections 46 are shared between threads via a pool. 47 48 >>> from restkit import * 49 >>> c = Client() 50 >>> r = c.request("http://google.com") 51 r>>> r.status 52 '301 Moved Permanently' 53 >>> r.body_string() 54 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n' 55 >>> c.follow_redirect = True 56 >>> r = c.request("http://google.com") 57 >>> r.status 58 '200 OK' 59 60 """ 61 62 version = (1, 1) 63 response_class=Response 64
65 - def __init__(self, 66 follow_redirect=False, 67 force_follow_redirect=False, 68 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 69 filters=None, 70 decompress=True, 71 max_status_line_garbage=None, 72 max_header_count=0, 73 session=None, 74 response_class=None, 75 timeout=None, 76 use_proxy=False, 77 max_tries=3, 78 wait_tries=1.0, 79 backend="thread", 80 **ssl_args):
81 """ 82 Client parameters 83 ~~~~~~~~~~~~~~~~~ 84 85 :param follow_redirect: follow redirection, by default False 86 :param max_ollow_redirect: number of redirections available 87 :filters: http filters to pass 88 :param decompress: allows the client to decompress the response 89 body 90 :param max_status_line_garbage: defines the maximum number of ignorable 91 lines before we expect a HTTP response's status line. With 92 HTTP/1.1 persistent connections, the problem arises that broken 93 scripts could return a wrong Content-Length (there are more 94 bytes sent than specified). Unfortunately, in some cases, this 95 cannot be detected after the bad response, but only before the 96 next one. So the client is abble to skip bad lines using this 97 limit. 0 disable garbage collection, None means unlimited number 98 of tries. 99 :param max_header_count: determines the maximum HTTP header count 100 allowed. by default no limit. 101 :param manager: the manager to use. By default we use the global 102 one. 103 :parama response_class: the response class to use 104 :param timeout: the default timeout of the connection 105 (SO_TIMEOUT) 106 107 :param max_tries: the number of tries before we give up a 108 connection 109 :param wait_tries: number of time we wait between each tries. 110 :param ssl_args: named argument, see ssl module for more 111 informations 112 """ 113 self.follow_redirect = follow_redirect 114 self.force_follow_redirect = force_follow_redirect 115 self.max_follow_redirect = max_follow_redirect 116 self.decompress = decompress 117 self.filters = filters or [] 118 self.max_status_line_garbage = max_status_line_garbage 119 self.max_header_count = max_header_count 120 self.use_proxy = use_proxy 121 122 self.request_filters = [] 123 self.response_filters = [] 124 self.load_filters() 125 126 127 # set manager 128 129 session_options = dict( 130 retry_delay=wait_tries, 131 retry_max = max_tries, 132 timeout = timeout) 133 134 135 if session is None: 136 session = get_session(backend, **session_options) 137 self._session = session 138 self.backend = backend 139 140 # change default response class 141 if response_class is not None: 142 self.response_class = response_class 143 144 self.max_tries = max_tries 145 self.wait_tries = wait_tries 146 self.timeout = timeout 147 148 self._nb_redirections = self.max_follow_redirect 149 self._url = None 150 self._initial_url = None 151 self._write_cb = None 152 self._headers = None 153 self._sock_key = None 154 self._sock = None 155 self._original = None 156 157 self.method = 'GET' 158 self.body = None 159 self.ssl_args = ssl_args or {}
160
161 - def load_filters(self):
162 """ Populate filters from self.filters. 163 Must be called each time self.filters is updated. 164 """ 165 for f in self.filters: 166 if hasattr(f, "on_request"): 167 self.request_filters.append(f) 168 if hasattr(f, "on_response"): 169 self.response_filters.append(f)
170 171 172
173 - def get_connection(self, request):
174 """ get a connection from the pool or create new one. """ 175 176 addr = parse_netloc(request.parsed_url) 177 is_ssl = request.is_ssl() 178 179 extra_headers = [] 180 conn = None 181 if self.use_proxy: 182 conn = self.proxy_connection(request, 183 addr, is_ssl) 184 if not conn: 185 conn = self._session.get(host=addr[0], port=addr[1], 186 pool=self._session, is_ssl=is_ssl, 187 extra_headers=extra_headers, **self.ssl_args) 188 189 190 return conn
191
192 - def proxy_connection(self, request, req_addr, is_ssl):
193 """ do the proxy connection """ 194 proxy_settings = os.environ.get('%s_proxy' % 195 request.parsed_url.scheme) 196 197 if proxy_settings and proxy_settings is not None: 198 request.is_proxied = True 199 200 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) 201 addr = parse_netloc(urlparse.urlparse(proxy_settings)) 202 203 if is_ssl: 204 if proxy_auth: 205 proxy_auth = 'Proxy-authorization: %s' % proxy_auth 206 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr 207 208 user_agent = request.headers.iget('user_agent') 209 if not user_agent: 210 user_agent = "User-Agent: restkit/%s\r\n" % __version__ 211 212 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, 213 user_agent) 214 215 216 conn = self._session.get(host=addr[0], port=addr[1], 217 pool=self._session, is_ssl=is_ssl, 218 extra_headers=[], **self.ssl_args) 219 220 221 conn.send(proxy_pieces) 222 p = HttpStream(SocketReader(conn.socket()), kind=1, 223 decompress=True) 224 225 if p.status_code != 200: 226 raise ProxyError("Tunnel connection failed: %d %s" % 227 (resp.status_int, body)) 228 229 _ = p.body_string() 230 231 else: 232 headers = [] 233 if proxy_auth: 234 headers = [('Proxy-authorization', proxy_auth)] 235 236 conn = self._session.get(host=addr[0], port=addr[1], 237 pool=self._session, is_ssl=False, 238 extra_headers=[], **self.ssl_args) 239 return conn 240 241 return
242
243 - def make_headers_string(self, request, extra_headers=None):
244 """ create final header string """ 245 headers = request.headers.copy() 246 if extra_headers is not None: 247 for k, v in extra_headers: 248 headers[k] = v 249 250 if not request.body and request.method in ('POST', 'PUT',): 251 headers['Content-Length'] = 0 252 253 if self.version == (1,1): 254 httpver = "HTTP/1.1" 255 else: 256 httpver = "HTTP/1.0" 257 258 ua = headers.iget('user_agent') 259 if not ua: 260 ua = USER_AGENT 261 host = request.host 262 263 accept_encoding = headers.iget('accept-encoding') 264 if not accept_encoding: 265 accept_encoding = 'identity' 266 267 if request.is_proxied: 268 full_path = ("https://" if request.is_ssl() else "http://") + request.host + request.path 269 else: 270 full_path = request.path 271 272 lheaders = [ 273 "%s %s %s\r\n" % (request.method, full_path, httpver), 274 "Host: %s\r\n" % host, 275 "User-Agent: %s\r\n" % ua, 276 "Accept-Encoding: %s\r\n" % accept_encoding 277 ] 278 279 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \ 280 headers.items() if k.lower() not in \ 281 ('user-agent', 'host', 'accept-encoding',)]) 282 if log.isEnabledFor(logging.DEBUG): 283 log.debug("Send headers: %s" % lheaders) 284 return "%s\r\n" % "".join(lheaders)
285
286 - def perform(self, request):
287 """ perform the request. If an error happen it will first try to 288 restart it """ 289 290 if log.isEnabledFor(logging.DEBUG): 291 log.debug("Start to perform request: %s %s %s" % 292 (request.host, request.method, request.path)) 293 conn = None 294 295 try: 296 # get or create a connection to the remote host 297 conn = self.get_connection(request) 298 299 # send headers 300 msg = self.make_headers_string(request, 301 conn.extra_headers) 302 303 # send body 304 if request.body is not None: 305 chunked = request.is_chunked() 306 if request.headers.iget('content-length') is None and \ 307 not chunked: 308 raise RequestError( 309 "Can't determine content length and " + 310 "Transfer-Encoding header is not chunked") 311 312 313 # handle 100-Continue status 314 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 315 hdr_expect = request.headers.iget("expect") 316 if hdr_expect is not None and \ 317 hdr_expect.lower() == "100-continue": 318 conn.send(msg) 319 msg = None 320 p = HttpStream(SocketReader(conn.socket()), kind=1, 321 decompress=True) 322 323 324 if p.status_code != 100: 325 self.reset_request() 326 if log.isEnabledFor(logging.DEBUG): 327 log.debug("return response class") 328 return self.response_class(conn, request, p) 329 330 chunked = request.is_chunked() 331 if log.isEnabledFor(logging.DEBUG): 332 log.debug("send body (chunked: %s)" % chunked) 333 334 335 if isinstance(request.body, types.StringTypes): 336 if msg is not None: 337 conn.send(msg + request.body, chunked) 338 else: 339 conn.send(request.body, chunked) 340 else: 341 if msg is not None: 342 conn.send(msg) 343 344 if hasattr(request.body, 'read'): 345 if hasattr(request.body, 'seek'): 346 request.body.seek(0) 347 conn.sendfile(request.body, chunked) 348 else: 349 conn.sendlines(request.body, chunked) 350 if chunked: 351 conn.send_chunk("") 352 else: 353 conn.send(msg) 354 355 return self.get_response(request, conn) 356 except socket.gaierror, e: 357 if conn is not None: 358 conn.close() 359 raise RequestError(str(e)) 360 except socket.timeout, e: 361 if conn is not None: 362 conn.close() 363 raise RequestTimeout(str(e)) 364 except socket.error, e: 365 if log.isEnabledFor(logging.DEBUG): 366 log.debug("socket error: %s" % str(e)) 367 if conn is not None: 368 conn.close() 369 raise RequestError("socket.error: %s" % str(e)) 370 except (StopIteration, NoMoreData): 371 if conn is not None: 372 conn.close() 373 if request.body is not None: 374 if not hasattr(request.body, 'read') and \ 375 not isinstance(request.body, types.StringTypes): 376 raise RequestError("connection closed and can't" 377 + "be resent") 378 else: 379 raise 380 except Exception: 381 # unkown error 382 log.debug("unhandled exception %s" % 383 traceback.format_exc()) 384 raise
385
386 - def request(self, url, method='GET', body=None, headers=None):
387 """ perform immediatly a new request """ 388 389 request = Request(url, method=method, body=body, 390 headers=headers) 391 392 # apply request filters 393 # They are applied only once time. 394 for f in self.request_filters: 395 ret = f.on_request(request) 396 if isinstance(ret, Response): 397 # a response instance has been provided. 398 # just return it. Useful for cache filters 399 return ret 400 401 # no response has been provided, do the request 402 self._nb_redirections = self.max_follow_redirect 403 return self.perform(request)
404
405 - def redirect(self, location, request):
406 """ reset request, set new url of request and perform it """ 407 if self._nb_redirections <= 0: 408 raise RedirectLimit("Redirection limit is reached") 409 410 if request.initial_url is None: 411 request.initial_url = self.url 412 413 # make sure location follow rfc2616 414 location = rewrite_location(request.url, location) 415 416 if log.isEnabledFor(logging.DEBUG): 417 log.debug("Redirect to %s" % location) 418 419 # change request url and method if needed 420 request.url = location 421 422 self._nb_redirections -= 1 423 424 #perform a new request 425 return self.perform(request)
426
427 - def get_response(self, request, connection):
428 """ return final respons, it is only accessible via peform 429 method """ 430 if log.isEnabledFor(logging.DEBUG): 431 log.debug("Start to parse response") 432 433 p = HttpStream(SocketReader(connection.socket()), kind=1, 434 decompress=self.decompress) 435 436 if log.isEnabledFor(logging.DEBUG): 437 log.debug("Got response: %s" % p.status()) 438 log.debug("headers: [%s]" % p.headers()) 439 440 location = p.headers().get('location') 441 442 if self.follow_redirect: 443 if p.status_code() in (301, 302, 307,): 444 connection.close() 445 if request.method in ('GET', 'HEAD',) or \ 446 self.force_follow_redirect: 447 if hasattr(self.body, 'read'): 448 try: 449 self.body.seek(0) 450 except AttributeError: 451 raise RequestError("Can't redirect %s to %s " 452 "because body has already been read" 453 % (self.url, location)) 454 return self.redirect(location, request) 455 456 elif p.status_code() == 303 and self.method == "POST": 457 connection.close() 458 request.method = "GET" 459 request.body = None 460 return self.redirect(location, request) 461 462 # create response object 463 resp = self.response_class(connection, request, p) 464 465 # apply response filters 466 for f in self.response_filters: 467 f.on_response(resp, request) 468 469 if log.isEnabledFor(logging.DEBUG): 470 log.debug("return response class") 471 472 # return final response 473 return resp
474 475
476 -def _get_proxy_auth(proxy_settings):
477 proxy_username = os.environ.get('proxy-username') 478 if not proxy_username: 479 proxy_username = os.environ.get('proxy_username') 480 proxy_password = os.environ.get('proxy-password') 481 if not proxy_password: 482 proxy_password = os.environ.get('proxy_password') 483 484 proxy_password = proxy_password or "" 485 486 if not proxy_username: 487 u = urlparse.urlparse(proxy_settings) 488 if u.username: 489 proxy_password = u.password or proxy_password 490 proxy_settings = urlparse.urlunparse((u.scheme, 491 u.netloc.split("@")[-1], u.path, u.params, u.query, 492 u.fragment)) 493 494 if proxy_username: 495 user_auth = base64.encodestring('%s:%s' % (proxy_username, 496 proxy_password)) 497 return proxy_settings, 'Basic %s\r\n' % (user_auth.strip()) 498 else: 499 return proxy_settings, ''
500