1
2
3
4
5 import base64
6 import errno
7 import logging
8 import os
9 import time
10 import socket
11 import ssl
12 import traceback
13 import types
14 import urlparse
15
16 try:
17 from http_parser.http import HttpStream
18 from http_parser.reader import SocketReader
19 except ImportError:
20 raise ImportError("""http-parser isn't installed.
21
22 pip install http-parser""")
23
24 from restkit import __version__
25
26 from restkit.conn import Connection
27 from restkit.errors import RequestError, RequestTimeout, RedirectLimit, \
28 NoMoreData, ProxyError
29 from restkit.session import get_session
30 from restkit.util import parse_netloc, rewrite_location
31 from restkit.wrappers import Request, Response
32
33 MAX_CLIENT_TIMEOUT=300
34 MAX_CLIENT_CONNECTIONS = 5
35 MAX_CLIENT_TRIES =3
36 CLIENT_WAIT_TRIES = 0.3
37 MAX_FOLLOW_REDIRECTS = 5
38 USER_AGENT = "restkit/%s" % __version__
39
40 log = logging.getLogger(__name__)
41
43
44 """ A client handle a connection at a time. A client is threadsafe,
45 but an handled shouldn't be shared between threads. All connections
46 are shared between threads via a pool.
47
48 >>> from restkit import *
49 >>> c = Client()
50 >>> r = c.request("http://google.com")
51 r>>> r.status
52 '301 Moved Permanently'
53 >>> r.body_string()
54 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n'
55 >>> c.follow_redirect = True
56 >>> r = c.request("http://google.com")
57 >>> r.status
58 '200 OK'
59
60 """
61
62 version = (1, 1)
63 response_class=Response
64
65 - def __init__(self,
66 follow_redirect=False,
67 force_follow_redirect=False,
68 max_follow_redirect=MAX_FOLLOW_REDIRECTS,
69 filters=None,
70 decompress=True,
71 max_status_line_garbage=None,
72 max_header_count=0,
73 session=None,
74 response_class=None,
75 timeout=None,
76 use_proxy=False,
77 max_tries=3,
78 wait_tries=1.0,
79 backend="thread",
80 **ssl_args):
81 """
82 Client parameters
83 ~~~~~~~~~~~~~~~~~
84
85 :param follow_redirect: follow redirection, by default False
86 :param max_ollow_redirect: number of redirections available
87 :filters: http filters to pass
88 :param decompress: allows the client to decompress the response
89 body
90 :param max_status_line_garbage: defines the maximum number of ignorable
91 lines before we expect a HTTP response's status line. With
92 HTTP/1.1 persistent connections, the problem arises that broken
93 scripts could return a wrong Content-Length (there are more
94 bytes sent than specified). Unfortunately, in some cases, this
95 cannot be detected after the bad response, but only before the
96 next one. So the client is abble to skip bad lines using this
97 limit. 0 disable garbage collection, None means unlimited number
98 of tries.
99 :param max_header_count: determines the maximum HTTP header count
100 allowed. by default no limit.
101 :param manager: the manager to use. By default we use the global
102 one.
103 :parama response_class: the response class to use
104 :param timeout: the default timeout of the connection
105 (SO_TIMEOUT)
106
107 :param max_tries: the number of tries before we give up a
108 connection
109 :param wait_tries: number of time we wait between each tries.
110 :param ssl_args: named argument, see ssl module for more
111 informations
112 """
113 self.follow_redirect = follow_redirect
114 self.force_follow_redirect = force_follow_redirect
115 self.max_follow_redirect = max_follow_redirect
116 self.decompress = decompress
117 self.filters = filters or []
118 self.max_status_line_garbage = max_status_line_garbage
119 self.max_header_count = max_header_count
120 self.use_proxy = use_proxy
121
122 self.request_filters = []
123 self.response_filters = []
124 self.load_filters()
125
126
127
128
129 session_options = dict(
130 retry_delay=wait_tries,
131 retry_max = max_tries,
132 timeout = timeout)
133
134
135 if session is None:
136 session = get_session(backend, **session_options)
137 self._session = session
138 self.backend = backend
139
140
141 if response_class is not None:
142 self.response_class = response_class
143
144 self.max_tries = max_tries
145 self.wait_tries = wait_tries
146 self.timeout = timeout
147
148 self._nb_redirections = self.max_follow_redirect
149 self._url = None
150 self._initial_url = None
151 self._write_cb = None
152 self._headers = None
153 self._sock_key = None
154 self._sock = None
155 self._original = None
156
157 self.method = 'GET'
158 self.body = None
159 self.ssl_args = ssl_args or {}
160
162 """ Populate filters from self.filters.
163 Must be called each time self.filters is updated.
164 """
165 for f in self.filters:
166 if hasattr(f, "on_request"):
167 self.request_filters.append(f)
168 if hasattr(f, "on_response"):
169 self.response_filters.append(f)
170
171
172
191
193 """ do the proxy connection """
194 proxy_settings = os.environ.get('%s_proxy' %
195 request.parsed_url.scheme)
196
197 if proxy_settings and proxy_settings is not None:
198 request.is_proxied = True
199
200 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings)
201 addr = parse_netloc(urlparse.urlparse(proxy_settings))
202
203 if is_ssl:
204 if proxy_auth:
205 proxy_auth = 'Proxy-authorization: %s' % proxy_auth
206 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr
207
208 user_agent = request.headers.iget('user_agent')
209 if not user_agent:
210 user_agent = "User-Agent: restkit/%s\r\n" % __version__
211
212 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth,
213 user_agent)
214
215
216 conn = self._session.get(host=addr[0], port=addr[1],
217 pool=self._session, is_ssl=is_ssl,
218 extra_headers=[], **self.ssl_args)
219
220
221 conn.send(proxy_pieces)
222 p = HttpStream(SocketReader(conn.socket()), kind=1,
223 decompress=True)
224
225 if p.status_code != 200:
226 raise ProxyError("Tunnel connection failed: %d %s" %
227 (resp.status_int, body))
228
229 _ = p.body_string()
230
231 else:
232 headers = []
233 if proxy_auth:
234 headers = [('Proxy-authorization', proxy_auth)]
235
236 conn = self._session.get(host=addr[0], port=addr[1],
237 pool=self._session, is_ssl=False,
238 extra_headers=[], **self.ssl_args)
239 return conn
240
241 return
242
244 """ create final header string """
245 headers = request.headers.copy()
246 if extra_headers is not None:
247 for k, v in extra_headers:
248 headers[k] = v
249
250 if not request.body and request.method in ('POST', 'PUT',):
251 headers['Content-Length'] = 0
252
253 if self.version == (1,1):
254 httpver = "HTTP/1.1"
255 else:
256 httpver = "HTTP/1.0"
257
258 ua = headers.iget('user_agent')
259 if not ua:
260 ua = USER_AGENT
261 host = request.host
262
263 accept_encoding = headers.iget('accept-encoding')
264 if not accept_encoding:
265 accept_encoding = 'identity'
266
267 if request.is_proxied:
268 full_path = ("https://" if request.is_ssl() else "http://") + request.host + request.path
269 else:
270 full_path = request.path
271
272 lheaders = [
273 "%s %s %s\r\n" % (request.method, full_path, httpver),
274 "Host: %s\r\n" % host,
275 "User-Agent: %s\r\n" % ua,
276 "Accept-Encoding: %s\r\n" % accept_encoding
277 ]
278
279 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \
280 headers.items() if k.lower() not in \
281 ('user-agent', 'host', 'accept-encoding',)])
282 if log.isEnabledFor(logging.DEBUG):
283 log.debug("Send headers: %s" % lheaders)
284 return "%s\r\n" % "".join(lheaders)
285
385
386 - def request(self, url, method='GET', body=None, headers=None):
404
406 """ reset request, set new url of request and perform it """
407 if self._nb_redirections <= 0:
408 raise RedirectLimit("Redirection limit is reached")
409
410 if request.initial_url is None:
411 request.initial_url = self.url
412
413
414 location = rewrite_location(request.url, location)
415
416 if log.isEnabledFor(logging.DEBUG):
417 log.debug("Redirect to %s" % location)
418
419
420 request.url = location
421
422 self._nb_redirections -= 1
423
424
425 return self.perform(request)
426
428 """ return final respons, it is only accessible via peform
429 method """
430 if log.isEnabledFor(logging.DEBUG):
431 log.debug("Start to parse response")
432
433 p = HttpStream(SocketReader(connection.socket()), kind=1,
434 decompress=self.decompress)
435
436 if log.isEnabledFor(logging.DEBUG):
437 log.debug("Got response: %s" % p.status())
438 log.debug("headers: [%s]" % p.headers())
439
440 location = p.headers().get('location')
441
442 if self.follow_redirect:
443 if p.status_code() in (301, 302, 307,):
444 connection.close()
445 if request.method in ('GET', 'HEAD',) or \
446 self.force_follow_redirect:
447 if hasattr(self.body, 'read'):
448 try:
449 self.body.seek(0)
450 except AttributeError:
451 raise RequestError("Can't redirect %s to %s "
452 "because body has already been read"
453 % (self.url, location))
454 return self.redirect(location, request)
455
456 elif p.status_code() == 303 and self.method == "POST":
457 connection.close()
458 request.method = "GET"
459 request.body = None
460 return self.redirect(location, request)
461
462
463 resp = self.response_class(connection, request, p)
464
465
466 for f in self.response_filters:
467 f.on_response(resp, request)
468
469 if log.isEnabledFor(logging.DEBUG):
470 log.debug("return response class")
471
472
473 return resp
474
475
477 proxy_username = os.environ.get('proxy-username')
478 if not proxy_username:
479 proxy_username = os.environ.get('proxy_username')
480 proxy_password = os.environ.get('proxy-password')
481 if not proxy_password:
482 proxy_password = os.environ.get('proxy_password')
483
484 proxy_password = proxy_password or ""
485
486 if not proxy_username:
487 u = urlparse.urlparse(proxy_settings)
488 if u.username:
489 proxy_password = u.password or proxy_password
490 proxy_settings = urlparse.urlunparse((u.scheme,
491 u.netloc.split("@")[-1], u.path, u.params, u.query,
492 u.fragment))
493
494 if proxy_username:
495 user_auth = base64.encodestring('%s:%s' % (proxy_username,
496 proxy_password))
497 return proxy_settings, 'Basic %s\r\n' % (user_auth.strip())
498 else:
499 return proxy_settings, ''
500