"""An extensible library for opening URLs using a variety of protocols
The simplest way to use this module is to call the urlopen function,
which accepts a string containing a URL or a Request object (described
below). It opens the URL and returns the results as file-like
object; the returned object has some extra methods described below.
The OpenerDirector manages a collection of Handler objects that do
all the actual work. Each Handler implements a particular protocol or
option. The OpenerDirector is a composite object that invokes the
Handlers needed to open the requested URL. For example, the
HTTPHandler performs HTTP GET and POST requests and deals with
non-error returns. The HTTPRedirectHandler automatically deals with
HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
deals with digest authentication.
urlopen(url, data=None) -- basic usage is that same as original
urllib. pass the url and optionally data to post to an HTTP URL, and
get a file-like object back. One difference is that you can also pass
a Request instance instead of URL. Raises a URLError (subclass of
IOError); for HTTP errors, raises an HTTPError, which can also be
treated as a valid response.
build_opener -- function that creates a new OpenerDirector instance.
will install the default handlers. accepts one or more Handlers as
arguments, either instances or Handler classes that it will
instantiate. if one of the argument is a subclass of the default
handler, the argument will be installed instead of the default.
install_opener -- installs a new opener as the default opener.
Request -- an object that encapsulates the state of a request. the
state can be a simple as the URL. it can also include extra HTTP
headers, e.g. a User-Agent.
URLError-- a subclass of IOError, individual protocols have their own
HTTPError-- also a valid HTTP response, so you can treat an HTTP error
as an exceptional event or valid response
# set up authentication info
authinfo = urllib2.HTTPBasicAuthHandler()
authinfo.add_password('realm', 'host', 'username', 'password')
proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
# build a new opener that adds authentication and caching FTP handlers
opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
urllib2.install_opener(opener)
f = urllib2.urlopen('http://www.python.org/')
# If an authentication error handler that tries to perform
# authentication for some reason but fails, how should the error be
# signalled? The client needs to know the HTTP error code. But if
# the handler knows that the problem was, e.g., that it didn't know
# that hash algo that requested in the challenge, it would be good to
# pass that information along to the client, too.
# documentation (getting there)
# abstract factory for opener
# ftp errors aren't handled cleanly
# gopher can return a socket.error
# check digest against correct (i.e. non-apache) implementation
from cStringIO
import StringIO
from StringIO
import StringIO
# not sure how many of these need to be gotten rid of
from urllib
import (unwrap
, unquote
, splittype
, splithost
,
addinfourl
, splitport
, splitgophertype
, splitquery
,
splitattr
, ftpwrapper
, noheaders
, splituser
, splitpasswd
, splitvalue
)
# support for FileHandler, proxies via environment variables
from urllib
import localhost
, url2pathname
, getproxies
def urlopen(url
, data
=None):
return _opener
.open(url
, data
)
def install_opener(opener
):
# do these error classes make sense?
# make sure all of the IOError stuff is overridden. we just want to be
# URLError is a sub-type of IOError, but it doesn't share any of
# the implementation. need to override __init__ and __str__.
# It sets self.args for compatibility with other EnvironmentError
# subclasses, but args doesn't have the typical format with errno in
# slot 0 and strerror in slot 1. This may be better than nothing.
def __init__(self
, reason
):
return '<urlopen error %s>' % self
.reason
class HTTPError(URLError
, addinfourl
):
"""Raised when HTTP error occurs, but also acts like non-error return"""
__super_init
= addinfourl
.__init
__
def __init__(self
, url
, code
, msg
, hdrs
, fp
):
# The addinfourl classes depend on fp being a valid file
# object. In some cases, the HTTPError may not have a valid
# file object. If this happens, the simplest workaround is to
# not initialize the base classes.
self
.__super
_init
(fp
, hdrs
, url
)
return 'HTTP Error %s: %s' % (self
.code
, self
.msg
)
class GopherError(URLError
):
def __init__(self
, url
, data
=None, headers
={},
origin_req_host
=None, unverifiable
=False):
# unwrap('<URL:type://host/path>') --> 'type://host/path'
self
.__original
= unwrap(url
)
# self.__r_type is what's left after doing the splittype
for key
, value
in headers
.items():
self
.add_header(key
, value
)
self
.unredirected_hdrs
= {}
if origin_req_host
is None:
origin_req_host
= cookielib
.request_host(self
)
self
.origin_req_host
= origin_req_host
self
.unverifiable
= unverifiable
def __getattr__(self
, attr
):
# XXX this is a fallback mechanism to guard against these
# methods getting called in a non-standard order. this may be
# too complicated and/or unnecessary.
# XXX should the __r_XXX attributes be public?
if attr
[:12] == '_Request__r_':
if hasattr(Request
, 'get_' + name
):
getattr(self
, 'get_' + name
)()
return getattr(self
, attr
)
raise AttributeError, attr
# XXX these helper methods are lame
def add_data(self
, data
):
return self
.data
is not None
self
.type, self
.__r
_type
= splittype(self
.__original
)
raise ValueError, "unknown url type: %s" % self
.__original
self
.host
, self
.__r
_host
= splithost(self
.__r
_type
)
self
.host
= unquote(self
.host
)
def set_proxy(self
, host
, type):
self
.host
, self
.type = host
, type
self
.__r
_host
= self
.__original
def get_origin_req_host(self
):
return self
.origin_req_host
def is_unverifiable(self
):
def add_header(self
, key
, val
):
# useful for something like authentication
self
.headers
[key
.capitalize()] = val
def add_unredirected_header(self
, key
, val
):
# will not be added to a redirected request
self
.unredirected_hdrs
[key
.capitalize()] = val
def has_header(self
, header_name
):
return (header_name
in self
.headers
or
header_name
in self
.unredirected_hdrs
)
def get_header(self
, header_name
, default
=None):
self
.unredirected_hdrs
.get(header_name
, default
))
hdrs
= self
.unredirected_hdrs
.copy()
hdrs
.update(self
.headers
)
server_version
= "Python-urllib/%s" % __version__
self
.addheaders
= [('User-agent', server_version
)]
# manage the individual handlers
self
.process_response
= {}
self
.process_request
= {}
def add_handler(self
, handler
):
for meth
in dir(handler
):
if condition
.startswith("error"):
j
= condition
.find("_") + i
+ 1
lookup
= self
.handle_error
.get(protocol
, {})
self
.handle_error
[protocol
] = lookup
elif condition
== "open":
lookup
= getattr(self
, "handle_"+condition
)
elif condition
in ["response", "request"]:
lookup
= getattr(self
, "process_"+condition
)
handlers
= lookup
.setdefault(kind
, [])
bisect
.insort(handlers
, handler
)
# XXX why does self.handlers need to be sorted?
bisect
.insort(self
.handlers
, handler
)
# Only exists for backwards compatibility.
def _call_chain(self
, chain
, kind
, meth_name
, *args
):
# XXX raise an exception if no one else should try to handle
# this url. return None if you can't but someone else could.
handlers
= chain
.get(kind
, ())
func
= getattr(handler
, meth_name
)
def open(self
, fullurl
, data
=None):
# accept a URL or a Request object
if isinstance(fullurl
, basestring
):
req
= Request(fullurl
, data
)
protocol
= req
.get_type()
meth_name
= protocol
+"_request"
for processor
in self
.process_request
.get(protocol
, []):
meth
= getattr(processor
, meth_name
)
response
= self
._open
(req
, data
)
meth_name
= protocol
+"_response"
for processor
in self
.process_response
.get(protocol
, []):
meth
= getattr(processor
, meth_name
)
response
= meth(req
, response
)
def _open(self
, req
, data
=None):
result
= self
._call
_chain
(self
.handle_open
, 'default',
protocol
= req
.get_type()
result
= self
._call
_chain
(self
.handle_open
, protocol
, protocol
+
return self
._call
_chain
(self
.handle_open
, 'unknown',
def error(self
, proto
, *args
):
if proto
in ['http', 'https']:
# XXX http[s] protocols are special-cased
dict = self
.handle_error
['http'] # https is not different than http
meth_name
= 'http_error_%s' % proto
meth_name
= proto
+ '_error'
args
= (dict, proto
, meth_name
) + args
result
= self
._call
_chain
(*args
)
args
= (dict, 'default', 'http_error_default') + orig_args
return self
._call
_chain
(*args
)
# XXX probably also want an abstract factory that knows when it makes
# sense to skip a superclass in favor of a subclass and when it might
# make sense to include both
def build_opener(*handlers
):
"""Create an opener object from a list of handlers.
The opener will use several default handlers, including support
If any of the handlers passed as arguments are subclasses of the
default handlers, the default handlers will not be used.
opener
= OpenerDirector()
default_classes
= [ProxyHandler
, UnknownHandler
, HTTPHandler
,
HTTPDefaultErrorHandler
, HTTPRedirectHandler
,
FTPHandler
, FileHandler
, HTTPErrorProcessor
]
if hasattr(httplib
, 'HTTPS'):
default_classes
.append(HTTPSHandler
)
for klass
in default_classes
:
if inspect
.isclass(check
):
if issubclass(check
, klass
):
elif isinstance(check
, klass
):
default_classes
.remove(klass
)
for klass
in default_classes
:
opener
.add_handler(klass())
def add_parent(self
, parent
):
# Only exists for backwards compatibility
if not hasattr(other
, "handler_order"):
# Try to preserve the old behavior of having custom classes
# inserted after default ones (works only for custom user
# classes which are not aware of handler_order).
return self
.handler_order
< other
.handler_order
class HTTPErrorProcessor(BaseHandler
):
"""Process HTTP error responses."""
handler_order
= 1000 # after all other processing
def http_response(self
, request
, response
):
code
, msg
, hdrs
= response
.code
, response
.msg
, response
.info()
if code
not in (200, 206):
response
= self
.parent
.error(
'http', request
, response
, code
, msg
, hdrs
)
https_response
= http_response
class HTTPDefaultErrorHandler(BaseHandler
):
def http_error_default(self
, req
, fp
, code
, msg
, hdrs
):
raise HTTPError(req
.get_full_url(), code
, msg
, hdrs
, fp
)
class HTTPRedirectHandler(BaseHandler
):
# maximum number of redirections to any single URL
# this is needed because of the state that cookies introduce
# maximum total number of redirections (regardless of URL) before
# assuming we're in a loop
def redirect_request(self
, req
, fp
, code
, msg
, headers
, newurl
):
"""Return a Request or None in response to a redirect.
This is called by the http_error_30x methods when a
redirection response is received. If a redirection should
take place, return a new Request to allow http_error_30x to
perform the redirect. Otherwise, raise HTTPError if no-one
else should try to handle this url. Return None if you can't
but another Handler might.
if (code
in (301, 302, 303, 307) and m
in ("GET", "HEAD")
or code
in (301, 302, 303) and m
== "POST"):
# Strictly (according to RFC 2616), 301 or 302 in response
# to a POST MUST NOT cause a redirection without confirmation
# from the user (of urllib2, in this case). In practice,
# essentially all clients do redirect in this case, so we
origin_req_host
=req
.get_origin_req_host(),
raise HTTPError(req
.get_full_url(), code
, msg
, headers
, fp
)
# Implementation note: To avoid the server sending us into an
# infinite loop, the request object needs to track what URLs we
# have already seen. Do this by adding a handler-specific
# attribute to the Request object.
def http_error_302(self
, req
, fp
, code
, msg
, headers
):
# Some servers (incorrectly) return multiple Location headers
# (so probably same goes for URI). Use first header.
if 'location' in headers
:
newurl
= headers
.getheaders('location')[0]
newurl
= headers
.getheaders('uri')[0]
newurl
= urlparse
.urljoin(req
.get_full_url(), newurl
)
# XXX Probably want to forget about the state of the current
# request, although that might interact poorly with other
# handlers that also use handler-specific request attributes
new
= self
.redirect_request(req
, fp
, code
, msg
, headers
, newurl
)
# .redirect_dict has a key url if url was previously visited.
if hasattr(req
, 'redirect_dict'):
visited
= new
.redirect_dict
= req
.redirect_dict
if (visited
.get(newurl
, 0) >= self
.max_repeats
or
len(visited
) >= self
.max_redirections
):
raise HTTPError(req
.get_full_url(), code
,
self
.inf_msg
+ msg
, headers
, fp
)
visited
= new
.redirect_dict
= req
.redirect_dict
= {}
visited
[newurl
] = visited
.get(newurl
, 0) + 1
# Don't close the fp until we are sure that we won't use it
return self
.parent
.open(new
)
http_error_301
= http_error_303
= http_error_307
= http_error_302
inf_msg
= "The HTTP server returned a redirect error that would " \
"lead to an infinite loop.\n" \
"The last 30x error message was:\n"
class ProxyHandler(BaseHandler
):
# Proxies must be in front
def __init__(self
, proxies
=None):
assert hasattr(proxies
, 'has_key'), "proxies must be a mapping"
for type, url
in proxies
.items():
setattr(self
, '%s_open' % type,
lambda r
, proxy
=url
, type=type, meth
=self
.proxy_open
: \
def proxy_open(self
, req
, proxy
, type):
orig_type
= req
.get_type()
type, r_type
= splittype(proxy
)
host
, XXX
= splithost(r_type
)
user_pass
, host
= host
.split('@', 1)
user
, password
= user_pass
.split(':', 1)
user_pass
= base64
.encodestring('%s:%s' % (unquote(user
),
unquote(password
))).strip()
req
.add_header('Proxy-authorization', 'Basic ' + user_pass
)
req
.set_proxy(host
, type)
# let other handlers take care of it
# XXX this only makes sense if the proxy is before the
# need to start over, because the other handlers don't
# grok the proxy's URL type
return self
.parent
.open(req
)
# feature suggested by Duncan Booth
# XXX custom is not a good name
# either pass a function to the constructor or override handle
def __init__(self
, proto
, func
=None, proxy_addr
=None):
if self
.func
and self
.func(req
):
class CustomProxyHandler(BaseHandler
):
# Proxies must be in front
def __init__(self
, *proxies
):
def proxy_open(self
, req
):
proxies
= self
.proxies
[proto
]
req
.set_proxy(p
.get_proxy())
return self
.parent
.open(req
)
def do_proxy(self
, p
, req
):
return self
.parent
.open(req
)
def add_proxy(self
, cpo
):
if cpo
.proto
in self
.proxies
:
self
.proxies
[cpo
.proto
].append(cpo
)
self
.proxies
[cpo
.proto
] = [cpo
]
def add_password(self
, realm
, uri
, user
, passwd
):
# uri could be a single URI or a sequence
if isinstance(uri
, basestring
):
uri
= tuple(map(self
.reduce_uri
, uri
))
if not realm
in self
.passwd
:
self
.passwd
[realm
][uri
] = (user
, passwd
)
def find_user_password(self
, realm
, authuri
):
domains
= self
.passwd
.get(realm
, {})
authuri
= self
.reduce_uri(authuri
)
for uris
, authinfo
in domains
.iteritems():
if self
.is_suburi(uri
, authuri
):
def reduce_uri(self
, uri
):
"""Accept netloc or URI and extract only the netloc and path"""
parts
= urlparse
.urlparse(uri
)
return parts
[1], parts
[2] or '/'
def is_suburi(self
, base
, test
):
"""Check if test is below base in a URI tree
Both args must be URIs in reduced form.
common
= posixpath
.commonprefix((base
[1], test
[1]))
if len(common
) == len(base
[1]):
class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr
):
def find_user_password(self
, realm
, authuri
):
user
, password
= HTTPPasswordMgr
.find_user_password(self
, realm
,
return HTTPPasswordMgr
.find_user_password(self
, None, authuri
)
class AbstractBasicAuthHandler
:
rx
= re
.compile('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', re
.I
)
# XXX there can actually be multiple auth-schemes in a
# www-authenticate header. should probably be a lot more careful
# in parsing them to extract multiple alternatives
def __init__(self
, password_mgr
=None):
password_mgr
= HTTPPasswordMgr()
self
.passwd
= password_mgr
self
.add_password
= self
.passwd
.add_password
def http_error_auth_reqed(self
, authreq
, host
, req
, headers
):
# XXX could be multiple headers
authreq
= headers
.get(authreq
, None)
mo
= AbstractBasicAuthHandler
.rx
.search(authreq
)
scheme
, realm
= mo
.groups()
if scheme
.lower() == 'basic':
return self
.retry_http_basic_auth(host
, req
, realm
)
def retry_http_basic_auth(self
, host
, req
, realm
):
user
,pw
= self
.passwd
.find_user_password(realm
, host
)
raw
= "%s:%s" % (user
, pw
)
auth
= 'Basic %s' % base64
.encodestring(raw
).strip()
if req
.headers
.get(self
.auth_header
, None) == auth
:
req
.add_header(self
.auth_header
, auth
)
return self
.parent
.open(req
)
class HTTPBasicAuthHandler(AbstractBasicAuthHandler
, BaseHandler
):
auth_header
= 'Authorization'
def http_error_401(self
, req
, fp
, code
, msg
, headers
):
host
= urlparse
.urlparse(req
.get_full_url())[1]
return self
.http_error_auth_reqed('www-authenticate',
class ProxyBasicAuthHandler(AbstractBasicAuthHandler
, BaseHandler
):
auth_header
= 'Proxy-authorization'
def http_error_407(self
, req
, fp
, code
, msg
, headers
):
return self
.http_error_auth_reqed('proxy-authenticate',
"""Return n random bytes."""
# Use /dev/urandom if it is available. Fall back to random module
# if not. It might be worthwhile to extend this function to use
# other platform-specific mechanisms for getting random bytes.
if os
.path
.exists("/dev/urandom"):
L
= [chr(random
.randrange(0, 256)) for i
in range(n
)]
class AbstractDigestAuthHandler
:
# Digest authentication is specified in RFC 2617.
# XXX The client does not inspect the Authentication-Info header
# in a successful response.
# XXX It should be possible to test this implementation against
# a mock server that just generates a static set of challenges.
# XXX qop="auth-int" supports is shaky
def __init__(self
, passwd
=None):
passwd
= HTTPPasswordMgr()
self
.add_password
= self
.passwd
.add_password
def reset_retry_count(self
):
def http_error_auth_reqed(self
, auth_header
, host
, req
, headers
):
authreq
= headers
.get(auth_header
, None)
# Don't fail endlessly - if we failed once, we'll probably
# fail a second time. Hm. Unless the Password Manager is
# prompting for the information. Crap. This isn't great
# but it's better than the current 'repeat until recursion
# depth exceeded' approach <wink>
raise HTTPError(req
.get_full_url(), 401, "digest auth failed",
scheme
= authreq
.split()[0]
if scheme
.lower() == 'digest':
return self
.retry_http_digest_auth(req
, authreq
)
raise ValueError("AbstractDigestAuthHandler doesn't know "
def retry_http_digest_auth(self
, req
, auth
):
token
, challenge
= auth
.split(' ', 1)
chal
= parse_keqv_list(parse_http_list(challenge
))
auth
= self
.get_authorization(req
, chal
)
auth_val
= 'Digest %s' % auth
if req
.headers
.get(self
.auth_header
, None) == auth_val
:
req
.add_header(self
.auth_header
, auth_val
)
resp
= self
.parent
.open(req
)
def get_cnonce(self
, nonce
):
# The cnonce-value is an opaque
# quoted string value provided by the client and used by both client
# and server to avoid chosen plaintext attacks, to provide mutual
# authentication, and to provide some message integrity protection.
# This isn't a fabulous effort, but it's probably Good Enough.
dig
= sha
.new("%s:%s:%s:%s" % (self
.nonce_count
, nonce
, time
.ctime(),
randombytes(8))).hexdigest()
def get_authorization(self
, req
, chal
):
algorithm
= chal
.get('algorithm', 'MD5')
# mod_digest doesn't send an opaque, even though it isn't
# supposed to be optional
opaque
= chal
.get('opaque', None)
H
, KD
= self
.get_algorithm_impls(algorithm
)
user
, pw
= self
.passwd
.find_user_password(realm
, req
.get_full_url())
# XXX not implemented yet
entdig
= self
.get_entity_digest(req
.get_data(), chal
)
A1
= "%s:%s:%s" % (user
, realm
, pw
)
A2
= "%s:%s" % (req
.get_method(),
# XXX selector: what about proxies and full urls
ncvalue
= '%08x' % self
.nonce_count
cnonce
= self
.get_cnonce(nonce
)
noncebit
= "%s:%s:%s:%s:%s" % (nonce
, ncvalue
, cnonce
, qop
, H(A2
))
respdig
= KD(H(A1
), noncebit
)
respdig
= KD(H(A1
), "%s:%s" % (nonce
, H(A2
)))
# XXX should the partial digests be encoded too?
base
= 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
'response="%s"' % (user
, realm
, nonce
, req
.get_selector(),
base
= base
+ ', opaque="%s"' % opaque
base
= base
+ ', digest="%s"' % entdig
base
= base
+ ', algorithm="%s"' % algorithm
base
= base
+ ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue
, cnonce
)
def get_algorithm_impls(self
, algorithm
):
# lambdas assume digest modules are imported at the top level
H
= lambda x
: md5
.new(x
).hexdigest()
H
= lambda x
: sha
.new(x
).hexdigest()
KD
= lambda s
, d
: H("%s:%s" % (s
, d
))
def get_entity_digest(self
, data
, chal
):
# XXX not implemented yet
class HTTPDigestAuthHandler(BaseHandler
, AbstractDigestAuthHandler
):
"""An authentication protocol defined by RFC 2069
Digest authentication improves on basic authentication because it
does not transmit passwords in the clear.
auth_header
= 'Authorization'
def http_error_401(self
, req
, fp
, code
, msg
, headers
):
host
= urlparse
.urlparse(req
.get_full_url())[1]
retry
= self
.http_error_auth_reqed('www-authenticate',
class ProxyDigestAuthHandler(BaseHandler
, AbstractDigestAuthHandler
):
auth_header
= 'Proxy-Authorization'
def http_error_407(self
, req
, fp
, code
, msg
, headers
):
retry
= self
.http_error_auth_reqed('proxy-authenticate',
class AbstractHTTPHandler(BaseHandler
):
def __init__(self
, debuglevel
=0):
self
._debuglevel
= debuglevel
def set_http_debuglevel(self
, level
):
def do_request_(self
, request
):
host
= request
.get_host()
raise URLError('no host given')
if request
.has_data(): # POST
data
= request
.get_data()
if not request
.has_header('Content-type'):
request
.add_unredirected_header(
'application/x-www-form-urlencoded')
if not request
.has_header('Content-length'):
request
.add_unredirected_header(
'Content-length', '%d' % len(data
))
scheme
, sel
= splittype(request
.get_selector())
sel_host
, sel_path
= splithost(sel
)
if not request
.has_header('Host'):
request
.add_unredirected_header('Host', sel_host
or host
)
for name
, value
in self
.parent
.addheaders
:
if not request
.has_header(name
):
request
.add_unredirected_header(name
, value
)
def do_open(self
, http_class
, req
):
"""Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib.
The addinfourl return value is a file-like object. It also
has methods and attributes including:
- info(): return a mimetools.Message object for the headers
- geturl(): return the original request URL
raise URLError('no host given')
h
= http_class(host
) # will parse host:port
h
.set_debuglevel(self
._debuglevel
)
headers
= dict(req
.headers
)
headers
.update(req
.unredirected_hdrs
)
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
headers
["Connection"] = "close"
h
.request(req
.get_method(), req
.get_selector(), req
.data
, headers
)
except socket
.error
, err
: # XXX what error?
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
# XXX It might be better to extract the read buffering code
# out of socket._fileobject() and into a base class.
fp
= socket
._fileobject
(r
)
resp
= addinfourl(fp
, r
.msg
, req
.get_full_url())
class HTTPHandler(AbstractHTTPHandler
):
def http_open(self
, req
):
return self
.do_open(httplib
.HTTPConnection
, req
)
http_request
= AbstractHTTPHandler
.do_request_
if hasattr(httplib
, 'HTTPS'):
class HTTPSHandler(AbstractHTTPHandler
):
def https_open(self
, req
):
return self
.do_open(httplib
.HTTPSConnection
, req
)
https_request
= AbstractHTTPHandler
.do_request_
class HTTPCookieProcessor(BaseHandler
):
def __init__(self
, cookiejar
=None):
cookiejar
= cookielib
.CookieJar()
self
.cookiejar
= cookiejar
def http_request(self
, request
):
self
.cookiejar
.add_cookie_header(request
)
def http_response(self
, request
, response
):
self
.cookiejar
.extract_cookies(response
, request
)
https_request
= http_request
https_response
= http_response
class UnknownHandler(BaseHandler
):
def unknown_open(self
, req
):
raise URLError('unknown url type: %s' % type)
"""Parse list of key=value strings where keys are not duplicated."""
if v
[0] == '"' and v
[-1] == '"':
"""Parse lists as described by RFC 2068 Section 2.
In particular, parse comma-separated lists where the elements of
the list may include quoted-strings. A quoted-string could
contain a comma. A non-quoted string could have quotes in the
middle. Neither commas nor quotes count if they are escaped.
Only double-quotes count, not single-quotes.
return [part
.strip() for part
in res
]
class FileHandler(BaseHandler
):
# Use local file or FTP depending on form of URL
def file_open(self
, req
):
if url
[:2] == '//' and url
[2:3] != '/':
return self
.parent
.open(req
)
return self
.open_local_file(req
)
# names for the localhost
if FileHandler
.names
is None:
FileHandler
.names
= (socket
.gethostbyname('localhost'),
socket
.gethostbyname(socket
.gethostname()))
# not entirely sure what the rules are here
def open_local_file(self
, req
):
file = req
.get_selector()
localfile
= url2pathname(file)
stats
= os
.stat(localfile
)
modified
= email
.Utils
.formatdate(stats
.st_mtime
, usegmt
=True)
mtype
= mimetypes
.guess_type(file)[0]
headers
= mimetools
.Message(StringIO(
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype
or 'text/plain', size
, modified
)))
host
, port
= splitport(host
)
(not port
and socket
.gethostbyname(host
) in self
.get_names()):
return addinfourl(open(localfile
, 'rb'),
raise URLError('file not on local host')
class FTPHandler(BaseHandler
):
raise IOError, ('ftp error', 'no host given')
host
, port
= splitport(host
)
# username/password handling
user
, host
= splituser(host
)
user
, passwd
= splitpasswd(user
)
user
= unquote(user
or '')
passwd
= unquote(passwd
or '')
host
= socket
.gethostbyname(host
)
except socket
.error
, msg
:
path
, attrs
= splitattr(req
.get_selector())
dirs
= map(unquote
, dirs
)
dirs
, file = dirs
[:-1], dirs
[-1]
fw
= self
.connect_ftp(user
, passwd
, host
, port
, dirs
)
type = file and 'I' or 'D'
attr
, value
= splitvalue(attr
)
if attr
.lower() == 'type' and \
value
in ('a', 'A', 'i', 'I', 'd', 'D'):
fp
, retrlen
= fw
.retrfile(file, type)
mtype
= mimetypes
.guess_type(req
.get_full_url())[0]
headers
+= "Content-type: %s\n" % mtype
if retrlen
is not None and retrlen
>= 0:
headers
+= "Content-length: %d\n" % retrlen
headers
= mimetools
.Message(sf
)
return addinfourl(fp
, headers
, req
.get_full_url())
except ftplib
.all_errors
, msg
:
raise IOError, ('ftp error', msg
), sys
.exc_info()[2]
def connect_ftp(self
, user
, passwd
, host
, port
, dirs
):
fw
= ftpwrapper(user
, passwd
, host
, port
, dirs
)
## fw.ftp.set_debuglevel(1)
class CacheFTPHandler(FTPHandler
):
# XXX would be nice to have pluggable cache strategies
# XXX this stuff is definitely not thread safe
def setMaxConns(self
, m
):
def connect_ftp(self
, user
, passwd
, host
, port
, dirs
):
key
= user
, host
, port
, '/'.join(dirs
)
self
.timeout
[key
] = time
.time() + self
.delay
self
.cache
[key
] = ftpwrapper(user
, passwd
, host
, port
, dirs
)
self
.timeout
[key
] = time
.time() + self
.delay
# first check for old ones
for k
, v
in self
.timeout
.items():
self
.soonest
= min(self
.timeout
.values())
if len(self
.cache
) == self
.max_conns
:
for k
, v
in self
.timeout
.items():
self
.soonest
= min(self
.timeout
.values())
class GopherHandler(BaseHandler
):
def gopher_open(self
, req
):
raise GopherError('no host given')
selector
= req
.get_selector()
type, selector
= splitgophertype(selector
)
selector
, query
= splitquery(selector
)
selector
= unquote(selector
)
fp
= gopherlib
.send_query(selector
, query
, host
)
fp
= gopherlib
.send_selector(selector
, host
)
return addinfourl(fp
, noheaders(), req
.get_full_url())
#bleck! don't use this yet
default_handlers
= [UnknownHandler
, HTTPHandler
,
HTTPDefaultErrorHandler
, HTTPRedirectHandler
,
replacement_handlers
= []
def add_handler(self
, h
):
self
.handlers
= self
.handlers
+ [h
]
def replace_handler(self
, h
):
opener
= OpenerDirector()
for ph
in self
.default_handlers
: