"""Open an arbitrary URL.
See the following document for more info on URLs:
"Names and Addresses, URIs, URLs, URNs, URCs", at
http://www.w3.org/pub/WWW/Addressing/Overview.html
See also the HTTP spec (from which the error codes are derived):
"HTTP - Hypertext Transfer Protocol", at
http://www.w3.org/pub/WWW/Protocols/
Related standards and specs:
- RFC1808: the "relative URL" spec. (authoritative status)
- RFC1738 - the "URL standard". (authoritative status)
- RFC1630 - the "URI spec". (informational status)
The object returned by URLopener().open(file) will differ per
protocol. All you know is that is has methods read(), readline(),
readlines(), fileno(), close() and info(). The read*(), fileno()
and close() methods work like those of open files.
The info() method returns a mimetools.Message object which can be
used to query various info about the object, if available.
(mimetools.Message objects are queried with the getheader() method.)
from urlparse
import urljoin
as basejoin
__all__
= ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
"urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
"urlencode", "url2pathname", "pathname2url", "splittag",
"localhost", "thishost", "ftperrors", "basejoin", "unwrap",
"splittype", "splithost", "splituser", "splitpasswd", "splitport",
"splitnport", "splitquery", "splitattr", "splitvalue",
"splitgophertype", "getproxies"]
__version__
= '1.16' # XXX This version is not always updated :-(
MAXFTPCACHE
= 10 # Trim the ftp cache beyond this size
# Helper for non-unix systems
from macurl2path
import url2pathname
, pathname2url
from nturl2path
import url2pathname
, pathname2url
elif os
.name
== 'riscos':
from rourl2path
import url2pathname
, pathname2url
def url2pathname(pathname
):
def pathname2url(pathname
):
# This really consists of two pieces:
# (1) a class which handles opening of all sorts of URLs
# (plus assorted utilities etc.)
# (2) a set of functions for parsing URLs
# XXX Should these be separated out into different modules?
# Shortcut for basic usage
def urlopen(url
, data
=None, proxies
=None):
"""urlopen(url [, data]) -> open file-like object"""
opener
= FancyURLopener(proxies
=proxies
)
opener
= FancyURLopener()
return opener
.open(url
, data
)
def urlretrieve(url
, filename
=None, reporthook
=None, data
=None):
_urlopener
= FancyURLopener()
return _urlopener
.retrieve(url
, filename
, reporthook
, data
)
# exception raised when downloaded size does not match content-length
class ContentTooShortError(IOError):
def __init__(self
, message
, content
):
IOError.__init
__(self
, message
)
This is a class rather than just a subroutine because we may need
more than one set of global protocol-specific options.
Note -- this is a base class for those who don't want the
automatic handling of errors type 302 (relocated) and 401
(authorization needed)."""
version
= "Python-urllib/%s" % __version__
def __init__(self
, proxies
=None, **x509
):
assert hasattr(proxies
, 'has_key'), "proxies must be a mapping"
self
.key_file
= x509
.get('key_file')
self
.cert_file
= x509
.get('cert_file')
self
.addheaders
= [('User-agent', self
.version
)]
self
.__unlink
= os
.unlink
# See cleanup()
# Undocumented feature: if you assign {} to tempcache,
# it is used to cache files retrieved with
# self.retrieve(). This is not enabled by default
# since it does not work for changing documents (and I
# haven't got the logic to check expiration headers
# Undocumented feature: you can use a different
# ftp cache by assigning to the .ftpcache member;
# in case you want logically independent URL openers
# XXX This is not threadsafe. Bah.
# This code sometimes runs when the rest of this module
# has already been deleted, so it can't use any globals
for file in self
.__tempfiles
:
def addheader(self
, *args
):
"""Add a header to be used by the HTTP interface only
e.g. u.addheader('Accept', 'sound/basic')"""
self
.addheaders
.append(args
)
def open(self
, fullurl
, data
=None):
"""Use URLopener().open(file) instead of open(file, 'r')."""
fullurl
= unwrap(toBytes(fullurl
))
if self
.tempcache
and fullurl
in self
.tempcache
:
filename
, headers
= self
.tempcache
[fullurl
]
fp
= open(filename
, 'rb')
return addinfourl(fp
, headers
, fullurl
)
urltype
, url
= splittype(fullurl
)
if urltype
in self
.proxies
:
proxy
= self
.proxies
[urltype
]
urltype
, proxyhost
= splittype(proxy
)
host
, selector
= splithost(proxyhost
)
url
= (host
, fullurl
) # Signal special case to open_*()
name
= name
.replace('-', '_')
if not hasattr(self
, name
):
return self
.open_unknown_proxy(proxy
, fullurl
, data
)
return self
.open_unknown(fullurl
, data
)
return getattr(self
, name
)(url
)
return getattr(self
, name
)(url
, data
)
except socket
.error
, msg
:
raise IOError, ('socket error', msg
), sys
.exc_info()[2]
def open_unknown(self
, fullurl
, data
=None):
"""Overridable interface to open unknown URL type."""
type, url
= splittype(fullurl
)
raise IOError, ('url error', 'unknown url type', type)
def open_unknown_proxy(self
, proxy
, fullurl
, data
=None):
"""Overridable interface to open unknown URL type."""
type, url
= splittype(fullurl
)
raise IOError, ('url error', 'invalid proxy for %s' % type, proxy
)
def retrieve(self
, url
, filename
=None, reporthook
=None, data
=None):
"""retrieve(url) returns (filename, headers) for a local object
or (tempfilename, headers) for a remote object."""
url
= unwrap(toBytes(url
))
if self
.tempcache
and url
in self
.tempcache
:
return self
.tempcache
[url
]
type, url1
= splittype(url
)
if filename
is None and (not type or type == 'file'):
fp
= self
.open_local_file(url1
)
return url2pathname(splithost(url1
)[1]), hdrs
fp
= self
.open(url
, data
)
tfp
= open(filename
, 'wb')
garbage
, path
= splittype(url
)
garbage
, path
= splithost(path
or "")
path
, garbage
= splitquery(path
or "")
path
, garbage
= splitattr(path
or "")
suffix
= os
.path
.splitext(path
)[1]
(fd
, filename
) = tempfile
.mkstemp(suffix
)
self
.__tempfiles
.append(filename
)
tfp
= os
.fdopen(fd
, 'wb')
result
= filename
, headers
if self
.tempcache
is not None:
self
.tempcache
[url
] = result
if "content-length" in headers
:
size
= int(headers
["Content-Length"])
reporthook(blocknum
, bs
, size
)
reporthook(blocknum
, bs
, size
)
# raise exception if actual size does not match content-length header
if size
>= 0 and read
< size
:
raise ContentTooShortError("retrieval incomplete: got only %i out "
"of %i bytes" % (read
, size
), result
)
# Each method named open_<type> knows how to open that type of URL
def open_http(self
, url
, data
=None):
host
, selector
= splithost(url
)
user_passwd
, host
= splituser(host
)
urltype
, rest
= splittype(selector
)
if urltype
.lower() != 'http':
realhost
, rest
= splithost(rest
)
user_passwd
, realhost
= splituser(realhost
)
selector
= "%s://%s%s" % (urltype
, realhost
, rest
)
if proxy_bypass(realhost
):
#print "proxy via http:", host, selector
if not host
: raise IOError, ('http error', 'no host given')
auth
= base64
.encodestring(user_passwd
).strip()
h
.putrequest('POST', selector
)
h
.putheader('Content-type', 'application/x-www-form-urlencoded')
h
.putheader('Content-length', '%d' % len(data
))
h
.putrequest('GET', selector
)
if auth
: h
.putheader('Authorization', 'Basic %s' % auth
)
if realhost
: h
.putheader('Host', realhost
)
for args
in self
.addheaders
: h
.putheader(*args
)
errcode
, errmsg
, headers
= h
.getreply()
return addinfourl(fp
, headers
, "http:" + url
)
return self
.http_error(url
, fp
, errcode
, errmsg
, headers
)
return self
.http_error(url
, fp
, errcode
, errmsg
, headers
, data
)
def http_error(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
Derived class can override this, or provide specific handlers
named http_error_DDD where DDD is the 3-digit error code."""
# First check if there's a specific handler for this error
name
= 'http_error_%d' % errcode
method
= getattr(self
, name
)
result
= method(url
, fp
, errcode
, errmsg
, headers
)
result
= method(url
, fp
, errcode
, errmsg
, headers
, data
)
return self
.http_error_default(url
, fp
, errcode
, errmsg
, headers
)
def http_error_default(self
, url
, fp
, errcode
, errmsg
, headers
):
"""Default error handler: close the connection and raise IOError."""
raise IOError, ('http error', errcode
, errmsg
, headers
)
if hasattr(socket
, "ssl"):
def open_https(self
, url
, data
=None):
"""Use HTTPS protocol."""
host
, selector
= splithost(url
)
user_passwd
, host
= splituser(host
)
urltype
, rest
= splittype(selector
)
if urltype
.lower() != 'https':
realhost
, rest
= splithost(rest
)
user_passwd
, realhost
= splituser(realhost
)
selector
= "%s://%s%s" % (urltype
, realhost
, rest
)
#print "proxy via https:", host, selector
if not host
: raise IOError, ('https error', 'no host given')
auth
= base64
.encodestring(user_passwd
).strip()
h
= httplib
.HTTPS(host
, 0,
cert_file
=self
.cert_file
)
h
.putrequest('POST', selector
)
h
.putheader('Content-type',
'application/x-www-form-urlencoded')
h
.putheader('Content-length', '%d' % len(data
))
h
.putrequest('GET', selector
)
if auth
: h
.putheader('Authorization', 'Basic %s' % auth
)
if realhost
: h
.putheader('Host', realhost
)
for args
in self
.addheaders
: h
.putheader(*args
)
errcode
, errmsg
, headers
= h
.getreply()
return addinfourl(fp
, headers
, "https:" + url
)
return self
.http_error(url
, fp
, errcode
, errmsg
, headers
)
return self
.http_error(url
, fp
, errcode
, errmsg
, headers
,
def open_gopher(self
, url
):
"""Use Gopher protocol."""
host
, selector
= splithost(url
)
if not host
: raise IOError, ('gopher error', 'no host given')
type, selector
= splitgophertype(selector
)
selector
, query
= splitquery(selector
)
selector
= unquote(selector
)
fp
= gopherlib
.send_query(selector
, query
, host
)
fp
= gopherlib
.send_selector(selector
, host
)
return addinfourl(fp
, noheaders(), "gopher:" + url
)
def open_file(self
, url
):
"""Use local file or FTP depending on form of URL."""
if url
[:2] == '//' and url
[2:3] != '/' and url
[2:12].lower() != 'localhost/':
return self
.open_ftp(url
)
return self
.open_local_file(url
)
def open_local_file(self
, url
):
import mimetypes
, mimetools
, email
.Utils
from cStringIO
import StringIO
from StringIO
import StringIO
host
, file = splithost(url
)
localname
= url2pathname(file)
stats
= os
.stat(localname
)
raise IOError(e
.errno
, e
.strerror
, e
.filename
)
modified
= email
.Utils
.formatdate(stats
.st_mtime
, usegmt
=True)
mtype
= mimetypes
.guess_type(url
)[0]
headers
= mimetools
.Message(StringIO(
'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
(mtype
or 'text/plain', size
, modified
)))
urlfile
= 'file://' + file
return addinfourl(open(localname
, 'rb'),
host
, port
= splitport(host
)
and socket
.gethostbyname(host
) in (localhost(), thishost()):
urlfile
= 'file://' + file
return addinfourl(open(localname
, 'rb'),
raise IOError, ('local file error', 'not on local host')
import mimetypes
, mimetools
from cStringIO
import StringIO
from StringIO
import StringIO
host
, path
= splithost(url
)
if not host
: raise IOError, ('ftp error', 'no host given')
host
, port
= splitport(host
)
user
, host
= splituser(host
)
if user
: user
, passwd
= splitpasswd(user
)
user
= unquote(user
or '')
passwd
= unquote(passwd
or '')
host
= socket
.gethostbyname(host
)
path
, attrs
= splitattr(path
)
dirs
, file = dirs
[:-1], dirs
[-1]
if dirs
and not dirs
[0]: dirs
= dirs
[1:]
if dirs
and not dirs
[0]: dirs
[0] = '/'
key
= user
, host
, port
, '/'.join(dirs
)
if len(self
.ftpcache
) > MAXFTPCACHE
:
# Prune the cache, rather arbitrarily
for k
in self
.ftpcache
.keys():
if not key
in self
.ftpcache
:
ftpwrapper(user
, passwd
, host
, port
, dirs
)
attr
, value
= splitvalue(attr
)
if attr
.lower() == 'type' and \
value
in ('a', 'A', 'i', 'I', 'd', 'D'):
(fp
, retrlen
) = self
.ftpcache
[key
].retrfile(file, type)
mtype
= mimetypes
.guess_type("ftp:" + url
)[0]
headers
+= "Content-Type: %s\n" % mtype
if retrlen
is not None and retrlen
>= 0:
headers
+= "Content-Length: %d\n" % retrlen
headers
= mimetools
.Message(StringIO(headers
))
return addinfourl(fp
, headers
, "ftp:" + url
)
raise IOError, ('ftp error', msg
), sys
.exc_info()[2]
def open_data(self
, url
, data
=None):
# dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
# mediatype := [ type "/" subtype ] *( ";" parameter )
# parameter := attribute "=" value
from cStringIO
import StringIO
from StringIO
import StringIO
[type, data
] = url
.split(',', 1)
raise IOError, ('data error', 'bad data URL')
type = 'text/plain;charset=US-ASCII'
if semi
>= 0 and '=' not in type[semi
:]:
msg
.append('Date: %s'%time
.strftime('%a, %d %b %Y %T GMT',
time
.gmtime(time
.time())))
msg
.append('Content-type: %s' % type)
data
= base64
.decodestring(data
)
msg
.append('Content-length: %d' % len(data
))
headers
= mimetools
.Message(f
, 0)
f
.fileno
= None # needed for addinfourl
return addinfourl(f
, headers
, url
)
class FancyURLopener(URLopener
):
"""Derived class with handlers for errors we can handle (perhaps)."""
def __init__(self
, *args
, **kwargs
):
URLopener
.__init
__(self
, *args
, **kwargs
)
def http_error_default(self
, url
, fp
, errcode
, errmsg
, headers
):
"""Default error handling -- don't raise an exception."""
return addinfourl(fp
, headers
, "http:" + url
)
def http_error_302(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
"""Error 302 -- relocated (temporarily)."""
if self
.maxtries
and self
.tries
>= self
.maxtries
:
if hasattr(self
, "http_error_500"):
meth
= self
.http_error_500
meth
= self
.http_error_default
return meth(url
, fp
, 500,
"Internal Server Error: Redirect Recursion", headers
)
result
= self
.redirect_internal(url
, fp
, errcode
, errmsg
, headers
,
def redirect_internal(self
, url
, fp
, errcode
, errmsg
, headers
, data
):
if 'location' in headers
:
newurl
= headers
['location']
# In case the server sent a relative URL, join with original:
newurl
= basejoin(self
.type + ":" + url
, newurl
)
def http_error_301(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
"""Error 301 -- also relocated (permanently)."""
return self
.http_error_302(url
, fp
, errcode
, errmsg
, headers
, data
)
def http_error_303(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
"""Error 303 -- also relocated (essentially identical to 302)."""
return self
.http_error_302(url
, fp
, errcode
, errmsg
, headers
, data
)
def http_error_307(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
"""Error 307 -- relocated, but turn POST into error."""
return self
.http_error_302(url
, fp
, errcode
, errmsg
, headers
, data
)
return self
.http_error_default(url
, fp
, errcode
, errmsg
, headers
)
def http_error_401(self
, url
, fp
, errcode
, errmsg
, headers
, data
=None):
"""Error 401 -- authentication required.
See this URL for a description of the basic authentication scheme:
http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt"""
if not 'www-authenticate' in headers
:
URLopener
.http_error_default(self
, url
, fp
,
errcode
, errmsg
, headers
)
stuff
= headers
['www-authenticate']
match
= re
.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff
)
URLopener
.http_error_default(self
, url
, fp
,
errcode
, errmsg
, headers
)
scheme
, realm
= match
.groups()
if scheme
.lower() != 'basic':
URLopener
.http_error_default(self
, url
, fp
,
errcode
, errmsg
, headers
)
name
= 'retry_' + self
.type + '_basic_auth'
return getattr(self
,name
)(url
, realm
)
return getattr(self
,name
)(url
, realm
, data
)
def retry_http_basic_auth(self
, url
, realm
, data
=None):
host
, selector
= splithost(url
)
user
, passwd
= self
.get_user_passwd(host
, realm
, i
)
if not (user
or passwd
): return None
host
= quote(user
, safe
='') + ':' + quote(passwd
, safe
='') + '@' + host
newurl
= 'http://' + host
+ selector
return self
.open(newurl
, data
)
def retry_https_basic_auth(self
, url
, realm
, data
=None):
host
, selector
= splithost(url
)
user
, passwd
= self
.get_user_passwd(host
, realm
, i
)
if not (user
or passwd
): return None
host
= quote(user
, safe
='') + ':' + quote(passwd
, safe
='') + '@' + host
newurl
= '//' + host
+ selector
return self
.open_https(newurl
, data
)
def get_user_passwd(self
, host
, realm
, clear_cache
= 0):
key
= realm
+ '@' + host
.lower()
if key
in self
.auth_cache
:
return self
.auth_cache
[key
]
user
, passwd
= self
.prompt_user_passwd(host
, realm
)
if user
or passwd
: self
.auth_cache
[key
] = (user
, passwd
)
def prompt_user_passwd(self
, host
, realm
):
"""Override this in a GUI environment!"""
user
= raw_input("Enter username for %s at %s: " % (realm
,
passwd
= getpass
.getpass("Enter password for %s in %s at %s: " %
except KeyboardInterrupt:
"""Return the IP address of the magic hostname 'localhost'."""
_localhost
= socket
.gethostbyname('localhost')
"""Return the IP address of the current host."""
_thishost
= socket
.gethostbyname(socket
.gethostname())
"""Return the set of errors raised by the FTP class."""
_ftperrors
= ftplib
.all_errors
"""Return an empty mimetools.Message object."""
from cStringIO
import StringIO
from StringIO
import StringIO
_noheaders
= mimetools
.Message(StringIO(), 0)
_noheaders
.fp
.close() # Recycle file descriptor
"""Class used by open_ftp() for cache of open FTP connections."""
def __init__(self
, user
, passwd
, host
, port
, dirs
):
self
.ftp
.connect(self
.host
, self
.port
)
self
.ftp
.login(self
.user
, self
.passwd
)
def retrfile(self
, file, type):
if type in ('d', 'D'): cmd
= 'TYPE A'; isdir
= 1
else: cmd
= 'TYPE ' + type; isdir
= 0
except ftplib
.all_errors
:
# Use nlst to see if the file exists at all
except ftplib
.error_perm
, reason
:
raise IOError, ('ftp error', reason
), sys
.exc_info()[2]
# Restore the transfer mode!
# Try to retrieve as a file
conn
= self
.ftp
.ntransfercmd(cmd
)
except ftplib
.error_perm
, reason
:
if str(reason
)[:3] != '550':
raise IOError, ('ftp error', reason
), sys
.exc_info()[2]
# Set transfer mode to ASCII!
self
.ftp
.voidcmd('TYPE A')
# Try a directory listing
if file: cmd
= 'LIST ' + file
conn
= self
.ftp
.ntransfercmd(cmd
)
# Pass back both a suitably decorated object and a retrieval length
return (addclosehook(conn
[0].makefile('rb'),
self
.endtransfer
), conn
[1])
"""Base class for addinfo and addclosehook."""
self
.readline
= self
.fp
.readline
if hasattr(self
.fp
, "readlines"): self
.readlines
= self
.fp
.readlines
if hasattr(self
.fp
, "fileno"): self
.fileno
= self
.fp
.fileno
if hasattr(self
.fp
, "__iter__"):
self
.__iter
__ = self
.fp
.__iter
__
if hasattr(self
.fp
, "next"):
return '<%s at %r whose fp = %r>' % (self
.__class
__.__name
__,
if self
.fp
: self
.fp
.close()
class addclosehook(addbase
):
"""Class to add a close hook to an open file."""
def __init__(self
, fp
, closehook
, *hookargs
):
addbase
.__init
__(self
, fp
)
self
.closehook
= closehook
self
.closehook(*self
.hookargs
)
"""class to add an info() method to an open file."""
def __init__(self
, fp
, headers
):
addbase
.__init
__(self
, fp
)
class addinfourl(addbase
):
"""class to add info() and geturl() methods to an open file."""
def __init__(self
, fp
, headers
, url
):
addbase
.__init
__(self
, fp
)
# Utilities to parse URLs (most of these return None for missing parts):
# unwrap('<URL:type://host/path>') --> 'type://host/path'
# splittype('type:opaquestring') --> 'type', 'opaquestring'
# splithost('//host[:port]/path') --> 'host[:port]', '/path'
# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
# splitpasswd('user:passwd') -> 'user', 'passwd'
# splitport('host:port') --> 'host', 'port'
# splitquery('/path?query') --> '/path', 'query'
# splittag('/path#tag') --> '/path', 'tag'
# splitattr('/path;attr1=value1;attr2=value2;...') ->
# '/path', ['attr1=value1', 'attr2=value2', ...]
# splitvalue('attr=value') --> 'attr', 'value'
# splitgophertype('/Xselector') --> 'X', 'selector'
# unquote('abc%20def') -> 'abc def'
# quote('abc def') -> 'abc%20def')
return isinstance(x
, unicode)
"""toBytes(u"URL") --> 'URL'."""
# Most URL schemes require ASCII. If that changes, the conversion
url
= url
.encode("ASCII")
raise UnicodeError("URL " + repr(url
) +
" contains non-ASCII characters")
"""unwrap('<URL:type://host/path>') --> 'type://host/path'."""
if url
[:1] == '<' and url
[-1:] == '>':
if url
[:4] == 'URL:': url
= url
[4:].strip()
"""splittype('type:opaquestring') --> 'type', 'opaquestring'."""
_typeprog
= re
.compile('^([^/:]+):')
match
= _typeprog
.match(url
)
return scheme
.lower(), url
[len(scheme
) + 1:]
"""splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
_hostprog
= re
.compile('^//([^/]*)(.*)$')
match
= _hostprog
.match(url
)
if match
: return match
.group(1, 2)
"""splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
_userprog
= re
.compile('^(.*)@(.*)$')
match
= _userprog
.match(host
)
if match
: return map(unquote
, match
.group(1, 2))
"""splitpasswd('user:passwd') -> 'user', 'passwd'."""
_passwdprog
= re
.compile('^([^:]*):(.*)$')
match
= _passwdprog
.match(user
)
if match
: return match
.group(1, 2)
# splittag('/path#tag') --> '/path', 'tag'
"""splitport('host:port') --> 'host', 'port'."""
_portprog
= re
.compile('^(.*):([0-9]+)$')
match
= _portprog
.match(host
)
if match
: return match
.group(1, 2)
def splitnport(host
, defport
=-1):
"""Split host and port, returning numeric port.
Return given default port if no ':' found; defaults to -1.
Return numerical port if a valid number are found after ':'.
Return None if ':' but not a valid number."""
_nportprog
= re
.compile('^(.*):(.*)$')
match
= _nportprog
.match(host
)
host
, port
= match
.group(1, 2)
if not port
: raise ValueError, "no digits"
"""splitquery('/path?query') --> '/path', 'query'."""
_queryprog
= re
.compile('^(.*)\?([^?]*)$')
match
= _queryprog
.match(url
)
if match
: return match
.group(1, 2)
"""splittag('/path#tag') --> '/path', 'tag'."""
_tagprog
= re
.compile('^(.*)#([^#]*)$')
match
= _tagprog
.match(url
)
if match
: return match
.group(1, 2)
"""splitattr('/path;attr1=value1;attr2=value2;...') ->
'/path', ['attr1=value1', 'attr2=value2', ...]."""
return words
[0], words
[1:]
"""splitvalue('attr=value') --> 'attr', 'value'."""
_valueprog
= re
.compile('^([^=]*)=(.*)$')
match
= _valueprog
.match(attr
)
if match
: return match
.group(1, 2)
def splitgophertype(selector
):
"""splitgophertype('/Xselector') --> 'X', 'selector'."""
if selector
[:1] == '/' and selector
[1:2]:
return selector
[1], selector
[2:]
_hextochr
= dict(('%02x' % i
, chr(i
)) for i
in range(256))
_hextochr
.update(('%02X' % i
, chr(i
)) for i
in range(256))
"""unquote('abc%20def') -> 'abc def'."""
for i
in xrange(1, len(res
)):
res
[i
] = _hextochr
[item
[:2]] + item
[2:]
"""unquote('%7e/abc+def') -> '~/abc def'"""
always_safe
= ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz'
def quote(s
, safe
= '/'):
"""quote('abc def') -> 'abc%20def'
Each part of a URL, e.g. the path info, the query, etc., has a
different set of reserved characters that must be quoted.
RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
the following reserved characters.
reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
Each of these characters is reserved in some component of a URL,
but not necessarily in all of them.
By default, the quote function is intended for quoting the path
section of a URL. Thus, it will not encode '/'. This character
is reserved, but in typical usage the quote function is being
called on a path where the existing slash characters are used as
cachekey
= (safe
, always_safe
)
safe_map
= _safemaps
[cachekey
]
safe_map
[c
] = (c
in safe
) and c
or ('%%%02X' % i
)
_safemaps
[cachekey
] = safe_map
res
= map(safe_map
.__getitem
__, s
)
def quote_plus(s
, safe
= ''):
"""Quote the query fragment of a URL; replacing ' ' with '+'"""
return s
.replace(' ', '+')
def urlencode(query
,doseq
=0):
"""Encode a sequence of two-element tuples or dictionary into a URL query string.
If any values in the query arg are sequences and doseq is true, each
sequence element is converted to a separate parameter.
If the query arg is a sequence of two-element tuples, the order of the
parameters in the output will match the order of parameters in the
if hasattr(query
,"items"):
# it's a bother at times that strings and string-like objects are
# non-sequence items should not work with len()
# non-empty strings will fail this
if len(query
) and not isinstance(query
[0], tuple):
# zero-length sequences of all types will get here and succeed,
# but that's a minor nit - since the original implementation
# allowed empty dicts that type of behavior probably should be
# preserved for consistency
ty
,va
,tb
= sys
.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
# is there a reasonable way to convert to ASCII?
# encode generates a string, but "replace" or "ignore"
# lose information and "strict" can raise UnicodeError
v
= quote_plus(v
.encode("ASCII","replace"))
# is this a sufficient test for sequence-ness?
l
.append(k
+ '=' + quote_plus(str(elt
)))
def getproxies_environment():
"""Return a dictionary of scheme -> proxy server URL mappings.
Scan the environment for variables named <scheme>_proxy;
this seems to be the standard convention. If you need a
different way, you can pass a proxies dictionary to the
[Fancy]URLopener constructor.
for name
, value
in os
.environ
.items():
if value
and name
[-6:] == '_proxy':
proxies
[name
[:-6]] = value
if sys
.platform
== 'darwin':
def getproxies_internetconfig():
"""Return a dictionary of scheme -> proxy server URL mappings.
By convention the mac uses Internet Config to store
proxies. An HTTP proxy, for instance, is stored under
if 'UseHTTPProxy' in config
and config
['UseHTTPProxy']:
value
= config
['HTTPProxyHost']
proxies
['http'] = 'http://%s' % value
# Gopher: XXXX To be done.
return getproxies_environment() or getproxies_internetconfig()
def getproxies_registry():
"""Return a dictionary of scheme -> proxy server URL mappings.
Win32 uses the registry to store proxies.
# Std module, so should be around - but you never know!
internetSettings
= _winreg
.OpenKey(_winreg
.HKEY_CURRENT_USER
,
r
'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
proxyEnable
= _winreg
.QueryValueEx(internetSettings
,
# Returned as Unicode but problems if not converted to ASCII
proxyServer
= str(_winreg
.QueryValueEx(internetSettings
,
for p
in proxyServer
.split(';'):
protocol
, address
= p
.split('=', 1)
# See if address has a type:// prefix
if not re
.match('^([^/:]+)://', address
):
address
= '%s://%s' % (protocol
, address
)
proxies
[protocol
] = address
# Use one setting for all protocols
if proxyServer
[:5] == 'http:':
proxies
['http'] = proxyServer
proxies
['http'] = 'http://%s' % proxyServer
proxies
['ftp'] = 'ftp://%s' % proxyServer
except (WindowsError, ValueError, TypeError):
# Either registry key not found etc, or the value in an
# proxies already set up to be empty so nothing to do
"""Return a dictionary of scheme -> proxy server URL mappings.
Returns settings gathered from the environment, if specified,
return getproxies_environment() or getproxies_registry()
# Std modules, so should be around - but you never know!
internetSettings
= _winreg
.OpenKey(_winreg
.HKEY_CURRENT_USER
,
r
'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
proxyEnable
= _winreg
.QueryValueEx(internetSettings
,
proxyOverride
= str(_winreg
.QueryValueEx(internetSettings
,
# ^^^^ Returned as Unicode but problems if not converted to ASCII
if not proxyEnable
or not proxyOverride
:
# try to make a host list from name and IP address.
addr
= socket
.gethostbyname(host
[0])
# make a check value list from the registry entry: replace the
# '<local>' string by the localhost entry and the corresponding
proxyOverride
= proxyOverride
.split(';')
while i
< len(proxyOverride
):
if proxyOverride
[i
] == '<local>':
proxyOverride
[i
:i
+1] = ['localhost',
# now check if we match one of the registry values.
for test
in proxyOverride
:
test
= test
.replace(".", r
"\.") # mask dots
test
= test
.replace("*", r
".*") # change glob sequence
test
= test
.replace("?", r
".") # change glob char
# print "%s <--> %s" %( test, val )
if re
.match(test
, val
, re
.I
):
# By default use environment variables
getproxies
= getproxies_environment
# Test and time quote() and unquote()
for i
in range(256): s
= s
+ chr(i
)
print round(t1
- t0
, 3), 'sec'
def reporthook(blocknum
, blocksize
, totalsize
):
# Report during remote transfers
print "Block number: %d, Block size: %d, Total size: %d" % (
blocknum
, blocksize
, totalsize
)
'file://localhost/etc/passwd',
'ftp://ftp.python.org/pub/python/README',
## 'gopher://gopher.micro.umn.edu/1/',
'http://www.python.org/index.html',
if hasattr(URLopener
, "open_https"):
args
.append('https://synergy.as.cmu.edu/~geek/')
print '-'*10, url
, '-'*10
fn
, h
= urlretrieve(url
, None, reporthook
)
for k
in h
.keys(): print k
+ ':', h
[k
]
table
= string
.maketrans("", "")
data
= data
.translate(table
, "\r")
opts
, args
= getopt
.getopt(sys
.argv
[1:], "th")
except getopt
.error
, msg
:
print "Usage: python urllib.py [-t] [url ...]"
print "-t runs self-test;",
print "otherwise, contents of urls are printed"
print urlopen(url
).read(),
# Run test program when run as a script
if __name__
== '__main__':