Here is my urllib class, it supports private proxies, cookies and the usual stuff like get and post requests
#!/usr/bin/python
# -*- coding: utf-8 -*-
import urllib, urllib2
class Http():
def __init__(self, proxy = False, cookie_support = False):
self.handlers = set()
if proxy:
self.handlers |= set([
urllib2.ProxyHandler({'http': 'http://%s'%self.proxy}),
urllib2.HTTPBasicAuthHandler()]
)
if cookie_support:
self.handlers |= set(
[urllib2.HTTPCookieProcessor()]
)
if self.handlers:
self.interface = urllib2.build_opener( *self.handlers )
else:
self.interface = urllib2.build_opener(urllib2.BaseHandler)
urllib2.install_opener(self.interface)
self.interface.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1')]
def prepare_request(self, url, post=False, referer=False):
request = urllib2.Request(url)
if referer:
request.add_header('Referer',referer)
return request
class Web():
def __init__(self, proxy = False, cookie_support = False, timeout = 60):
self.timeout = timeout
self.web = Http(proxy, cookie_support)
def fetch(self, url, post=False, referer=False):
request = self.web.prepare_request(url, referer)
if post:
post = urllib.urlencode(post)
response = self.web.interface.open(request, post, timeout = self.timeout)
else:
response = self.web.interface.open(request, timeout = self.timeout)
return response.read()
#example usage
if __name__ == "__main__":
w = Web()
print w.fetch('http://google.com')
