I wrote a light weight python based screen scraper, which seems to be working great.
Some of the features:
- Session Management when using cookies
- Seperate functions for get,post and downloading large files
- Automatic handling of redirections
Here is the code for it.
import httplib,urllib,random,sys,re,os
from urlparse import urlparse
class PyScraper:
def __init__(self):
self.cookie=""
self.currenturl=""
self.urlhist=[]
def __str__(self):
ret=''
for item in self.urlhist:
ret=ret+item+'->'
return ret
def download(self,url,localfolder):
bufsize = 1024
self.urlhist.append(url)
o=urlparse(url)
scheme,hostname,path,q,query,position=o
head,fname = os.path.split(path)
if(query!=''):
path=path+"?"+query
conn=httplib.HTTPConnection(hostname)
conn.request('GET', path,None,{'Cookie':self.cookie})
resp=conn.getresponse()
total=int(resp.getheader('content-length'))
f = open(localfolder+"/"+fname,'wb')
sofar = 0
while 1:
data = resp.read(bufsize)
f.write(data)
sofar += len(data)
perc = (float(sofar)/float(total))
count = int(perc * 20)
sys.stdout.write("\r%-30s|%-20s|%3d percent" % (fname,'#'*count,perc*100))
sys.stdout.flush()
#sys.stdout.write("\r" + str(sofar) + " / " + str(total))total
if len(data)==0:
break
f.close()
if(resp.getheader('set-cookie')!=None):
self.cookie=resp.getheader('set-cookie')
conn.close()
if(resp.status==302 or resp.status ==301):
return self.get(resp.getheader('location'))
return data
def get(self,url):
self.urlhist.append(url)
o=urlparse(url)
scheme,hostname,path,q,query,position=o
if(query!=''):
path=path+"?"+query
conn=httplib.HTTPConnection(hostname)
conn.request('GET', path,None,{'Cookie':self.cookie})
resp=conn.getresponse()
data= resp.read()
if(resp.getheader('set-cookie')!=None):
self.cookie=resp.getheader('set-cookie')
conn.close()
if(resp.status==302 or resp.status ==301):
return self.get(resp.getheader('location'))
return data
def post(self,url,data):
self.urlhist.append(url)
o=urlparse(url)
scheme,hostname,path,q,query,position=o
conn=httplib.HTTPConnection(hostname)
conn.request('POST', path,data,
{'Content-Type':'application/x-www-form-urlencoded','Cookie':self.cookie})
resp=conn.getresponse()
data= resp.read()
if(resp.getheader('set-cookie')!=None):
self.cookie=resp.getheader('set-cookie')
conn.close()
if(resp.status==302 or resp.status ==301):
return self.post(resp.getheader('location'),data)
return data
Here is some snippet of code on how to use it.
from pyscraper import PyScraper
p=PyScraper()
data=p.get("http://www.yahoo.com/")
print data
