#!usr/bin/env python
# -*- coding:utf-8 -*-
import re
from lxml import etree
from bs4 import beautifulsoup as sp
import requests
import urllib2
import stringio
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
headers={'user-agent' : 'mozilla/4.0 (compatible; msie 5.5; windows nt)'}
def urlread(url):
try:
req=requests.get(url,headers=headers)
req.encoding="utf-8"
return req.text.encode("utf-8")
except:
req=urllib2.request(url,headers=headers)
response=urllib2.urlopen(req)
return response.read().encode("utf-8")
class item:
def __init__(self,title,link,date,description):
self.title=title.strip()
self.link=link.strip()
self.pubdate=date.strip()
self.decription=self.filter(description).strip()
def filter(self,description):
description=re.sub("<.*?>",'',description)
description=re.sub("\r",'',description)
description=re.sub("\n",'',description)
description=re.sub(" "," ",description)
if len(description)>240:
description=description[:240]+'...'
return description
def __str__(self):
return "%s\n%s\n%s\n<%s>\n" % (
self.title,
self.link,
self.decription,
self.pubdate
)
__repr__=__str__
class bsparser(object):
#url=''
def __init__(self,url):
xml=urlread(url)
self.reset(xml)
def reset(self,xml=none):
if xml==none:
self.soup=sp("<xml> </xml>")
else:
self.soup=sp(xml,"xml")
def callback(self,method,obj,tags):
rst=none
attr=method.lower()
for tag in tags:
try:
rst=getattr(obj,attr)(tag)
except:
continue
if rst:
break
return rst
def getfields(self,tags=["item",'entry']):
return self.callback(method="find_all",
obj=self.soup,
tags=tags)
def gettitle(self,obj,tags=["title"]):
return self.callback("find",obj,tags).text
def getlink(self,obj,tags=["link"]):
rst=self.callback("find",obj,tags).text
if not rst:
rst=self.callback("find",obj,tags).get("href")
return rst
def getdate(self,obj,tags=["pubdate","published"]):
return self.callback("find",obj,tags).text
def getdescription(self,obj,tags=["description","content"]):
return self.callback("find",obj,tags).text
def run(self):
for item in self.getfields():
title=self.gettitle(item)
link=self.getlink(item)
date=self.getdate(item)
description=self.getdescription(item)
newsitem=item(title,link,date,description)
yield newsitem
def test():
parser=parser()
for item in parser.run():
print item
if __name__=="__main__":
test()
以上就是分享一个简单的rss阅读工具的详细内容。
