Commit 5e3614b4 authored by Sylvain Thénault's avatar Sylvain Thénault
Browse files

make feedparser,rdflib,datafeed only recommands

parent 852e7c11f252
......@@ -5,14 +5,25 @@ import re
from datetime import datetime
from lxml.html import fromstring, tostring
import feedparser
import rdflib
from cubes.datafeed.sobjects import DataFeedParser
RDF = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
SIOC = rdflib.Namespace('http://rdfs.org/sioc/ns#')
DCTERMS = rdflib.Namespace('http://purl.org/dc/terms/')
try:
import feedparser
except ImportError:
feedparser = None
try:
import rdflib
except ImportError:
rdflib = None
else:
RDF = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
SIOC = rdflib.Namespace('http://rdfs.org/sioc/ns#')
DCTERMS = rdflib.Namespace('http://purl.org/dc/terms/')
try:
from cubes.datafeed.sobjects import DataFeedParser
except ImportError:
DataFeedParser = None
def get_subject(g, pred, obj):
subjects = list(g.subjects(pred, obj))
......@@ -135,50 +146,54 @@ def get_twitter_avatar(screen_name):
AVATAR_CACHE[screen_name] = user['profile_image_url']
return AVATAR_CACHE[screen_name]
class BlogPostParser(DataFeedParser):
__abstract__ = True
entity_type = 'BlogEntry'
def process(self, url):
for item in self.parse(url):
author = item.pop('author', None)
avatar = item.pop('avatar', None)
euri = self.sget_entity('ExternalUri', uri=item.pop('uri'))
if euri.same_as:
sys.stdout.write('.')
post = self.update_blogpost(euri.same_as[0], item)
else:
sys.stdout.write('+')
post = self.create_blogpost(item, euri)
if author:
account = self.sget_entity('UserAccount', name=author)
self.sget_relation(post.eid, 'has_creator', account.eid)
if avatar:
auri = self.sget_entity('ExternalUri', uri=avatar)
self.sget_relation(account.eid, 'has_avatar', auri.eid)
sys.stdout.flush()
def create_blogpost(self, item, uri):
entity = self._cw.create_entity(self.entity_type, **item)
entity.set_relations(same_as=uri)
return entity
def update_blogpost(self, entity, item):
entity.set_attributes(**item)
return entity
class BlogPostSiocParser(BlogPostParser):
__regid__ = 'blogpost-sioc'
parse = staticmethod(parse_blogpost_sioc)
class BlogPostRSSParser(BlogPostParser):
__regid__ = 'blogpost-rss'
parse = staticmethod(parse_blogpost_rss)
class MicroBlogPostRSSParser(BlogPostParser):
__regid__ = 'microblogpost-rss'
entity_type = 'MicroBlogEntry'
parse = staticmethod(parse_microblogpost_rss)
if DataFeedParser is not None:
class BlogPostParser(DataFeedParser):
__abstract__ = True
entity_type = 'BlogEntry'
def process(self, url):
for item in self.parse(url):
author = item.pop('author', None)
avatar = item.pop('avatar', None)
euri = self.sget_entity('ExternalUri', uri=item.pop('uri'))
if euri.same_as:
sys.stdout.write('.')
post = self.update_blogpost(euri.same_as[0], item)
else:
sys.stdout.write('+')
post = self.create_blogpost(item, euri)
if author:
account = self.sget_entity('UserAccount', name=author)
self.sget_relation(post.eid, 'has_creator', account.eid)
if avatar:
auri = self.sget_entity('ExternalUri', uri=avatar)
self.sget_relation(account.eid, 'has_avatar', auri.eid)
sys.stdout.flush()
def create_blogpost(self, item, uri):
entity = self._cw.create_entity(self.entity_type, **item)
entity.set_relations(same_as=uri)
return entity
def update_blogpost(self, entity, item):
entity.set_attributes(**item)
return entity
if rdflib is not None:
class BlogPostSiocParser(BlogPostParser):
__regid__ = 'blogpost-sioc'
parse = staticmethod(parse_blogpost_sioc)
if feedparser is not None:
class BlogPostRSSParser(BlogPostParser):
__regid__ = 'blogpost-rss'
parse = staticmethod(parse_blogpost_rss)
class MicroBlogPostRSSParser(BlogPostParser):
__regid__ = 'microblogpost-rss'
entity_type = 'MicroBlogEntry'
parse = staticmethod(parse_microblogpost_rss)
if __name__ == '__main__':
import sys
......@@ -189,4 +204,3 @@ if __name__ == '__main__':
parser = globals()[name]
pprint(list(parser(url)))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment