Rewrite scraper to be more modular, and convert the Coursera crawler to the new model
Sven Slootweg
12 years ago
parent
c2a8a66dac
commit
fb6c43a38f
@ -0,0 +1,26 @@
|
|||||||
|
import inspect, os, sys
|
||||||
|
|
||||||
|
my_path = os.path.dirname(inspect.getfile(inspect.currentframe()))
|
||||||
|
|
||||||
|
def _import_module_into_scope(modulename):
|
||||||
|
module = __import__(modulename)
|
||||||
|
|
||||||
|
for name in vars(module):
|
||||||
|
data = getattr(module, name)
|
||||||
|
globals()[name] = data
|
||||||
|
|
||||||
|
sys.path.insert(0, my_path)
|
||||||
|
|
||||||
|
for fname in os.listdir(my_path):
|
||||||
|
fpath = os.path.join(my_path, fname)
|
||||||
|
fbasename, fext = os.path.splitext(fname)
|
||||||
|
|
||||||
|
if os.path.isdir(fpath):
|
||||||
|
if os.path.isfile(os.path.join(my_path, fname, "__init__.py")):
|
||||||
|
# This is a python directory module
|
||||||
|
_import_module_into_scope(fname)
|
||||||
|
elif os.path.isfile(fpath) and fext == ".py" and fbasename != "__init__":
|
||||||
|
# This is a python file module
|
||||||
|
_import_module_into_scope(fbasename)
|
||||||
|
|
||||||
|
sys.path.remove(my_path)
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
import datetime, json, sys
|
||||||
|
import requests, oursql
|
||||||
|
import shared
|
||||||
|
|
||||||
|
class Coursera(shared.Scraper):
|
||||||
|
provider_id = 2
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.retrieve_dataset()
|
||||||
|
self.parse_dataset()
|
||||||
|
|
||||||
|
def retrieve_dataset(self):
|
||||||
|
self.dataset = requests.get("https://www.coursera.org/maestro/api/topic/list?full=1").json()
|
||||||
|
|
||||||
|
def parse_dataset(self):
|
||||||
|
for item in self.dataset:
|
||||||
|
self.process_item(item)
|
||||||
|
|
||||||
|
def process_item(self, item):
|
||||||
|
inserted, rowid = self.insert_topic(str(item["id"]), item["name"], description=item["short_description"], needs_enrollment=True)
|
||||||
|
|
||||||
|
if inserted:
|
||||||
|
self.env.log("Inserted topic %s" % item["name"])
|
||||||
|
else:
|
||||||
|
self.env.log("Skipped topic %s" % item["name"])
|
||||||
|
|
||||||
|
for course in item["courses"]:
|
||||||
|
self.process_course(course, rowid)
|
||||||
|
|
||||||
|
def process_course(self, course, topicid):
|
||||||
|
try:
|
||||||
|
start_date = datetime.datetime(course["start_year"], course["start_month"], course["start_day"])
|
||||||
|
except TypeError, e:
|
||||||
|
start_date = None
|
||||||
|
|
||||||
|
title = self.generate_title(course['name'], start_date)
|
||||||
|
|
||||||
|
inserted, itemid = self.insert_item(str(course["id"]), title, course["home_link"], has_topic=True, itemtype=self.COURSE, description=course["certificate_description"], start_date=start_date, topic_id=topicid)
|
||||||
|
|
||||||
|
if inserted:
|
||||||
|
self.env.log("Inserted item %s" % title)
|
||||||
|
else:
|
||||||
|
self.env.log("Skipped item %s" % title)
|
||||||
|
|
||||||
|
def generate_title(self, name, date):
|
||||||
|
if date is None:
|
||||||
|
return "%s (date undetermined)" % name
|
||||||
|
else:
|
||||||
|
return "%s (starting %s)" % (name, date.strftime("%b %d, %Y"))
|
||||||
|
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
import inspect, os, sys
|
||||||
|
|
||||||
|
my_path = os.path.dirname(inspect.getfile(inspect.currentframe()))
|
||||||
|
|
||||||
|
def _import_module_into_scope(modulename):
|
||||||
|
module = __import__(modulename)
|
||||||
|
|
||||||
|
for name in vars(module):
|
||||||
|
data = getattr(module, name)
|
||||||
|
globals()[name] = data
|
||||||
|
|
||||||
|
sys.path.insert(0, my_path)
|
||||||
|
|
||||||
|
for fname in os.listdir(my_path):
|
||||||
|
fpath = os.path.join(my_path, fname)
|
||||||
|
fbasename, fext = os.path.splitext(fname)
|
||||||
|
|
||||||
|
if os.path.isdir(fpath):
|
||||||
|
if os.path.isfile(os.path.join(my_path, fname, "__init__.py")):
|
||||||
|
# This is a python directory module
|
||||||
|
_import_module_into_scope(fname)
|
||||||
|
elif os.path.isfile(fpath) and fext == ".py" and fbasename != "__init__":
|
||||||
|
# This is a python file module
|
||||||
|
_import_module_into_scope(fbasename)
|
||||||
|
|
||||||
|
sys.path.remove(my_path)
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
import oursql
|
||||||
|
|
||||||
|
class Environment(object):
|
||||||
|
def connect(self, host="localhost", username="root", password="", database="learn"):
|
||||||
|
self.db = oursql.connect(host=host, user=username, passwd=password, db=database)
|
||||||
|
self.connected = True
|
||||||
|
|
||||||
|
def log(self, text):
|
||||||
|
print text
|
||||||
|
|
||||||
|
def Scraper(self, scraper_class):
|
||||||
|
s = scraper_class(self.db)
|
||||||
|
s.env = self
|
||||||
|
return s
|
||||||
@ -0,0 +1,100 @@
|
|||||||
|
class Scraper(object):
|
||||||
|
UNKNOWN = 0
|
||||||
|
TOPIC = 1
|
||||||
|
COURSE = 2
|
||||||
|
VIDEO = 3
|
||||||
|
ARTICLE = 4
|
||||||
|
EXERCISE = 5
|
||||||
|
QUIZ = 6
|
||||||
|
TEST = 7
|
||||||
|
BOOK = 8
|
||||||
|
AUDIOBOOK = 9
|
||||||
|
LECTURE = 10
|
||||||
|
|
||||||
|
provider_id = 0
|
||||||
|
|
||||||
|
def __init__(self, database=None):
|
||||||
|
if database is not None:
|
||||||
|
self.db = database
|
||||||
|
self.can_store = True
|
||||||
|
else:
|
||||||
|
self.can_store = False
|
||||||
|
|
||||||
|
def run(self, *args, **kwargs):
|
||||||
|
raise Exception("No run() method was specified for this scraper.")
|
||||||
|
|
||||||
|
def insert_topic(self, unique_id, title, override=False, **kwargs):
|
||||||
|
defaults = {
|
||||||
|
"needs_enrollment": False,
|
||||||
|
"creation_date": None,
|
||||||
|
"start_date": None,
|
||||||
|
"end_date": None,
|
||||||
|
"parent_id": 0,
|
||||||
|
"description": "",
|
||||||
|
"provider_name": ""
|
||||||
|
}
|
||||||
|
|
||||||
|
for kwarg, val in defaults.iteritems():
|
||||||
|
try:
|
||||||
|
if kwargs[kwarg] == None:
|
||||||
|
kwargs[kwarg] = defaults[kwarg]
|
||||||
|
except KeyError, e:
|
||||||
|
kwargs[kwarg] = defaults[kwarg]
|
||||||
|
|
||||||
|
c = self.db.cursor()
|
||||||
|
|
||||||
|
if override == True:
|
||||||
|
exists = False
|
||||||
|
else:
|
||||||
|
c.execute("SELECT `Id` FROM topics WHERE `Provider` = ? AND `ProviderId` = ? LIMIT 1", (self.provider_id, unique_id))
|
||||||
|
results = c.fetchall()
|
||||||
|
exists = (len(results) > 0)
|
||||||
|
|
||||||
|
if exists == True:
|
||||||
|
return (False, results[0][0])
|
||||||
|
else:
|
||||||
|
c.execute("INSERT INTO topics (`ParentId`, `Provider`, `ProviderId`, `Title`, `Description`, `Created`, `NeedsEnrollment`, `StartDate`, `EndDate`, `CustomProviderName`)"
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (kwargs['parent_id'], self.provider_id, unique_id, title, kwargs['description'], kwargs['creation_date'],
|
||||||
|
kwargs['needs_enrollment'], kwargs['start_date'], kwargs['end_date'], kwargs["provider_name"]))
|
||||||
|
|
||||||
|
return (True, c.lastrowid)
|
||||||
|
|
||||||
|
def insert_item(self, unique_id, title, item_url, override=False, **kwargs):
|
||||||
|
defaults = {
|
||||||
|
"views": None,
|
||||||
|
"has_topic": False,
|
||||||
|
"itemtype": 0,
|
||||||
|
"source_url": item_url,
|
||||||
|
"topic_id": 0,
|
||||||
|
"parent_id": 0,
|
||||||
|
"description": "",
|
||||||
|
"date": None,
|
||||||
|
"start_date": None,
|
||||||
|
"end_date": None,
|
||||||
|
"provider_name": ""
|
||||||
|
}
|
||||||
|
|
||||||
|
for kwarg, val in defaults.iteritems():
|
||||||
|
try:
|
||||||
|
if kwargs[kwarg] == None:
|
||||||
|
kwargs[kwarg] = defaults[kwarg]
|
||||||
|
except KeyError, e:
|
||||||
|
kwargs[kwarg] = defaults[kwarg]
|
||||||
|
|
||||||
|
c = self.db.cursor()
|
||||||
|
|
||||||
|
if override == True:
|
||||||
|
exists = False
|
||||||
|
else:
|
||||||
|
c.execute("SELECT `Id` FROM items WHERE `Provider` = ? AND `ProviderId` = ? LIMIT 1", (self.provider_id, unique_id))
|
||||||
|
results = c.fetchall()
|
||||||
|
exists = (len(results) > 0)
|
||||||
|
|
||||||
|
if exists == True:
|
||||||
|
return (False, results[0][0])
|
||||||
|
else:
|
||||||
|
c.execute("INSERT INTO items (`HasTopic`, `Type`, `Provider`, `ProviderId`, `Title`, `Description`, `ItemUrl`, `SourceUrl`, `Views`, `TopicId`, `ParentId`, `Date`, `StartDate`, `EndDate`, `CustomProviderName`)"
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (kwargs["has_topic"], kwargs["itemtype"], self.provider_id, unique_id, title, kwargs["description"], item_url, kwargs["source_url"],
|
||||||
|
kwargs["views"], kwargs["topic_id"], kwargs["parent_id"], kwargs["date"], kwargs["start_date"], kwargs["end_date"], kwargs["provider_name"]))
|
||||||
|
|
||||||
|
return (True, c.lastrowid)
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import shared, scrapers
|
||||||
|
|
||||||
|
env = shared.Environment()
|
||||||
|
env.connect(host="localhost", username="root", password="", database="learn")
|
||||||
|
|
||||||
|
scraper = env.Scraper(scrapers.Coursera)
|
||||||
|
scraper.run()
|
||||||
@ -1,47 +0,0 @@
|
|||||||
import requests
|
|
||||||
import oursql
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import lib
|
|
||||||
|
|
||||||
class CourseraCrawler(object):
|
|
||||||
def __init__(self):
|
|
||||||
self.db = lib.Database("localhost", "root")
|
|
||||||
|
|
||||||
def retrieve_dataset(self):
|
|
||||||
#self.dataset = requests.get("https://www.coursera.org/maestro/api/topic/list?full=1").json()
|
|
||||||
self.dataset = json.loads(open("coursera.json", "r").read())
|
|
||||||
|
|
||||||
def parse_dataset(self):
|
|
||||||
for item in self.dataset:
|
|
||||||
self.process_item(item)
|
|
||||||
|
|
||||||
def process_item(self, item):
|
|
||||||
inserted, rowid = self.db.insert_topic(2, str(item["id"]), item["name"], description=item["short_description"], needs_enrollment=True)
|
|
||||||
|
|
||||||
if inserted:
|
|
||||||
print "Inserted %s" % item["name"]
|
|
||||||
else:
|
|
||||||
print "Skipped %s" % item["name"]
|
|
||||||
|
|
||||||
for course in item["courses"]:
|
|
||||||
self.process_course(course, rowid)
|
|
||||||
|
|
||||||
def process_course(self, course, topicid):
|
|
||||||
try:
|
|
||||||
start_date = datetime.datetime(course["start_year"], course["start_month"], course["start_day"])
|
|
||||||
title = "%s: %s-%s-%s" % (course["name"], str(course["start_year"]).zfill(4), str(course["start_month"]).zfill(2), str(course["start_day"]).zfill(2))
|
|
||||||
except TypeError, e:
|
|
||||||
start_date = None
|
|
||||||
title = "%s (date undetermined)" % (course["name"])
|
|
||||||
|
|
||||||
inserted, itemid = self.db.insert_item(2, str(course["id"]), True, self.db.COURSE, title, course["home_link"], description=course["certificate_description"], start_date=start_date, topic_id=topicid)
|
|
||||||
|
|
||||||
if inserted:
|
|
||||||
print "\tInserted %s" % title
|
|
||||||
else:
|
|
||||||
print "\tSkipped %s" % title
|
|
||||||
|
|
||||||
crawler = CourseraCrawler()
|
|
||||||
crawler.retrieve_dataset()
|
|
||||||
crawler.parse_dataset()
|
|
||||||
Loading…
Reference in New Issue