You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
#!/usr/bin/python
|
|
start_page = "http://www.devilskitchen.me.uk/"
|
|
default_headers = {
|
|
'User-Agent': "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13",
|
|
'Referer': start_page
|
|
}
|
|
|
|
import re, urllib2, datetime, argparse, os, json, time
|
|
from bs4 import BeautifulSoup
|
|
|
|
parser = argparse.ArgumentParser(description='Archive posts for devilskitchen.me.uk in JSON format')
|
|
|
|
parser.add_argument('-O', dest='output_dir', action='store', default='.',
|
|
help='output directory for archived posts')
|
|
|
|
args = parser.parse_args()
|
|
options = vars(args)
|
|
|
|
try:
|
|
os.mkdir(options['output_dir'])
|
|
except:
|
|
pass
|
|
|
|
def fetch_page_headers(url, headers):
|
|
request = urllib2.Request(url, headers=headers)
|
|
opener = urllib2.build_opener()
|
|
response = opener.open(request)
|
|
return (response.code, response.headers, response.read())
|
|
|
|
def fetch_archives():
|
|
status_code, headers, response = fetch_page_headers(start_page, default_headers)
|
|
|
|
if status_code == 200:
|
|
archive_pages = re.findall("(http:\/\/www\.devilskitchen\.me\.uk\/([0-9]{4})_([0-9]{2})_[0-9]{2}_archive\.html)", response)
|
|
|
|
for page in archive_pages:
|
|
if os.path.exists("%s/%s-%d" % (options['output_dir'], page[1], int(page[2]))):
|
|
print "%s-%s already exists, skipping..." % (page[1], page[2])
|
|
else:
|
|
print "Scraping %s..." % page[0]
|
|
fetch_articles(page[0])
|
|
time.sleep(5)
|
|
else:
|
|
print "ERROR: Failed to retrieve archive index! Exiting..."
|
|
exit(1)
|
|
|
|
|
|
def fetch_articles(url):
|
|
try:
|
|
status_code, headers, response = fetch_page_headers(url, default_headers)
|
|
except urllib2.HTTPError, e:
|
|
print "ERROR: 403 encountered on %s" % url
|
|
return False
|
|
|
|
if status_code == 200:
|
|
soup = BeautifulSoup(response)
|
|
posts = soup.find_all("div", class_="post")
|
|
|
|
for post in posts:
|
|
try:
|
|
post_title = post.h5.string
|
|
except AttributeError, e:
|
|
print "WARNING: Article with missing title"
|
|
post_title = ""
|
|
|
|
author_details = post.find_all("p", class_="author-details")[0]
|
|
author_name = author_details.find_all("span", class_="author-name")[0].string
|
|
post_date = author_details.find_all("a")[0].string
|
|
post_body = post.find_all("div", class_="post-body")[0].div.prettify()
|
|
|
|
actual_date = datetime.datetime.strptime(post_date, "%m/%d/%Y %I:%M:%S %p")
|
|
|
|
try:
|
|
os.mkdir("%s/%d-%d" % (options['output_dir'], actual_date.year, actual_date.month))
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
json_file = open("%s/%d-%d/%d-%d-%d-%d-%d-%d.json" % (options['output_dir'], actual_date.year, actual_date.month, actual_date.year,
|
|
actual_date.month, actual_date.day, actual_date.hour, actual_date.minute, actual_date.second), 'w')
|
|
|
|
json.dump({
|
|
'title': post_title,
|
|
'date': actual_date.isoformat(),
|
|
'author': author_name,
|
|
'body': post_body
|
|
}, json_file)
|
|
|
|
json_file.close()
|
|
except:
|
|
raise
|
|
|
|
|
|
print "Archived '%s', posted at %s by %s" % (post_title, actual_date.isoformat(), author_name)
|
|
else:
|
|
print "ERROR: Failed to retrieve %s! Status code was %d" % (url, status_code)
|
|
|
|
#soup = BeautifulSoup(html_doc)
|
|
fetch_archives()
|