You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
141 lines
5.0 KiB
Python
141 lines
5.0 KiB
Python
import urllib, gzip, sys, argparse, sqlite3, datetime, time
|
|
from lxml.etree import iterparse
|
|
|
|
xml_url = "http://img.jamendo.com/data/dbdump_artistalbumtrack.xml.gz"
|
|
|
|
def update_progress(count, blocksize, totalsize):
|
|
percent = int(count * blocksize * 100 / totalsize)
|
|
if options['no_output'] == False:
|
|
sys.stdout.write("\rRetrieving Jamendo database... %2d%%" % percent)
|
|
sys.stdout.flush()
|
|
|
|
def get_attribute(element, tagname):
|
|
val = element.find(tagname)
|
|
|
|
if val is None:
|
|
return ""
|
|
else:
|
|
if val.text == "None":
|
|
return ""
|
|
else:
|
|
return val.text
|
|
|
|
parser = argparse.ArgumentParser(description='Downloads and parses the Jamendo XML dump, and creates an SQLite database with all artist, album, track, and tag data.')
|
|
|
|
parser.add_argument('-D', dest='no_download', action='store_true',
|
|
help='don\'t download the XML dump and use an existing XML dump instead')
|
|
|
|
parser.add_argument('-N', dest='no_output', action='store_true',
|
|
help='prevents the application from outputting anything')
|
|
|
|
parser.add_argument('-d', dest='database', action='store', default='jamendo.db',
|
|
help='path of the database that should be used to store the data (will be created if it does not exist yet)')
|
|
|
|
parser.add_argument('-x', dest='xml_path', action='store', default='jamendo.xml.gz',
|
|
help='path to the Jamendo XML dump (this is the file that will be created when a new dump is downloaded)')
|
|
|
|
args = parser.parse_args()
|
|
options = vars(args)
|
|
|
|
xml_file = options['xml_path']
|
|
|
|
if options['no_download'] == False:
|
|
urllib.urlretrieve(xml_url, xml_file, reporthook=update_progress)
|
|
if options['no_output'] == False:
|
|
print ""
|
|
|
|
database = sqlite3.connect(options['database'])
|
|
cursor = database.cursor()
|
|
|
|
try:
|
|
# Try to create artists table
|
|
cursor.execute("CREATE TABLE artists (`id`, `name`, `url`, `image`, `mbgid`, `location`)")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
try:
|
|
# Try to create albums table
|
|
cursor.execute("CREATE TABLE albums (`id`, `artist_id`, `name`, `url`, `releasedate`, `filename`, `mbgid`, `license_artwork`)")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
try:
|
|
# Try to create tracks table
|
|
cursor.execute("CREATE TABLE tracks (`id`, `artist_id`, `album_id`, `name`, `filename`, `mbgid`, `tracknumber`, `genre`, `license`)")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
try:
|
|
# Try to create tags table
|
|
cursor.execute("CREATE TABLE tags (`track_id`, `name`, `weight`)")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
xml = gzip.open(xml_file)
|
|
total = 0
|
|
|
|
for event, element in iterparse(xml, tag="artist"):
|
|
# id, name, url, image, mbgid, location, Albums
|
|
artistid = get_attribute(element, 'id')
|
|
name = get_attribute(element, 'name')
|
|
url = get_attribute(element, 'url')
|
|
image = get_attribute(element, 'image')
|
|
mbgid = get_attribute(element, 'mbgid')
|
|
|
|
location_element = element.find('location')
|
|
try:
|
|
country = get_attribute(location_element, 'country')
|
|
except AttributeError:
|
|
country = ""
|
|
|
|
cursor.execute("INSERT INTO artists VALUES (?, ?, ?, ?, ?, ?)", (artistid, name, url, image, mbgid, country))
|
|
|
|
for album in element.find('Albums'):
|
|
# id, name, url, releasedate, filename, mbgid, license_artwork, Tracks
|
|
albumid = get_attribute(album, 'id')
|
|
albumname = get_attribute(album, 'name')
|
|
albumurl = get_attribute(album, 'url')
|
|
albumrelease = int(time.mktime(datetime.datetime.strptime(get_attribute(album, 'releasedate').split('+')[0], '%Y-%m-%dT%H:%M:%S').timetuple()))
|
|
albumfilename = get_attribute(album, 'filename')
|
|
albummbgid = get_attribute(album, 'mbgid')
|
|
albumartworklicense = get_attribute(album, 'license_artwork')
|
|
|
|
cursor.execute("INSERT INTO albums VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (albumid, artistid, albumname, albumurl, albumrelease, albumfilename, albummbgid, albumartworklicense))
|
|
|
|
for track in album.find('Tracks'):
|
|
# id, name, filename, mbgid, numalbum, id3genre, license, Tags
|
|
trackid = get_attribute(track, 'id')
|
|
trackname = get_attribute(track, 'name')
|
|
trackfilename = get_attribute(track, 'filename')
|
|
trackmbgid = get_attribute(track, 'mbgid')
|
|
tracknumber = get_attribute(track, 'numalbum')
|
|
trackgenre = get_attribute(track, 'id3genre')
|
|
tracklicense = get_attribute(track, 'license')
|
|
|
|
cursor.execute("INSERT INTO tracks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (trackid, artistid, albumid, trackname, trackfilename, trackmbgid, tracknumber, trackgenre, tracklicense))
|
|
|
|
taglist = track.find('Tags')
|
|
|
|
if taglist is not None:
|
|
for tag in taglist:
|
|
# idstr, weight
|
|
tagid = get_attribute(tag, 'idstr')
|
|
tagweight = get_attribute(tag, 'weight')
|
|
cursor.execute("INSERT INTO tags VALUES (?, ?, ?)", (trackid, tagid, tagweight))
|
|
|
|
if options['no_output'] == False:
|
|
sys.stdout.write("\rInserting artists... %6d done" % (total + 1))
|
|
sys.stdout.flush()
|
|
|
|
total += 1
|
|
element.clear()
|
|
|
|
if options['no_output'] == False:
|
|
print ""
|
|
print "Parsed and inserted a total of %d artists." % total
|
|
|
|
database.commit()
|
|
|
|
if options['no_output'] == False:
|
|
print "Changes committed to database."
|