Automatically migrated from Gitolite
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

156 lines
6.0 KiB

10 years ago
  1. #!/usr/bin/python
  2. import os, argparse, hashlib, email, email.header, email.utils, glob, sqlite3, time
  3. parser = argparse.ArgumentParser(description='Parses emails into an SQLite database, and optionally renders static HTML files.')
  4. parser.add_argument('-p', dest='pattern', action='store', default='*',
  5. help='glob pattern (including path) that has to be matched for a file to be parsed')
  6. parser.add_argument('-d', dest='database', action='store', default='emails.db',
  7. help='path of the database that should be used to store the emails (will be created if it does not exist yet)')
  8. parser.add_argument('-a', dest='attachment_dir', action='store', default='attachments',
  9. help='path where attachments should be stored (will be created if it does not exist yet)')
  10. parser.add_argument('-r', '--render', dest='render', action='store_true',
  11. help='render static HTML files using template files')
  12. parser.add_argument('-f', '--forced', dest='forced', action='store_true',
  13. help='force insertion into database, even if entries already exist')
  14. args = parser.parse_args()
  15. options = vars(args)
  16. def getheader(header_text, default="ascii"):
  17. headers = email.header.decode_header(header_text)
  18. header_sections = [unicode(text, charset or default) for text, charset in headers]
  19. return u"".join(header_sections)
  20. def find_submessages(message):
  21. if message.is_multipart():
  22. return [find_submessages(part) for part in message.get_payload()]
  23. else:
  24. return message
  25. def flatten(x):
  26. # http://kogs-www.informatik.uni-hamburg.de/~meine/python_tricks
  27. result = []
  28. for el in x:
  29. if hasattr(el, "__iter__") and not isinstance(el, basestring):
  30. result.extend(flatten(el))
  31. else:
  32. result.append(el)
  33. return result
  34. def get_charset(part):
  35. if part.get_content_charset():
  36. return part.get_content_charset()
  37. elif part.get_charset():
  38. return part.get_charset()
  39. else:
  40. return "ascii"
  41. # Connect to database
  42. database = sqlite3.connect(options['database'])
  43. cursor = database.cursor()
  44. try:
  45. # Create attachment directory first
  46. os.makedirs(options['attachment_dir'])
  47. except OSError:
  48. pass
  49. try:
  50. # Try to create emails table
  51. cursor.execute("CREATE TABLE emails (`message_id`, `from`, `to`, `subject`, `date`, `body`, `html`, `hash`)")
  52. except sqlite3.OperationalError:
  53. pass
  54. try:
  55. # Try to create attachments table
  56. cursor.execute("CREATE TABLE attachments (`message_hash`, `filename`, `type`, `hash`, `size`)")
  57. except sqlite3.OperationalError:
  58. pass
  59. # Select all files matching the given pattern
  60. file_list = glob.glob(options['pattern'])
  61. for email_file in file_list:
  62. message = email.message_from_file(open(email_file))
  63. if message['message-id'] is None:
  64. print "%s is not a valid e-mail file." % email_file
  65. else:
  66. if 'subject' not in message or message['subject'] is None:
  67. subject = ""
  68. else:
  69. subject = message['subject']
  70. textbody = ""
  71. htmlbody = ""
  72. attachment_list = []
  73. sha1_hash = hashlib.sha1("%s/%s/%s/%s" % (message['from'], message['to'], message['message-id'], subject)).hexdigest()
  74. message_parts = [find_submessages(message)]
  75. message_parts = flatten(message_parts)
  76. for part in message_parts:
  77. if part.get_filename() is None:
  78. # Part of the message
  79. if part.get_content_type() == "text/plain":
  80. if textbody == "":
  81. textbody = part.get_payload(decode=True).decode(get_charset(part))
  82. elif part.get_content_type() == "text/html" or part.get_content_type == "text/xhtml+xml" or part.get_content_type == "application/xhtml+xml":
  83. if htmlbody == "":
  84. htmlbody = part.get_payload(decode=True).decode(get_charset(part))
  85. else:
  86. # Technically this is supposed to be part of the message body, but we have no idea what format it is in...
  87. print "WARNING: Unknown message format encountered in %s, message may be incomplete." % sha1_hash
  88. else:
  89. # Attachment
  90. attachment_data = part.get_payload(decode=True)
  91. attachment_sha1 = hashlib.sha1(attachment_data).hexdigest()
  92. attachment_filename = part.get_filename()
  93. attachment_type = part.get_content_type()
  94. attachment_extension = os.path.splitext(attachment_filename)[1][1:]
  95. attachment_size = len(attachment_data)
  96. attachment_destination = "%s/%s.%s" % (options['attachment_dir'], attachment_sha1, attachment_extension)
  97. attachment_list.append((attachment_filename, attachment_type, attachment_sha1, attachment_size))
  98. attachment_file = open(attachment_destination, "w")
  99. attachment_file.write(attachment_data)
  100. attachment_file.close()
  101. try:
  102. timestamp = int(time.mktime(email.utils.parsedate(message['date'])))
  103. except TypeError:
  104. timestamp = 0
  105. print "WARNING: Failed to determine unix timestamp for %s." % sha1_hash
  106. cursor.execute("SELECT * FROM emails WHERE `Hash` = ?", (sha1_hash,))
  107. if len(cursor.fetchall()) == 0 or options['forced'] == True:
  108. new_row = (getheader(message['message-id']), getheader(message['from']), getheader(message['to']), getheader(subject), timestamp, textbody, htmlbody, sha1_hash)
  109. cursor.execute("INSERT INTO emails VALUES (?, ?, ?, ?, ?, ?, ?, ?)", new_row)
  110. print "Successfully parsed and inserted e-mail with SHA1 hash %s." % sha1_hash
  111. else:
  112. print "Skipping %s, already exists in the database." % sha1_hash
  113. if len(attachment_list) > 0:
  114. inserted = 0
  115. for attachment in attachment_list:
  116. cursor.execute("SELECT * FROM attachments WHERE `Hash` = ?", (attachment[2],))
  117. if len(cursor.fetchall()) == 0 or options['forced'] == True:
  118. new_row = (sha1_hash, attachment[0], attachment[1], attachment[2], attachment[3])
  119. cursor.execute("INSERT INTO attachments VALUES (?, ?, ?, ?, ?)", new_row)
  120. inserted += 1
  121. else:
  122. print "Skipping attachment %s, already exists in the database." % attachment[2]
  123. if inserted > 0:
  124. print "Successfully inserted %d attachment(s) for %s." % (inserted, sha1_hash)
  125. database.commit()
  126. print "Changes successfully committed to database."