You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
115 lines
2.8 KiB
Python
115 lines
2.8 KiB
Python
#!/usr/bin/python
|
|
|
|
import re, argparse, os
|
|
|
|
def show_sorted(dictionary):
|
|
for entry in sorted(dictionary, key=dictionary.get, reverse=True):
|
|
if options.has_key('minimum') == False or dictionary[entry] > int(options['minimum']):
|
|
print str(dictionary[entry]).rjust(total_digits), entry
|
|
print ""
|
|
|
|
parser = argparse.ArgumentParser(description='Parse a lighttpd access log.')
|
|
|
|
parser.add_argument('logfiles', metavar='logfile', type=str, nargs='+',
|
|
help='path(s) of the logfile(s)')
|
|
|
|
parser.add_argument('-e', '--extensions', dest='extensions', action='store',
|
|
help='specify a comma-separated list of extensions to ignore during parsing')
|
|
|
|
parser.add_argument('-m', '--minimum', dest='minimum', action='store',
|
|
help='the counting threshold that has to be exceeded to display the entry')
|
|
|
|
args = parser.parse_args()
|
|
options = vars(args)
|
|
|
|
referers = {}
|
|
days = {}
|
|
hosts = {}
|
|
files = {}
|
|
urls = {}
|
|
extensions = {}
|
|
|
|
total_digits = 10
|
|
current_lines = 0
|
|
|
|
try:
|
|
ignore_extensions = options['extensions'].split(',')
|
|
except AttributeError:
|
|
ignore_extensions = []
|
|
|
|
for logpath in options['logfiles']:
|
|
try:
|
|
log = open(logpath, 'r')
|
|
|
|
for line in log:
|
|
try:
|
|
ip, hostname, dash, datetime, timezone, method, uri, version, status, size, referer, useragent = line.split(' ', 11)
|
|
datetime = datetime[1:]
|
|
date = datetime.split(':')[0]
|
|
timezone = timezone[:-1]
|
|
method = method[1:]
|
|
version = version[:-1]
|
|
useragent = useragent[1:-2]
|
|
referer = referer[1:-1]
|
|
filename = uri.split('?')[0]
|
|
extension = os.path.splitext(filename)[1][1:]
|
|
|
|
if extension not in ignore_extensions:
|
|
if hostname not in hosts:
|
|
hosts[hostname] = 0
|
|
|
|
if referer not in referers:
|
|
referers[referer] = 0
|
|
|
|
if date not in days:
|
|
days[date] = 0
|
|
|
|
if filename not in files:
|
|
files[filename] = 0
|
|
|
|
if uri not in urls:
|
|
urls[uri] = 0
|
|
|
|
if extension not in extensions:
|
|
extensions[extension] = 0
|
|
|
|
hosts[hostname] += 1
|
|
referers[referer] += 1
|
|
days[date] += 1
|
|
files[filename] += 1
|
|
urls[uri] += 1
|
|
extensions[extension] += 1
|
|
except ValueError:
|
|
print "Corrupt log line at line %d, contents: %s" % (current_lines + 1, line[:-1])
|
|
|
|
current_lines += 1
|
|
|
|
if current_lines % 1000 == 0:
|
|
print "Processed %d lines." % current_lines
|
|
except IOError:
|
|
print "Could not find file %s, ignored entry." % logpath
|
|
|
|
print "Top days:"
|
|
show_sorted(days)
|
|
print ""
|
|
|
|
print "Top requested hostnames:"
|
|
show_sorted(hosts)
|
|
print ""
|
|
|
|
print "Top files:"
|
|
show_sorted(files)
|
|
print ""
|
|
|
|
print "Top extensions:"
|
|
show_sorted(extensions)
|
|
print ""
|
|
|
|
print "Top referers:"
|
|
show_sorted(referers)
|
|
print ""
|
|
|
|
print "Top URLs:"
|
|
show_sorted(urls)
|
|
print ""
|