From b4edc85dbecab9ba853694ae969f33a2fc657a84 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 18 Sep 2012 00:42:35 +0200 Subject: [PATCH 01/21] Implement support for both UnrealIRCd and InspIRCd --- tools/irc/rakill.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/irc/rakill.py b/tools/irc/rakill.py index 193802d..5f1d544 100644 --- a/tools/irc/rakill.py +++ b/tools/irc/rakill.py @@ -41,10 +41,20 @@ parser.add_argument('--gline', dest='action_gline', action='store_true', parser.add_argument('--kill', dest='action_kill', action='store_true', help='kill all matching users') + +parser.add_argument('--unrealircd', dest='ircd_unreal', action='store_true', + help='assume UnrealIRCd') + +parser.add_argument('--inspircd', dest='ircd_insp', action='store_true', + help='assume InspIRCd') args = parser.parse_args() options = vars(args) +if options['ircd_unreal'] == False and options['ircd_insp'] == False: + print "You did not specify an IRCd. Try again with --inspircd or --unrealircd switch." + exit(1) + print "Connecting...", sock = socket.socket() @@ -77,7 +87,12 @@ while True: sock.send("OPER %s %s\r\n" % (options['username'], options['password'])) elif parts[0] == "381": print "Authenticated as oper." - sock.send("WHO ** h\r\n") + if options['ircd_insp'] == True: + # InspIRCd + sock.send("WHO ** h\r\n") + elif options['ircd_unreal'] == True: + # UnrealIRCd + sock.send("WHO +hR **\r\n") print "Requested userlist." elif parts[0] == "352": try: From d3e302d35ac8aa5a2432c00b9e3a367ceb5be9cf Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Wed, 19 Sep 2012 22:02:59 +0200 Subject: [PATCH 02/21] Add script for parsing the leaked RevTT account list --- tools/file-processing/revtt_clean.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tools/file-processing/revtt_clean.py diff --git a/tools/file-processing/revtt_clean.py b/tools/file-processing/revtt_clean.py new file mode 100644 index 0000000..4a314a6 --- /dev/null +++ b/tools/file-processing/revtt_clean.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +import sys + +lines = open(sys.argv[1]).readlines() + +accounts = {} +password_count = 0 + +for line in lines: + try: + username, password = line.split("-") + + username = username.strip()[1:-1] + password = password.strip()[1:-1] + + if username not in accounts: + accounts[username] = [password] + password_count += 1 + else: + if password not in accounts[username]: + accounts[username].append(password) + password_count += 1 + except ValueError, e: + pass + + +for username, passwords in accounts.iteritems(): + for password in passwords: + print "%s\t\t%s" % (username, password) + +print "Done, %d accounts with a total of %d passwords." % (len(accounts), password_count) From d2c5148346da4e8b1e728e6080b0aa3b55f5f685 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 22 Sep 2012 04:50:11 +0200 Subject: [PATCH 03/21] Add scraper for devils kitchen blog --- tools/scrapers/devilskitchen.py | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tools/scrapers/devilskitchen.py diff --git a/tools/scrapers/devilskitchen.py b/tools/scrapers/devilskitchen.py new file mode 100644 index 0000000..32c7fef --- /dev/null +++ b/tools/scrapers/devilskitchen.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +start_page = "http://www.devilskitchen.me.uk/" +default_headers = { + 'User-Agent': "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13", + 'Referer': start_page +} +date_regex = '(?P[0-9]{1,2})\/(?P[0-9]{1,2})\/(?P[0-9]{4})\s(?P[0-9]{1,2})[:.](?P[0-9]{1,2})[:.](?P[0-9]{1,2})\s(?PAM|PM)' + +import re, urllib2, datetime, argparse, os, json, time +from bs4 import BeautifulSoup + +parser = argparse.ArgumentParser(description='Archive posts for devilskitchen.me.uk in JSON format') + +parser.add_argument('-O', dest='output_dir', action='store', default='.', + help='output directory for archived posts') + +args = parser.parse_args() +options = vars(args) + +try: + os.mkdir(options['output_dir']) +except: + pass + +def fetch_page_headers(url, headers): + request = urllib2.Request(url, headers=headers) + opener = urllib2.build_opener() + response = opener.open(request) + return (response.code, response.headers, response.read()) + +def fetch_archives(): + status_code, headers, response = fetch_page_headers(start_page, default_headers) + + if status_code == 200: + archive_pages = re.findall("http:\/\/www\.devilskitchen\.me\.uk\/[0-9]{4}_[0-9]{2}_[0-9]{2}_archive\.html", response) + + for page in archive_pages: + print "Scraping %s..." % page + fetch_articles(page) + time.sleep(20) + else: + print "ERROR: Failed to retrieve archive index! Exiting..." + exit(1) + + +def fetch_articles(url): + status_code, headers, response = fetch_page_headers(url, default_headers) + + if status_code == 200: + soup = BeautifulSoup(response) + posts = soup.find_all("div", class_="post") + + for post in posts: + try: + post_title = post.h5.string + except AttributeError, e: + print "WARNING: Article with missing title" + post_title = "" + + author_details = post.find_all("p", class_="author-details")[0] + author_name = author_details.find_all("span", class_="author-name")[0].string + post_date = author_details.find_all("a")[0].string + post_body = post.find_all("div", class_="post-body")[0].div.prettify() + + actual_date = datetime.datetime.strptime(post_date, "%m/%d/%Y %I:%M:%S %p") + + try: + os.mkdir("%s/%d-%d" % (options['output_dir'], actual_date.year, actual_date.month)) + except: + pass + + try: + json_file = open("%s/%d-%d/%d-%d-%d-%d-%d-%d.json" % (options['output_dir'], actual_date.year, actual_date.month, actual_date.year, + actual_date.month, actual_date.day, actual_date.hour, actual_date.minute, actual_date.second), 'w') + + json.dump({ + 'title': post_title, + 'date': actual_date.isoformat(), + 'author': author_name, + 'body': post_body + }, json_file) + + json_file.close() + except: + raise + + + print "Archived '%s', posted at %s by %s" % (post_title, actual_date.isoformat(), author_name) + else: + print "ERROR: Failed to retrieve %s! Status code was %d" % (url, status_code) + +#soup = BeautifulSoup(html_doc) +fetch_archives() From aedb32cf76ee22b96d38b1ec769b3033a81aca7c Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 22 Sep 2012 04:50:42 +0200 Subject: [PATCH 04/21] Add DNSBL checker for IRC --- tools/irc/dnsbl.txt | 8 +++ tools/irc/ipcheck.py | 141 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tools/irc/dnsbl.txt create mode 100644 tools/irc/ipcheck.py diff --git a/tools/irc/dnsbl.txt b/tools/irc/dnsbl.txt new file mode 100644 index 0000000..f1e004c --- /dev/null +++ b/tools/irc/dnsbl.txt @@ -0,0 +1,8 @@ +dnsbl.dronebl.org +combined.abuse.ch +opm.tornevall.org +torserver.tor.dnsbl.sectoor.de +socks.dnsbl.sorbs.net +http.dnsbl.sorbs.net +b.barracudacentral.org +sbl.spamhaus.org diff --git a/tools/irc/ipcheck.py b/tools/irc/ipcheck.py new file mode 100644 index 0000000..1e7347a --- /dev/null +++ b/tools/irc/ipcheck.py @@ -0,0 +1,141 @@ +import socket, argparse, sys, time, re +from collections import deque +from threading import Thread + +dnsbls = open("dnsbl.txt").readlines() + +socket.setdefaulttimeout(0.5) + +total_checking = 0 + +class CheckThread(Thread): + def __init__(self, username, ip): + Thread.__init__(self) + self.username = username + self.ip = ip + + def run(self): + global total_checking + reversed_ip = ".".join(self.ip.split(".")[::-1]) + + for dnsbl in dnsbls: + dnsbl = dnsbl.strip() + + try: + socket.gethostbyname(reversed_ip + "." + dnsbl) + + # This is bad. + #sock.send("MODE #bottest +b *@%s\r\n" % self.ip) + #sock.send("KICK #bottest %s :User %s is blacklisted in %s!\r\n" % (self.username, self.username, dnsbl)) + #sock.send("PRIVMSG #bottest :User %s is blacklisted in %s!\r\n" % (self.username, dnsbl)) + #sock.send("PRIVMSG #bottest :User %s is blacklisted in %s!\r\n" % (self.username, dnsbl)) + sock.send("KILL %s :User %s is blacklisted in %s!\r\n" % (self.username, self.username, dnsbl)) + print "IP %s is blacklisted in %s!" % (self.ip, dnsbl) + total_checking -= 1 + return False + except: + # This is good. + print "No blacklist matches found for %s" % self.ip + pass + + sock.send("PRIVMSG #bottest :User %s is clean.\r\n" % self.username) + total_checking -= 1 + return True + +def run_tocheck(): + if total_checking <= 20 and len(to_check) > 0: + nickname = to_check.popleft() + ip = users[nickname] + t = CheckThread(nickname, ip) + t.start() + +def split_irc(message): + message = re.sub("(?<=[0-9A-Fa-fI]):(?=[0-9A-Fa-fI])", "[..]", message) + + if ":" in message: + first, second = message.split(":", 1) + return first.rstrip().split(" ") + [second] + else: + return message.split(" ") + +parser = argparse.ArgumentParser(description='Connects to an IRC network and scans for proxies.') + +parser.add_argument('-H', dest='hostname', action='store', required=True, + help='server to connect to') + +parser.add_argument('-o', dest='port', action='store', required=True, + help='port* to connect to') + +parser.add_argument('-u', dest='username', action='store', required=True, + help='oper username') + +parser.add_argument('-p', dest='password', action='store', required=True, + help='oper password') + +args = parser.parse_args() +options = vars(args) + +to_check = deque([]) +users = {} + +print "Connecting...", + +sock = socket.socket() +sock.settimeout(None) +sock.connect((options['hostname'], int(options['port']))) +readbuffer = "" + +print "connected" + +sock.send("NICK botkill\r\n") +sock.send("USER botkill %s 0 :ipcheck.py\r\n" % options['hostname']) + +print "Registered." + +while True: + readbuffer = readbuffer + sock.recv(1024) + lines = readbuffer.split("\n") + readbuffer = lines.pop() + + for line in lines: + run_tocheck() + + if line.startswith(":"): + origin, line = line.split(" ", 1) + + line = line.rstrip() + parts = split_irc(line) + + if parts[0] == "PING": + sock.send("PONG %s\r\n" % parts[1]) + print "Completed connection challenge." + elif parts[0] == "001": + sock.send("OPER %s %s\r\n" % (options['username'], options['password'])) + elif parts[0] == "381": + print "Authenticated as oper." + sock.send("JOIN #bottest\r\n") + elif parts[0] == "JOIN": + if parts[1].lower() == "#bottest": + username = origin[1:].split("!")[0] + print username + " joined" + sock.send("USERIP %s\r\n" % username) + elif parts[0] == "340": + try: + data = parts[2].split("=", 1) + nickname = data[0] + + if nickname.endswith("*"): + nickname = nickname[:-1] + + ip = data[1].split("@", 1)[1] + users[nickname] = ip + to_check.append(nickname) + run_tocheck() + print "User %s has IP %s" % (nickname, ip) + except: + pass + elif parts[0] == "491": + print "Invalid oper credentials given." + exit(1) + + time.sleep(0.005) From 1ed4df5fa1e43ecce4d455753d833506806b0d54 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 22 Sep 2012 04:51:10 +0200 Subject: [PATCH 05/21] We don't need that regex --- tools/scrapers/devilskitchen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/scrapers/devilskitchen.py b/tools/scrapers/devilskitchen.py index 32c7fef..5b37f00 100644 --- a/tools/scrapers/devilskitchen.py +++ b/tools/scrapers/devilskitchen.py @@ -4,7 +4,6 @@ default_headers = { 'User-Agent': "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13", 'Referer': start_page } -date_regex = '(?P[0-9]{1,2})\/(?P[0-9]{1,2})\/(?P[0-9]{4})\s(?P[0-9]{1,2})[:.](?P[0-9]{1,2})[:.](?P[0-9]{1,2})\s(?PAM|PM)' import re, urllib2, datetime, argparse, os, json, time from bs4 import BeautifulSoup From 8a8db12690195191cb56f65826e0914a2348154f Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 22 Sep 2012 05:02:19 +0200 Subject: [PATCH 06/21] Skip pages that are already scraped --- tools/scrapers/devilskitchen.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/scrapers/devilskitchen.py b/tools/scrapers/devilskitchen.py index 5b37f00..34085fe 100644 --- a/tools/scrapers/devilskitchen.py +++ b/tools/scrapers/devilskitchen.py @@ -31,12 +31,15 @@ def fetch_archives(): status_code, headers, response = fetch_page_headers(start_page, default_headers) if status_code == 200: - archive_pages = re.findall("http:\/\/www\.devilskitchen\.me\.uk\/[0-9]{4}_[0-9]{2}_[0-9]{2}_archive\.html", response) + archive_pages = re.findall("(http:\/\/www\.devilskitchen\.me\.uk\/([0-9]{4})_([0-9]{2})_[0-9]{2}_archive\.html)", response) for page in archive_pages: - print "Scraping %s..." % page - fetch_articles(page) - time.sleep(20) + if os.path.exists("%s/%s-%s" % (options['output_dir'], page[1], page[2])): + print "%s-%s already exists, skipping..." % (page[1], page[2]) + else: + print "Scraping %s..." % page[0] + fetch_articles(page[0]) + time.sleep(5) else: print "ERROR: Failed to retrieve archive index! Exiting..." exit(1) From f8af89449c73bfe78f4eee407b6047672263576e Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 22 Sep 2012 05:25:07 +0200 Subject: [PATCH 07/21] Fix resume functionality and skip 403ing archive pages --- tools/scrapers/devilskitchen.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/scrapers/devilskitchen.py b/tools/scrapers/devilskitchen.py index 34085fe..6ec7324 100644 --- a/tools/scrapers/devilskitchen.py +++ b/tools/scrapers/devilskitchen.py @@ -34,7 +34,7 @@ def fetch_archives(): archive_pages = re.findall("(http:\/\/www\.devilskitchen\.me\.uk\/([0-9]{4})_([0-9]{2})_[0-9]{2}_archive\.html)", response) for page in archive_pages: - if os.path.exists("%s/%s-%s" % (options['output_dir'], page[1], page[2])): + if os.path.exists("%s/%s-%d" % (options['output_dir'], page[1], int(page[2]))): print "%s-%s already exists, skipping..." % (page[1], page[2]) else: print "Scraping %s..." % page[0] @@ -46,7 +46,11 @@ def fetch_archives(): def fetch_articles(url): - status_code, headers, response = fetch_page_headers(url, default_headers) + try: + status_code, headers, response = fetch_page_headers(url, default_headers) + except urllib2.HTTPError, e: + print "ERROR: 403 encountered on %s" % url + return False if status_code == 200: soup = BeautifulSoup(response) From 84cf44088b578bcc7bd721517a4d2a395b5c2fad Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 2 Oct 2012 23:21:58 +0200 Subject: [PATCH 08/21] Add simple script for searching for music via the ex.fm API --- tools/downloading/musicsearch.py | 89 ++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tools/downloading/musicsearch.py diff --git a/tools/downloading/musicsearch.py b/tools/downloading/musicsearch.py new file mode 100644 index 0000000..3190991 --- /dev/null +++ b/tools/downloading/musicsearch.py @@ -0,0 +1,89 @@ +#!/usr/bin/python +#http://ex.fm/api/v3/song/search/rameau?results= +import argparse, os, urllib, json, subprocess, urllib2 + +parser = argparse.ArgumentParser(description='Searches for music files') + +parser.add_argument('-c', dest='limit', action='store', default='10', + help='maximum amount of results to show (don\'t overdo it please)') + +parser.add_argument('-v', dest='vlc', action='store_true', default='false', + help='play in VLC media player') + +parser.add_argument('query', metavar='QUERY', type=str, nargs='+', + help='destination path for generated class files.') + +args = parser.parse_args() +options = vars(args) + +query = " ".join(options['query']) +limit = int(options['limit']) + +results = urllib.urlopen("http://ex.fm/api/v3/song/search/%s?results=%d" % (urllib.quote_plus(query), limit)).read() + +try: + result_object = json.loads(results) +except ValueError: + print "No valid result was returned from the ex.fm API. Exiting..." + exit(1) + +if result_object['status_code'] != 200: + print "An error code was returned by the ex.fm API. Exiting..." + exit(1) + +print "Searching for '%s'..." % query + +if result_object['total'] <= 0: + print "No results." + exit(1) + +print "" + +for track in result_object['songs']: + if track['artist'] is None: + artist = "Unknown" + else: + artist = track['artist'] + + if track['album'] is None: + album = "Unknown" + else: + album = track['album'] + + if track['title'] is None: + title = "Unknown" + else: + title = track['title'] + + print "Artist: %s\t Album: %s" % (artist, album) + print "Title: %s" % title + print " %s" % track['url'] + print "" + +if options['vlc'] == True: + print "Playing the first working result in VLC media player..." + + working_url = "" + + for track in result_object['songs']: + try: + response = urllib2.urlopen(track['url']) + except urllib2.URLError, e: + continue + + headers = response.info() + + if "text/html" in headers['Content-Type']: + continue + + working_url = track['url'] + + break + + if working_url != "": + with open(os.devnull, 'w') as stfu: + subprocess.Popen(["vlc", "--one-instance", working_url], stdin=None, stdout=stfu, stderr=stfu) + exit(0) + else: + print "No working URLs found." + exit(1) From bfacb21d7023f783b9765dcb263078d51dd81c8c Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 2 Oct 2012 23:22:43 +0200 Subject: [PATCH 09/21] Remove a comment --- tools/downloading/musicsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/downloading/musicsearch.py b/tools/downloading/musicsearch.py index 3190991..5cd4050 100644 --- a/tools/downloading/musicsearch.py +++ b/tools/downloading/musicsearch.py @@ -1,5 +1,5 @@ #!/usr/bin/python -#http://ex.fm/api/v3/song/search/rameau?results= + import argparse, os, urllib, json, subprocess, urllib2 parser = argparse.ArgumentParser(description='Searches for music files') From cd7f6af54194032c9ce71f1a7a77a05bb205646f Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 4 Oct 2012 01:16:00 +0200 Subject: [PATCH 10/21] Add snippet for root path validation in PHP --- snippets/php/path_root_validation.php | 138 ++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 snippets/php/path_root_validation.php diff --git a/snippets/php/path_root_validation.php b/snippets/php/path_root_validation.php new file mode 100644 index 0000000..07bc79d --- /dev/null +++ b/snippets/php/path_root_validation.php @@ -0,0 +1,138 @@ +ParsePath($path); + + if($parsed_path != null) + { + $this->path = $parsed_path; + } + else + { + throw new Exception("Invalid path specified"); + } + } + + public function ValidatePath($root) + { + $root_path = $this->ParsePath($root); + + if($root_path != null) + { + $root_stack = explode("/", $root_path); + $path_stack = explode("/", $this->path); + + for($i = 0; $i < count($root_stack); $i++) + { + if($root_stack[$i] != $path_stack[$i]) + { + return false; + } + } + + return true; + } + else + { + throw new Exception("Specified root path is invalid."); + } + } + + public function RelativeDepth($root) + { + $root_length = substr_count($this->ParsePath($root), "/"); + $path_length = substr_count($this->path, "/"); + + return $path_length - $root_length; + } + + private function RemoveTrailingSlash($path) + { + if(substr($path, strlen($path) - 1) == "/") + { + return substr($path, 0, strlen($path) - 1); + } + else + { + return $path; + } + } + + private function ParsePath($path) + { + /* We use a custom function for this since we just want to resolve the path no matter what, + * and the realpath() function will return false if the path either doesn't exist or is not + * accessible. */ + + $path = $this->RemoveTrailingSlash($path); + + if(substr($path, 0, 1) == "/") + { + /* Absolute path */ + return $path; + } + else + { + $path_elements = explode("/", $path); + + if(substr($path, 0, 1) == "~") + { + /* Home directory path */ + if(!empty($_SERVER['home'])) + { + $homedir = $_SERVER['home']; + } + elseif(getenv("HOME") != null) + { + $homedir = getenv("HOME"); + } + elseif(function_exists("posix_getuid") && function_exists("posix_getpwuid")) + { + $userinfo = posix_getpwuid(posix_getuid()); + $homedir = $userinfo['dir']; + } + + $homedir = $this->RemoveTrailingSlash($homedir); + + $stack = explode("/", $homedir); + array_shift($path_elements); + } + else + { + /* Relative path */ + $basepath = $this->RemoveTrailingSlash(getcwd()); + $stack = explode("/", $basepath); + } + + foreach($path_elements as $element) + { + if($element == ".") + { + /* Ignore */ + } + elseif($element == "..") + { + /* Go up one directory */ + if(count($stack) > 1) + { + array_pop($stack); + } + else + { + /* There are no elements left to pop, this is an invalid path. */ + return null; + } + } + else + { + /* Append to path */ + $stack[] = $element; + } + } + + return implode("/", $stack); + } + } +} From 8c1b3464ffbd667feb005efea4969cadbb37140c Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 4 Oct 2012 01:17:46 +0200 Subject: [PATCH 11/21] Throw exception if the homedir cannot be determined --- snippets/php/path_root_validation.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/snippets/php/path_root_validation.php b/snippets/php/path_root_validation.php index 07bc79d..315e113 100644 --- a/snippets/php/path_root_validation.php +++ b/snippets/php/path_root_validation.php @@ -93,6 +93,10 @@ class PathValidator $userinfo = posix_getpwuid(posix_getuid()); $homedir = $userinfo['dir']; } + else + { + throw new Exception("Could not find a way to get the home directory of the current user."); + } $homedir = $this->RemoveTrailingSlash($homedir); From a3ca2ffcfb1fba28c8d2e8e34776d104ab155813 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 4 Oct 2012 01:20:48 +0200 Subject: [PATCH 12/21] Also return false for validation if the path is shorter than the root path --- snippets/php/path_root_validation.php | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/snippets/php/path_root_validation.php b/snippets/php/path_root_validation.php index 315e113..d3c2533 100644 --- a/snippets/php/path_root_validation.php +++ b/snippets/php/path_root_validation.php @@ -21,18 +21,25 @@ class PathValidator if($root_path != null) { - $root_stack = explode("/", $root_path); - $path_stack = explode("/", $this->path); - - for($i = 0; $i < count($root_stack); $i++) + if($this->RelativeDepth($root) >= 0) { - if($root_stack[$i] != $path_stack[$i]) + $root_stack = explode("/", $root_path); + $path_stack = explode("/", $this->path); + + for($i = 0; $i < count($root_stack); $i++) { - return false; + if($root_stack[$i] != $path_stack[$i]) + { + return false; + } } + + return true; + } + else + { + return false; } - - return true; } else { From 63adc8b797c64ce78ce92af9afd1ed9cc2ade852 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Wed, 10 Oct 2012 22:10:40 +0200 Subject: [PATCH 13/21] Add catarc utility --- tools/file-processing/catarc.py | 125 ++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100755 tools/file-processing/catarc.py diff --git a/tools/file-processing/catarc.py b/tools/file-processing/catarc.py new file mode 100755 index 0000000..026c36b --- /dev/null +++ b/tools/file-processing/catarc.py @@ -0,0 +1,125 @@ +#!/usr/bin/python + +# This script will run the appropriate command to print archive contents to stdout + +import os, argparse, sys, subprocess + +parser = argparse.ArgumentParser(description='Prints the uncompressed contents of archives to stdout, based on certain criteria.') + +parser.add_argument('pattern', metavar='FILES', type=str, nargs='+', + help='files to parse') + +parser.add_argument('-s', dest='size', action='store', default=None, + help='size requirement that has to be satisfied for a file to be printed (use < and >)') + +args = parser.parse_args() +options = vars(args) + +def exit_script(): + sys.stderr.write("Exiting...\n") + exit(1) + +def to_bytes(size): + size = size.lower().strip() + + if size.endswith("t"): + return int(size) * 1024 * 1024 * 1024 * 1024 + elif size.endswith("g"): + return int(size) * 1024 * 1024 * 1024 + elif size.endswith("m"): + return int(size) * 1024 * 1024 + elif size.endswith("k"): + return int(size) * 1024 + else: + return int(size) + +def from_bytes(size, unit): + size = int(size) + unit = unit.lower() + + if unit == 't': + return size * 1.0 / (1024 * 1024 * 1024 * 1024) + elif unit == 'g': + return size * 1.0 / (1024 * 1024 * 1024) + elif unit == 'm': + return size * 1.0 / (1024 * 1024) + elif unit == 'k': + return size * 1.0 / (1024) + else: + return size + +specifications = {} + +if options['size'] is not None: + # Parse size specification + for specification in options['size'].split(","): + if specification[:1] == "<": + specifications['<'] = specification[1:].strip() + elif specification[:1] == ">": + specifications['>'] = specification[1:].strip() + elif specification[:1] == "=": + specifications['='] = specification[1:].strip() + else: + sys.stderr.write("Incorrect size specification: %s\n" % specification) + exit_script() + +# Select all files matching the given pattern +file_list = options['pattern'] + +for item in file_list: + data = os.stat(item) + filesize = data.st_size + + try: + if filesize >= to_bytes(specifications['<']): + continue + except KeyError, e: + pass + + + try: + if filesize <= to_bytes(specifications['>']): + continue + except KeyError, e: + pass + + + try: + if int(from_bytes(filesize, specifications['='][-1:])) != specifications['=']: + continue + except KeyError, e: + pass + + # Passed all size tests, let's process it + with open(os.devnull, 'w') as stfu: + if item.endswith(".7z"): + processor = "7zip" + returncode = subprocess.call(['7z', 'e', '-so', item], stderr=stfu) + elif item.endswith(".tar"): + processor = "tar" + returncode = subprocess.call(['tar', '-Oxf', item], stderr=stfu) + elif item.endswith(".tar.gz"): + processor = "tar/gzip" + returncode = subprocess.call(['tar', '-Oxzf', item], stderr=stfu) + elif item.endswith(".tar.bz2"): + processor = "tar/bzip2" + returncode = subprocess.call(['tar', '-Oxjf', item], stderr=stfu) + elif item.endswith(".gz"): + processor = "gzip" + returncode = subprocess.call(['gzip', '-cd', item], stderr=stfu) + elif item.endswith(".bz2"): + processor = "bzip2" + returncode = subprocess.call(['bzip2', '-cd', item], stderr=stfu) + elif item.endswith(".zip"): + processor = "unzip" + returncode = subprocess.call(['unzip', '-p', item], stderr=stfu) + else: + sys.stderr.write("WARNING: Skipping item %s, not a recognized archive type\n" % item) + continue + + if returncode == 0: + sys.stderr.write("Successfully output %s\n" % item) + elif returncode == 2: + sys.stderr.write("ERROR: Could not run the needed command - are you sure you have %s installed?\n" % processor) + else: + sys.stderr.write("ERROR: Failed to output %s\n" % item) From 1547557bd05932a801df54c2e55f2de61e380675 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Wed, 10 Oct 2012 22:14:51 +0200 Subject: [PATCH 14/21] Fix size specification parsing --- tools/file-processing/catarc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/file-processing/catarc.py b/tools/file-processing/catarc.py index 026c36b..d19b173 100755 --- a/tools/file-processing/catarc.py +++ b/tools/file-processing/catarc.py @@ -23,13 +23,13 @@ def to_bytes(size): size = size.lower().strip() if size.endswith("t"): - return int(size) * 1024 * 1024 * 1024 * 1024 + return int(size[:-1]) * 1024 * 1024 * 1024 * 1024 elif size.endswith("g"): - return int(size) * 1024 * 1024 * 1024 + return int(size[:-1]) * 1024 * 1024 * 1024 elif size.endswith("m"): - return int(size) * 1024 * 1024 + return int(size[:-1]) * 1024 * 1024 elif size.endswith("k"): - return int(size) * 1024 + return int(size[:-1]) * 1024 else: return int(size) From 6272c210e1166cea0bc604aa55bfac877327e8ba Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Wed, 10 Oct 2012 22:20:54 +0200 Subject: [PATCH 15/21] Add license comment --- tools/file-processing/catarc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/file-processing/catarc.py b/tools/file-processing/catarc.py index d19b173..3c4d6d9 100755 --- a/tools/file-processing/catarc.py +++ b/tools/file-processing/catarc.py @@ -1,6 +1,7 @@ #!/usr/bin/python # This script will run the appropriate command to print archive contents to stdout +# Written by Sven Slootweg, Licensed under WTFPL - in other words, feel free to reuse for whatever purpose you desire. import os, argparse, sys, subprocess From 929d0a83205bec395f6cda123af97bef8c26e2a6 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 11 Oct 2012 00:24:35 +0200 Subject: [PATCH 16/21] Add webshots installer --- installers/webshots/INSTALL | 1 + installers/webshots/setup.sh | 16 ++++++++++++++++ installers/webshots/webshots_debian.sh | 24 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 installers/webshots/INSTALL create mode 100644 installers/webshots/setup.sh create mode 100644 installers/webshots/webshots_debian.sh diff --git a/installers/webshots/INSTALL b/installers/webshots/INSTALL new file mode 100644 index 0000000..3c7fa4f --- /dev/null +++ b/installers/webshots/INSTALL @@ -0,0 +1 @@ +wget http://cryto.net/projects/webshots/webshots_debian.sh && chmod +x webshots_debian.sh && ./webshots_debian.sh diff --git a/installers/webshots/setup.sh b/installers/webshots/setup.sh new file mode 100644 index 0000000..962f2cb --- /dev/null +++ b/installers/webshots/setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd ~ +git clone git://github.com/ArchiveTeam/webshots-grab.git +if [ $? -ne 0 ]; then echo "[X] Retrieving the grabber code failed. Exiting..."; exit 1; fi + +cd webshots-grab +./get-wget-lua.sh +if [ $? -ne 0 ]; then echo "[X] Compiling wget-lua failed. Exiting..."; exit 1; fi + +echo "What username would you like to use? " +read USERNAME +echo "How many threads? (start out with 2 or so) " +read THREADCOUNT +echo "run-pipeline --concurrent $THREADCOUNT ~/webshots-grab/pipeline.py $USERNAME" > ./start.sh +chmod +x ./start.sh +echo "Done! Run ~/webshots-grab/start.sh as the 'archiveteam' user to start grabbing." diff --git a/installers/webshots/webshots_debian.sh b/installers/webshots/webshots_debian.sh new file mode 100644 index 0000000..aa20d1c --- /dev/null +++ b/installers/webshots/webshots_debian.sh @@ -0,0 +1,24 @@ +#!/bin/bash +apt-get update +apt-get install -y lua50 liblua5.1-0-dev python python-setuptools git-core openssl libssl-dev bzip2 build-essential +if [ $? -ne 0 ]; then echo "[X] Installing dependencies failed. Exiting..."; exit 1; fi + +easy_install pip +if [ $? -ne 0 ]; then echo "[X] Installing pip failed. Exiting..."; exit 1; fi + +pip install seesaw +if [ $? -ne 0 ]; then echo "[X] Installing seesaw failed. Exiting..."; exit 1; fi + +useradd -m archiveteam +if [ $? -ne 0 ]; then echo "[X] Creating archiveteam user failed. Exiting..."; exit 1; fi + +wget -O /home/archiveteam/setup.sh http://cryto.net/projects/webshots/setup.sh +if [ $? -ne 0 ]; then echo "[X] Retrieving the user setup script failed. Exiting..."; exit 1; fi + +chown archiveteam:archiveteam /home/archiveteam/setup.sh +if [ $? -ne 0 ]; then echo "[X] Chowning the setup script failed. Exiting..."; exit 1; fi + +chmod +x /home/archiveteam/setup.sh +if [ $? -ne 0 ]; then echo "[X] Chmodding the setup script failed. Exiting..."; exit 1; fi + +su -c "/home/archiveteam/setup.sh" archiveteam From 7a4cf6b0aaa4a4602447f0cc717f27f4e052f607 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 13 Oct 2012 01:11:48 +0200 Subject: [PATCH 17/21] Include cURL dependency --- installers/webshots/webshots_debian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installers/webshots/webshots_debian.sh b/installers/webshots/webshots_debian.sh index aa20d1c..805cbc6 100644 --- a/installers/webshots/webshots_debian.sh +++ b/installers/webshots/webshots_debian.sh @@ -1,6 +1,6 @@ #!/bin/bash apt-get update -apt-get install -y lua50 liblua5.1-0-dev python python-setuptools git-core openssl libssl-dev bzip2 build-essential +apt-get install -y lua50 liblua5.1-0-dev python python-setuptools git-core openssl libssl-dev bzip2 build-essential curl if [ $? -ne 0 ]; then echo "[X] Installing dependencies failed. Exiting..."; exit 1; fi easy_install pip From 6900dcd75269eec6f59fc7afe8bf9b2f1b9650e8 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 30 Oct 2012 13:45:06 +0100 Subject: [PATCH 18/21] Add script for fixing Instiki HTML dump URLs for special characters --- tools/file-processing/fix_instiki_html_dump.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 tools/file-processing/fix_instiki_html_dump.sh diff --git a/tools/file-processing/fix_instiki_html_dump.sh b/tools/file-processing/fix_instiki_html_dump.sh new file mode 100644 index 0000000..2df998e --- /dev/null +++ b/tools/file-processing/fix_instiki_html_dump.sh @@ -0,0 +1 @@ +find ./ -name "*.xhtml" | xargs sed -i.bak -r -e "s/%([0-9A-Z]{2})/%25\\1/g" From d39acb36af46e3ec1e83a5ef78ff6b94d973f2bd Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 6 Nov 2012 02:12:50 +0100 Subject: [PATCH 19/21] Add script for setting up flask/lighttpd --- installers/flask-lighttpd.py | 135 +++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 installers/flask-lighttpd.py diff --git a/installers/flask-lighttpd.py b/installers/flask-lighttpd.py new file mode 100644 index 0000000..2fc6647 --- /dev/null +++ b/installers/flask-lighttpd.py @@ -0,0 +1,135 @@ +import os, stat, argparse + +template_fcgi = '''#!/usr/bin/env python +from flup.server.fcgi import WSGIServer +from app import app +from werkzeug.contrib.fixers import LighttpdCGIRootFix + +if __name__ == '__main__': + WSGIServer(LighttpdCGIRootFix(app)).run() +''' + +template_config = '''$HTTP["host"] =~ "%s" { + server.document-root = "%s" + + fastcgi.server = ("/" => + (( + "socket" => "%s", + "bin-path" => "%s", + "check-local" => "disable", + "max-procs" => 1 + )) + ) + + alias.url = ( + "/static" => "%s" + ) + + url.rewrite-once = ( + "^(/static($|/.*))$" => "$1" + ) +} +''' + +template_app = '''#!/usr/bin/env python +from flask import Flask +app = Flask(__name__) +''' + +template_run = '''from tornado.wsgi import WSGIContainer +from tornado.httpserver import HTTPServer +from tornado.ioloop import IOLoop +from app import app +app.debug = True + +http_server = HTTPServer(WSGIContainer(app)) +http_server.listen(1234) +IOLoop.instance().start() +''' + +parser = argparse.ArgumentParser(description='Creates and configures a new Flask project for a server running lighttpd.') + +parser.add_argument('--name', dest='name', action='store', required=True, + help='(required) lowercase alphanumeric name of the project') + +parser.add_argument('--hostname', dest='hostname', action='store', required=True, + help='(required) the hostname on which the project should be available') + +parser.add_argument('--config', dest='config', action='store', default="/etc/lighttpd/lighttpd.conf", + help='path to the lighttpd configuration file') + +parser.add_argument('--path', dest='path', action='store', default="/var/apps", + help='path to the apps directory') + +args = parser.parse_args() +options = vars(args) + +# Figure out the docroot +docroot = "%s/%s" % (options['path'], options['name']) + +# The lighttpd configuration file can only include relative paths, so we'll have to figure out how deep we are. +config_depth = len([x for x in options['config'].split("/") if x != ""]) - 1 +basepath = "../" * config_depth + +# Generate included configuration paths +relative_include = "%s/%s/lighttpd.conf" % (basepath[:-1], docroot[1:]) +absolute_include = "%s/lighttpd.conf" % docroot + +# Generate path to socket +socket_path = "/tmp/%s-fcgi.sock" % options['name'] + +# Generate path to main .fcgi file +fcgi_path = "%s/app.fcgi" % docroot + +# Generate path to static directory +static_path = "%s/static" % docroot + +print "Document root: %s" % docroot +print "Relative include path: %s" % relative_include +print "Absolute include path: %s" % absolute_include +print "Socket path: %s" % socket_path +print "FCGI path: %s" % fcgi_path +print "Static file path: %s" % static_path +print "Hostname: %s" % options['hostname'] +raw_input("Press enter to continue...") + +print "Creating document root..." +# Create document root +os.makedirs(docroot) + +print "Creating static file directory..." +# Create static file directory +os.makedirs(static_path) + +print "Creating main .fcgi file..." +# Create main .fcgi file +f = open(fcgi_path, "w") +f.write(template_fcgi) +f.close() + +print "Creating configuration include..." +# Create configuration include +f = open(absolute_include, "w") +f.write(template_config % (options['hostname'], docroot, socket_path, fcgi_path, static_path)) +f.close() + +print "Creating application template..." +f = open("%s/app.py" % docroot, "w") +f.write(template_app) +f.close() + +print "Creating run script..." +f = open("%s/run.py" % docroot, "w") +f.write(template_run) +f.close() + +print "Setting main .fcgi file as executable..." +os.chmod(fcgi_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) + +print "Appending include path to main lighttpd configuration file..." +# Append include to the main lighttpd configuration file +f = open(options['config'], "a") +f.write("\ninclude \"%s\"\n" % relative_include) +f.close() + +print "Done!" From 4047b9484d6320e5e3574265ec699b9b8d4742a6 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 8 Jan 2013 23:13:40 +0100 Subject: [PATCH 20/21] Add first version of ZippyDoc --- tools/documentation/test.html | 128 ++++++ .../documentation/zippydoc/documentation.html | 275 ++++++++++++ .../documentation/zippydoc/documentation.zpy | 325 +++++++++++++++ tools/documentation/zippydoc/example.html | 177 ++++++++ tools/documentation/zippydoc/example.zpy | 51 +++ tools/documentation/zippydoc/index.html | 167 ++++++++ tools/documentation/zippydoc/index.zpy | 62 +++ tools/documentation/zippydoc/zpy2html.py | 391 ++++++++++++++++++ 8 files changed, 1576 insertions(+) create mode 100644 tools/documentation/test.html create mode 100644 tools/documentation/zippydoc/documentation.html create mode 100644 tools/documentation/zippydoc/documentation.zpy create mode 100644 tools/documentation/zippydoc/example.html create mode 100644 tools/documentation/zippydoc/example.zpy create mode 100644 tools/documentation/zippydoc/index.html create mode 100644 tools/documentation/zippydoc/index.zpy create mode 100644 tools/documentation/zippydoc/zpy2html.py diff --git a/tools/documentation/test.html b/tools/documentation/test.html new file mode 100644 index 0000000..8ea984b --- /dev/null +++ b/tools/documentation/test.html @@ -0,0 +1,128 @@ +

Engine.Random

The Random library provides several functions to pick random numbers or items.

Table of contents

Engine.Random.Number(min, max, precision)
Selects a random number between `min` and `max`, excluding `max` itself.
min
The minimum number. Defaults to 0.
max
The maximum number (itself excluded). Defaults to 1.
precision
The precision; this is what the result will be rounded to. Defaults to 0.00000001.
Example: Get a whole number from 0 to 9
Code:
Engine.Random.Number(0, 10, 1);
Output:
7
Example: Get a one-tenth-precision number from 0 to 9.9
Code:
Engine.Random.Number(0, 10, 0.1);
Output:
3.7
Example: Get a one-fifth-precision number from 5 to 9.8:
Code:
Engine.Random.Number(5, 10, 0.2);
Output:
6.4
Engine.Random.Choose(item [, item [, item ...]])
Engine.Random.Choose(itemarray)
Selects a random item from the supplied items.
item
An item to choose from.
itemarray
An array of items to choose from.
Example: Select a random color from a list using multiple arguments
Code:
Engine.Random.Choose("blue", "green", "red", "yellow");
Output:
"green"
Example: Select a random day from a list using an array
Code:
var days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
+Engine.Random.Choose(days);
Output:
"Thursday"
Engine.Random.Pick(amount, item [, item [, item ...]])
Engine.Random.Pick(amount, itemarray)
Selects amount unique random items from the supplied items. Each item can only appear in the result once.
Important: The amount argument must always be equal to or higher than the amount of supplied items!
amount
The amount of items to select.
item
An item to choose from.
itemarray
An array of items to choose from.
Example: Select three random colors from a list using multiple arguments
Code:
Engine.Random.Pick(3, "red", "green", "blue", "yellow", "purple", "brown", "black", "white", "orange");
Output:
["blue", "orange", "red"]
Example: Select two vegetables from a list using an array
Code:
var vegetables = ["celery", "potato", "tomato", "coleslaw", "onion"];
+Engine.Random.Pick(2, vegetables);
Output:
["tomato", "onion"]
diff --git a/tools/documentation/zippydoc/documentation.html b/tools/documentation/zippydoc/documentation.html new file mode 100644 index 0000000..bbb35a5 --- /dev/null +++ b/tools/documentation/zippydoc/documentation.html @@ -0,0 +1,275 @@ + + + + + + + +

ZippyDoc format documentation

Download the ZippyDoc source of this page.
ZippyDoc is a compact, light-weight and code-oriented documentation markup language. It allows you to easily write documentation for your code or APIs, and batch-convert it to HTML.

Table of contents

  • Definition block A definition block is prefixed with a caret, and contains something along the...
  • Argument block An argument block shows a particular argument or parameter, and its explanation....
  • Example block An example block shows an example of the function you are documenting, with code...
  • Code block The code block is used in an example to show example code. It is prefixed with a...
  • Output block The output block is used to display sample output in an example. Just like the...
  • Exclamation block The exclamation block allows you to mark a block of text as "important". In the...
  • Header block A header block is a generic header to indicate the start of a new section. It is...
  • Text block A text block is any block that is not prefixed by a special character. It is...
  • Emphasized text Emphasized text is typically displayed as italic. You can emphasize text by...
  • Strong text Strong text is typically displayed as bold. You can make text strong by...
  • Internal references (hyperlinks) Internal references are hyperlinks that point to other documents in the same...
  • External references (hyperlinks) External references are hyperlinks just like the internal references, but they...
  • Fixed-width text Fixed-width text can be useful to indicate code elements or other things that...
  • Table of contents To insert a table of contents that is automatically generated from all...

Format overview

ZippyDoc is a paragraph-oriented format, much like Markdown. Each paragraph represents a "block" of something, and no linebreaks are used anywhere - to start on a new line, you simply start out with a new paragraph. A block is indicated by a specific prefix. Tabs (not spaces!) are used to indent blocks and indicate children of previous blocks. A new "paragraph" is started by having two or more successive line-endings - this basically comes down to at least one empty line inbetween paragraphs.
There is also some inline markup available, including emphasis, strong text, and hyperlinks to both other ZippyDoc documents and external locations.

Blocks

Several block types exist. Some of them have "continuation characters" that indicate the continuation of the previous block in a new paragraph, as opposed to starting a new block.
Definition block
A definition block is prefixed with a caret, and contains something along the lines of a function definition or type. In this particular document, it is used to denote the types of blocks and markup elements. Inline markup (such as the emphasis for arguments in the below example) is applied. These blocks are used for a table of contents if you choose to use one.
Example: Using a definition block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
You can also have an alternative notation for a definition. This can be useful if you have aliased functions, or multiple input forms. The alternative notation should be put on the same indentation as the original definition, but without a caret in front. The alternative notations (there is no limit) are shown separately in the table of contents if you choose to use one. You may indent the alternative notations with spaces to make the definition easier to read.
Example: Using a definition block
Code:
^ my_function(**argument**, **another_argument**)
+  alt_func(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
Argument block
An argument block shows a particular argument or parameter, and its explanation. The argument name is suffixed with a double colon (::), and the explanation follows on the next line. If so desired, the explanation can be indented with a tab - the tabs in front of the explanation will be eaten by the parser. The explanation can also consist of multiple lines - the newlines are preserved in the HTML version. Inline markup is applied only in the explanation.
Example: Using an argument block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
Example block
An example block shows an example of the function you are documenting, with code and output. The example block itself is prefixed with an @, and only specifies a title; you will use indented children of the Code and Output block variety to show respectively the code and the output for the example. These blocks will be explained later on. The title of your example block will be prefixed with "Example:" in the output automatically.
Example: Using an example block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ Some code goes here.
+
+		> Some output goes here.
Code block
The code block is used in an example to show example code. It is prefixed with a dollar sign ($), and all text following it will be show on the HTML page verbatim, without any further markup processing being done. It even allows you to display ZippyDoc formatting characters without having them interpreted, as is done on this page!
Example: Using a code block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> Some output goes here.
It is also possible to have a code block spanning multiple paragraphs, without each paragraph being broken up into a separate code block (as would normally happen if you just used the dollar sign). To do this, you can use two dollar signs at the start of the block. Note that after these two dollar signs, whitespace (except for spaces) is not eaten, meaning you can use tabs to indent further blocks of your code!
Example: Using a multi-paragraph code block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ if some_variable == True:
+
+		$$	my_function(42, "The answer to everything")
+
+		> Some output goes here.
Output block
The output block is used to display sample output in an example. Just like the code block, it is shown exactly as it originally was, without any further formatting applied. It is prefixed by a >, and like the code block it has a continuation character - in this case, that is >>.
Example: Using an output block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
Example: Using a multi-paragraph output block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
+
+		>> Did you know The answer to everything is 42?
Exclamation block
The exclamation block allows you to mark a block of text as "important". In the standard HTML layout, it will have a yellow-ish background, and will be prefixed with "Important!". It is prefixed with an exclamation mark (!). Inline markup is applied.
Example: Using an exclamation block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	! Only ever use this function with the number '42'!
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
Header block
A header block is a generic header to indicate the start of a new section. It is treated as a separate element, not as a "container". The header blocks in ZippyDoc work similarly to those in Markdown: they are prefixed by a hash (#), and the amount of hash characters defines what level of header it is.
Example: Using header blocks
Code:
# This is a level 1 (largest) header.
+
+## This is a level 2 header.
+
+...
+
+####### This is a level 7 (smallest) header.
Text block
A text block is any block that is not prefixed by a special character. It is shown as defined, with inline markup applied.

Inline markup

There are also various forms of inline markup that you can use in your documentation.
Emphasized text
Emphasized text is typically displayed as italic. You can emphasize text by enclosing it in two asterisks on each side.
Example: Emphasizing text
Code:
This is just some text, and **this part is emphasized**.
Strong text
Strong text is typically displayed as bold. You can make text strong by enclosing it in two underscores on each side.
Example: Making text strong
Code:
This is just some text, __and this part is strong__.
Internal references (hyperlinks)
Internal references are hyperlinks that point to other documents in the same documentation set. Depending on the export format (currently only HTML is supported), the appropriate extension is automatically appended. The paths should resemble the directory structure you are storing the ZippyDoc source files in. The target of the reference is enclosed in curly braces and prefixed with a >. If you wish to give the reference a friendly description, you can do so by appending it, enclosed in parentheses.
Example: Referencing another documentation page
Code:
You can also view the API documentation at {>api/index}.
Example: Referencing another documentation page with a friendly description
Code:
You can also view the {>api/index}(API documentation).
External references (hyperlinks)
External references are hyperlinks just like the internal references, but they refer to an external resources. The syntax is identical to that of internal references, except for the > disappearing. Note that external references are only picked up when the text enclosed in the braces is an actual URI of some sort.
You can also force an external reference to be created by prefixing the URI with <. This is useful when you want to for example link to a download relative to the current page.
Example: Referencing Google
Code:
You could also search {http://www.google.com/}.
Example: Referencing another documentation page with a friendly description
Code:
You could also search {http://www.google.com/}(Google).
Example: Referencing a relative file that is not a ZippyDoc document
Code:
You can download it by {<file.zip}(clicking here).
Fixed-width text
Fixed-width text can be useful to indicate code elements or other things that would benefit from being displayed in a terminal-like font. You can make text fixed-width by enclosing it in backticks.
Example: Fixed-width text
Code:
Now enter `./run.sh` into your terminal.

Special tags

Currently there is only one special tag. Special tags can be inserted anywhere in the document to insert a generated element.
Table of contents
To insert a table of contents that is automatically generated from all definition blocks on the page, simply insert {TOC} on the page where you want it to appear (it has to be in its own paragraph). Typically you will want to place it just below the main page header.
Every item in a table of contents will be followed by a snippet of text, that is grabbed from the first Text block for that definition. Alternative notations are shown after this description.
Example: Including a table of contents
Code:
# Sample documentation
+
+{TOC}
+
+^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	...

Full example

You can view a full example here of a ZippyDoc source file and its result.
+ + diff --git a/tools/documentation/zippydoc/documentation.zpy b/tools/documentation/zippydoc/documentation.zpy new file mode 100644 index 0000000..6441371 --- /dev/null +++ b/tools/documentation/zippydoc/documentation.zpy @@ -0,0 +1,325 @@ +# ZippyDoc format documentation + +{ Some output goes here. + +^ Code block + + The code block is used in an example to show example code. It is prefixed with a dollar sign ($), and all text following it will be show on the HTML page verbatim, without + any further markup processing being done. It even allows you to display ZippyDoc formatting characters without having them interpreted, as is done on this page! + + @ Using a code block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > Some output goes here. + + It is also possible to have a code block spanning multiple paragraphs, without each paragraph being broken up into a separate code block (as would normally happen if you + just used the dollar sign). To do this, you can use two dollar signs at the start of the block. Note that after these two dollar signs, whitespace (except for spaces) is + **not** eaten, meaning you can use tabs to indent further blocks of your code! + + @ Using a multi-paragraph code block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ if some_variable == True: + + $$ $$ my_function(42, "The answer to everything") + + $$ > Some output goes here. + +^ Output block + + The output block is used to display sample output in an example. Just like the code block, it is shown exactly as it originally was, without any further formatting applied. + It is prefixed by a >, and like the code block it has a continuation character - in this case, that is >>. + + @ Using an output block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + @ Using a multi-paragraph output block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + $$ >> Did you know The answer to everything is 42? + +^ Exclamation block + + The exclamation block allows you to mark a block of text as "important". In the standard HTML layout, it will have a yellow-ish background, and will be prefixed + with "__Important!__". It is prefixed with an exclamation mark (!). Inline markup is applied. + + @ Using an exclamation block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ ! Only ever use this function with the number '42'! + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + +^ Header block + + A header block is a generic header to indicate the start of a new section. It is treated as a separate element, not as a "container". The header blocks in ZippyDoc + work similarly to those in Markdown: they are prefixed by a hash (#), and the amount of hash characters defines what level of header it is. + + @ Using header blocks + + $ # This is a level 1 (largest) header. + + $$ ## This is a level 2 header. + + $$ ... + + $$ ####### This is a level 7 (smallest) header. + + +^ Text block + + A text block is any block that is not prefixed by a special character. It is shown as defined, with inline markup applied. + + +## Inline markup + +There are also various forms of inline markup that you can use in your documentation. + +^ Emphasized text + + Emphasized text is typically displayed as italic. You can emphasize text by enclosing it in two asterisks on each side. + + @ Emphasizing text + + $ This is just some text, and **this part is emphasized**. + +^ Strong text + + Strong text is typically displayed as bold. You can make text strong by enclosing it in two underscores on each side. + + @ Making text strong + + $ This is just some text, __and this part is strong__. + +^ Internal references (hyperlinks) + + Internal references are hyperlinks that point to other documents in the same documentation set. Depending on the export format (currently only HTML is supported), + the appropriate extension is automatically appended. The paths should resemble the directory structure you are storing the ZippyDoc source files in. The target + of the reference is enclosed in curly braces and prefixed with a >. If you wish to give the reference a friendly description, you can do so by appending it, + enclosed in parentheses. + + @ Referencing another documentation page + + $ You can also view the API documentation at {>api/index}. + + @ Referencing another documentation page with a friendly description + + $ You can also view the {>api/index}(API documentation). + + +^ External references (hyperlinks) + + External references are hyperlinks just like the internal references, but they refer to an external resources. The syntax is identical to that of internal references, + except for the > disappearing. Note that external references are only picked up when the text enclosed in the braces is an actual URI of some sort. + + You can also force an external reference to be created by prefixing the URI with <. This is useful when you want to for example link to a download relative to the current + page. + + @ Referencing Google + + $ You could also search {http://www.google.com/}. + + @ Referencing another documentation page with a friendly description + + $ You could also search {http://www.google.com/}(Google). + + @ Referencing a relative file that is not a ZippyDoc document + + $ You can download it by {example}(view a full example here) of a ZippyDoc source file and its result. diff --git a/tools/documentation/zippydoc/example.html b/tools/documentation/zippydoc/example.html new file mode 100644 index 0000000..458d331 --- /dev/null +++ b/tools/documentation/zippydoc/example.html @@ -0,0 +1,177 @@ + + + + + + + +

A complete example

Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here. `Also some mono-spaced text.`
+
+	! Only ever use this function with the number '42'!
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+		We can also add an {>documentation}(internal link) and an {http://google.com/}(external link).
+
+	@ Using this function
+
+		$ if some_variable == True:
+
+		$$	my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
+
+		>>	Did you know The answer to everything is 42?

Result

my_function(argument, another_argument)
Some kind of text describing the function goes here. Also some mono-spaced text.
Important: Only ever use this function with the number '42'!
argument
This is the first argument to this example function.
another_argument
This is the second argument to this example function. As you can see, it's possible to split the explanation over multiple lines as well. We can also add an internal link and an external link.
Example: Using this function
Code:
if some_variable == True:
+
+	my_function(42, "The answer to everything")
Output:
The answer to everything is 42!
+
+	Did you know The answer to everything is 42?
+ + diff --git a/tools/documentation/zippydoc/example.zpy b/tools/documentation/zippydoc/example.zpy new file mode 100644 index 0000000..bd4110e --- /dev/null +++ b/tools/documentation/zippydoc/example.zpy @@ -0,0 +1,51 @@ +# A complete example + +$ ^ my_function(**argument**, **another_argument**) + +$$ Some kind of text describing the function goes here. `Also some mono-spaced text.` + +$$ ! Only ever use this function with the number '42'! + +$$ argument:: + This is the first argument to this example function. + +$$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + We can also add an {>documentation}(internal link) and an {http://google.com/}(external link). + +$$ @ Using this function + +$$ $ if some_variable == True: + +$$ $$ my_function(42, "The answer to everything") + +$$ > The answer to everything is 42! + +$$ >> Did you know The answer to everything is 42? + +## Result + +^ my_function(**argument**, **another_argument**) + + Some kind of text describing the function goes here. `Also some mono-spaced text.` + + ! Only ever use this function with the number '42'! + + argument:: + This is the first argument to this example function. + + another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + We can also add an {>documentation}(internal link) and an {http://google.com/}(external link). + + @ Using this function + + $ if some_variable == True: + + $$ my_function(42, "The answer to everything") + + > The answer to everything is 42! + + >> Did you know The answer to everything is 42? diff --git a/tools/documentation/zippydoc/index.html b/tools/documentation/zippydoc/index.html new file mode 100644 index 0000000..04ffd4d --- /dev/null +++ b/tools/documentation/zippydoc/index.html @@ -0,0 +1,167 @@ + + + + + + + +

ZippyDoc

Hi, this is the website of ZippyDoc, a compact, light-weight and human-readable format for documenting code, APIs, and other things, that can be easily converted to HTML.
It is designed primarily to be simple to use (unlike complex markup languages like reStructuredText), and very code-oriented (unlike other simple markup languages like Markdown). You will probably learn the entire syntax in about 10 minutes.
ZippyDoc (both the format and the parser) are licensed under the WTFPL, meaning you can basically do with it whatever you want, and reuse it in any fashion you see fit. I hope it will help you write nicer, easier, and more complete documentation!
While ZippyDoc is technically intended for documentation, I decided to whip up a simple index page in ZippyDoc as well - you're looking at it! :)

What does the ZippyDoc format look like?

Code:
^ my_function(argument1, argument2)
+
+	! This is just an example!
+
+	This is a function.
+
+	argument1::
+		This is the first argument.
+
+	argument2::
+		This is the second argument.
+
+	@ How to call my_function
+
+		$ my_function("ZippyDoc", "awesome")
+
+		> "ZippyDoc is awesome!"
Result:
my_function(argument1, argument2)
Important: This is just an example!
This is a function.
argument1
This is the first argument.
argument2
This is the second argument.
Example: How to call my_function
Code:
my_function("ZippyDoc", "awesome")
Output:
"ZippyDoc is awesome!"

Documentation

The documentation for ZippyDoc can be found here.

Downloading ZippyDoc

ZippyDoc is still in a pretty messy stage, but it should already work reliably according to the current documentation. GitHub repository is coming soon, until that time you can download the conversion script here. It's a Python script, so you'll need a Python interpreter of some sort. No dependencies are necessary, it only uses standard library functionality. Simply run it with all files you wish to convert as arguments, and it will convert each of them into a file with the same name, but a .html extension instead of the original extension. It's strongly recommended to name your ZippyDoc source files with the .zpy extension.
+ + diff --git a/tools/documentation/zippydoc/index.zpy b/tools/documentation/zippydoc/index.zpy new file mode 100644 index 0000000..436d40e --- /dev/null +++ b/tools/documentation/zippydoc/index.zpy @@ -0,0 +1,62 @@ +# ZippyDoc + +Hi, this is the website of ZippyDoc, a compact, light-weight and human-readable format for documenting code, APIs, and other things, that can be easily converted to HTML. + +It is designed primarily to be simple to use (unlike complex markup languages like reStructuredText), and very code-oriented (unlike other simple markup languages like Markdown). +You will probably learn the entire syntax in about 10 minutes. + +ZippyDoc (both the format and the parser) are licensed under the {http://www.wtfpl.net/}(WTFPL), meaning you can basically do with it whatever you want, and reuse it in any +fashion you see fit. I hope it will help you write nicer, easier, and more complete documentation! + +While ZippyDoc is technically intended for documentation, I decided to whip up a simple index page in ZippyDoc as well - you're looking at it! :) + +## What does the ZippyDoc format look like? + +$ ^ my_function(argument1, argument2) + +$$ ! This is just an example! + +$$ This is a function. + +$$ argument1:: + This is the first argument. + +$$ argument2:: + This is the second argument. + +$$ @ How to call my_function + +$$ $ my_function("ZippyDoc", "awesome") + +$$ > "ZippyDoc is awesome!" + +####### Result: + +^ my_function(argument1, argument2) + + ! This is just an example! + + This is a function. + + argument1:: + This is the first argument. + + argument2:: + This is the second argument. + + @ How to call my_function + + $ my_function("ZippyDoc", "awesome") + + > "ZippyDoc is awesome!" + +## Documentation + +The documentation for ZippyDoc can be found {>documentation}(here). + +## Downloading ZippyDoc + +ZippyDoc is still in a pretty messy stage, but it should already work reliably according to the current documentation. GitHub repository is coming soon, until that time you can +{ + + + + + + %s + + +""" + +class TreeLevel: + def __init__(self, indentation, data): + self.elements = [] + self.indentation = indentation + self.data = data + + def add(self, element): + self.elements.append(element) + + def output(self): + return self.render() + + def render_children(self): + child_output = "" + + for child in self.elements: + child_output += child.output() + + return '
%s
' % child_output + + def process_inline_markup(self, text): + text = re.sub("`([^`]+)`", '\\1', text) # Emphasized + text = re.sub("\*\*([^*]+)\*\*", "\\1", text) # Emphasized + text = re.sub("__([^_]+)__", "\\1", text) # Strong + text = re.sub("{>([^}]+)}\(([^)]+)\)", '\\2', text) # Hyperlink with text + text = re.sub("{>([^}]+)}", '\\1', text) # Hyperlink + text = re.sub("{([^}]+:[^}]+)}\(([^)]+)\)", '\\2', text) # External hyperlink with text + text = re.sub("{([^}]+:[^}]+)}", '\\1', text) # External hyperlink + text = re.sub("{<([^}]+)}\(([^)]+)\)", '\\2', text) # Forced external hyperlink with text + text = re.sub("{<([^}]+)}", '\\1', text) # Forced external hyperlink + + return text + + def fix_preformatted(self, text): + return text.replace("<", "<").replace(">", ">") + + def clear_markup(self, text): + return re.sub("\*\*([^*]+)\*\*", "\\1", text) + + def render(self): + return self.render_children() + +class Example(TreeLevel): + def render(self): + return '
Example: %s %s
' % (self.data, self.render_children()) + +class Code(TreeLevel): + def render(self): + return 'Code:
%s
' % self.fix_preformatted(self.data) + +class Output(TreeLevel): + def render(self): + return 'Output:
%s
' % self.fix_preformatted(self.data) + +class Definition(TreeLevel): + def get_anchor(self): + first = self.clear_markup(self.data.splitlines()[0]) + anchor = first.replace("...", "") + anchor = anchor.replace(".", "_") + anchor = re.sub("[^a-zA-Z0-9_]", "", anchor) + return anchor + + def get_description(self): + for element in self.elements: + if element.__class__.__name__ == "Text": + data = self.process_inline_markup(element.data) + + if len(data) > 80: + matches = re.match("^(.{0,80})\W", data) + return matches.group(1) + "..." + else: + return data + + return "" + + def render(self): + return '
%s %s
' % (self.get_anchor(), self.process_inline_markup(self.data.replace("\n", "
")), self.render_children()) + +class Exclamation(TreeLevel): + def render(self): + return '
Important: %s
' % self.process_inline_markup(self.data) + +class Argument(TreeLevel): + def __init__(self, indentation, data, argname): + self.elements = [] + self.indentation = indentation + self.argname = argname + self.data = data + + def render(self): + return '
%s
%s
' % (self.argname, self.process_inline_markup(self.data)) + +class Header(TreeLevel): + def __init__(self, indentation, data, depth): + self.elements = [] + self.indentation = indentation + self.depth = depth + self.data = data + + def render(self): + if self.depth <= 7: + title_type = "h%d" % self.depth + else: + title_type = "h7" + + return "<%s>%s" % (title_type, self.data, title_type) + +class Text(TreeLevel): + def render(self): + return '
%s
' % self.process_inline_markup(self.data) + +class Index(TreeLevel): + def render(self): + global toc_items + + rendered = "" + + for item in toc_items: + forms = item.data.splitlines() + first = self.clear_markup(forms[0]) + + if len(forms) > 1: + rest = '(also: ' + ', '.join(self.clear_markup(form) for form in forms[1:]) + ")" + else: + rest = "" + + anchor = item.get_anchor() + description = item.get_description() + rendered += '
  • %s %s %s
    • ' % (anchor, first, description, rest) + + return '

    Table of contents

      %s
    ' % rendered + +for zpy in files: + destination = os.path.splitext(zpy)[0] + ".html" + + f = open(zpy, "r") + data = f.read() + f.close() + + paragraphs = re.split("\s*\n\s*\n", data) + toc_items = [] + current_level = 0 + current_paragraph = 0 + current_elements = {0: TreeLevel(0, "root")} + + for paragraph in paragraphs: + if paragraph.strip() == "": + continue + + current_paragraph += 1 + indentation = len(paragraph) - len(paragraph.lstrip("\t")) + 1 + + if indentation > current_level + 1: + raise Exception("Invalid indentation found in paragraph %d" % current_paragraph) + + element_type = TreeLevel + start = indentation - 1 + + lines = [line[start:] for line in paragraph.splitlines()] + + if lines[0].startswith("#"): + element_type = Header + depth = len(lines[0]) - len(lines[0].lstrip("#")) + lines[0] = lines[0].lstrip("# ") + data = " ".join(lines) + elif lines[0].startswith("^"): + element_type = Definition + lines[0] = lines[0].lstrip("^ ") + data = "\n".join(lines) + elif lines[0].startswith("@"): + element_type = Example + lines[0] = lines[0].lstrip("@ ") + data = " ".join(lines) + elif lines[0].startswith("$$") and current_elements[current_level].__class__.__name__ == "Code": + current_elements[current_level].data += "\n\n" + "\n".join(lines).lstrip("$ ") + continue + elif lines[0].startswith("$"): + element_type = Code + lines[0] = lines[0].lstrip("$ ") + data = "\n".join(lines) + elif lines[0].startswith(">>") and current_elements[current_level].__class__.__name__ == "Output": + current_elements[current_level].data += "\n\n" + "\n".join(lines).lstrip("> ") + continue + elif lines[0].startswith(">"): + element_type = Output + lines[0] = lines[0].lstrip("> ") + data = "\n".join(lines) + elif lines[0].startswith("!"): + element_type = Exclamation + lines[0] = lines[0].lstrip("! ") + data = " ".join(lines) + elif re.match(".*::\s*$", lines[0]): + element_type = Argument + argname = lines[0][:-2] + data = " ".join(line.lstrip() for line in lines[1:]) + elif lines[0].strip() == "{TOC}": + element_type = Index + data = "" + else: + element_type = Text + data = " ".join(lines) + + #print "Found element of type %s at indentation %d with data %s" % (element_type.__name__, indentation, data[:80]) + + if element_type.__name__ == "Header": + element = Header(indentation, data, depth) + elif element_type.__name__ == "Argument": + element = Argument(indentation, data, argname) + else: + element = element_type(indentation, data) + + if element_type.__name__ == "Definition": + toc_items.append(element) + + current_elements[indentation - 1].add(element) + + current_level = indentation + current_elements[current_level] = element + + rendered = template % (current_elements[0].output()) + + f = open(destination, "w") + f.write(rendered) + f.close() + + print "Rendered %s" % destination From 8920251d2c697da28564645e22d44f7ef99c6e72 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Tue, 8 Jan 2013 23:14:26 +0100 Subject: [PATCH 21/21] Add unfinished speedtest.py script --- tools/server-management/speedtest.py | 132 +++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 tools/server-management/speedtest.py diff --git a/tools/server-management/speedtest.py b/tools/server-management/speedtest.py new file mode 100755 index 0000000..c952d75 --- /dev/null +++ b/tools/server-management/speedtest.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +import urllib2, time, urllib +from lxml import etree + +def avg(inp): + return (reduce(lambda x, y: x + y, inp) / len(inp)) + +referer = "http://c.speedtest.net/flash/speedtest.swf?v=316125" +num = 1353968072002 +config_url = "http://speedtest.net/speedtest-config.php?x=%d" % num +server_url = "http://speedtest.net/speedtest-servers.php?x=%d" % num +download_path = "/random%dx%d.jpg?x=%d&y=%d" +upload_path = "/upload.php?x=%d" % num +latency_path = "/latency.txt?x=%d" % num +sizes = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000] + +servers = [] +server_count = 0 + +# First, get our own details. +result = urllib2.urlopen(config_url) + +for event, element in etree.iterparse(result): + if element.tag == "client": + my_ip = element.get("ip") + my_isp = element.get("isp") + my_latitude = float(element.get("lat")) + my_longitude = float(element.get("lon")) + dl_average = float(element.get("ispdlavg")) + ul_average = float(element.get("ispulavg")) + + print "You are %s (%s), with latitude %f and longitude %f. Your ISPs average download speed is %.3f MB/sec, and their average upload speed is %.3f MB/sec." % (my_ip, my_isp, my_latitude, my_longitude, dl_average / 8 / 1024 / 1024, ul_average / 8 / 1024 / 1024) + +print "Retrieving server list...", + +# Retrieve and parse list of servers. +result = urllib2.urlopen(server_url) + +for event, element in etree.iterparse(result): + if element.tag == "server": + hostname = element.get("url").replace("/upload.php", "") + latitude = float(element.get("lat")) + longitude = float(element.get("lon")) + location = element.get("name") + country = element.get("country") + sponsor = element.get("sponsor") + + distance = abs(my_latitude - latitude) + abs(my_longitude - longitude) + + servers.append((distance, hostname, latitude, longitude, location, country, sponsor)) + server_count += 1 + + print "\rRetrieving server list... %d servers found so far" % server_count, + +# Sort the server list by distance. +servers = sorted(servers, key=lambda server: server[0]) + +print "\nFound 5 closest servers. Determining optimal latency..." + +fastest_server = () +fastest_latency = 0 + +for server in servers[:5]: + # Take 3 samples of each server. + latencies = [] + + for i in xrange(0, 3): + request = urllib2.Request(server[1] + latency_path) + + start_time = time.time() + urllib2.urlopen(request) + end_time = time.time() + + latencies.append((end_time - start_time) * 1000) + + latency = avg(latencies) + + if fastest_latency == 0 or latency < fastest_latency: + fastest_latency = latency + fastest_server = server + + print "\rFastest server so far is '%s' with %dms ping... (%s)" % (fastest_server[6], fastest_latency, latencies), + +print "\nTarget server is '%s'. Testing download speed..." % fastest_server[6] + +latency = fastest_latency + +size = 0 + +while size < 8: + # Take 3 samples + speeds = [] + times = [] + + for i in xrange(0, 3): + target_file = download_path % (sizes[size], sizes[size], num, i + 1) + request = urllib2.urlopen(fastest_server[1] + target_file) + filesize = int(request.info()['Content-Length']) + block_size = 4096 + r = 0 + start_time = time.time() + + while r < filesize: + request.read(block_size) + r += block_size + speed = r / (time.time() - start_time) + print "\rSize %d, attempt %d... %.3f MB/sec" % (sizes[size], i + 1, speed / 1024 / 1024), + + end_time = time.time() + + speeds.append(speed) + times.append(end_time - start_time) + + print "" + request.close() + + if avg(times) < 4: + size += 1 + else: + break + +# Take result from last speedtest as authorative. +if size >= 8: + size = 7 + +download_speed = avg(speeds) + +#print "Average speed, sample size %d, is %.3f MB/sec" % (sizes[size], download_speed / 1024 / 1024) + +print "Latency: %dms\tDownload speed: %.3f MB/sec" % (latency, download_speed / 1024 / 1024) +print "NOTE: Due to function call overhead, the latency is, at best, an estimation."