diff --git a/installers/flask-lighttpd.py b/installers/flask-lighttpd.py new file mode 100644 index 0000000..2fc6647 --- /dev/null +++ b/installers/flask-lighttpd.py @@ -0,0 +1,135 @@ +import os, stat, argparse + +template_fcgi = '''#!/usr/bin/env python +from flup.server.fcgi import WSGIServer +from app import app +from werkzeug.contrib.fixers import LighttpdCGIRootFix + +if __name__ == '__main__': + WSGIServer(LighttpdCGIRootFix(app)).run() +''' + +template_config = '''$HTTP["host"] =~ "%s" { + server.document-root = "%s" + + fastcgi.server = ("/" => + (( + "socket" => "%s", + "bin-path" => "%s", + "check-local" => "disable", + "max-procs" => 1 + )) + ) + + alias.url = ( + "/static" => "%s" + ) + + url.rewrite-once = ( + "^(/static($|/.*))$" => "$1" + ) +} +''' + +template_app = '''#!/usr/bin/env python +from flask import Flask +app = Flask(__name__) +''' + +template_run = '''from tornado.wsgi import WSGIContainer +from tornado.httpserver import HTTPServer +from tornado.ioloop import IOLoop +from app import app +app.debug = True + +http_server = HTTPServer(WSGIContainer(app)) +http_server.listen(1234) +IOLoop.instance().start() +''' + +parser = argparse.ArgumentParser(description='Creates and configures a new Flask project for a server running lighttpd.') + +parser.add_argument('--name', dest='name', action='store', required=True, + help='(required) lowercase alphanumeric name of the project') + +parser.add_argument('--hostname', dest='hostname', action='store', required=True, + help='(required) the hostname on which the project should be available') + +parser.add_argument('--config', dest='config', action='store', default="/etc/lighttpd/lighttpd.conf", + help='path to the lighttpd configuration file') + +parser.add_argument('--path', dest='path', action='store', default="/var/apps", + help='path to the apps directory') + +args = parser.parse_args() +options = vars(args) + +# Figure out the docroot +docroot = "%s/%s" % (options['path'], options['name']) + +# The lighttpd configuration file can only include relative paths, so we'll have to figure out how deep we are. +config_depth = len([x for x in options['config'].split("/") if x != ""]) - 1 +basepath = "../" * config_depth + +# Generate included configuration paths +relative_include = "%s/%s/lighttpd.conf" % (basepath[:-1], docroot[1:]) +absolute_include = "%s/lighttpd.conf" % docroot + +# Generate path to socket +socket_path = "/tmp/%s-fcgi.sock" % options['name'] + +# Generate path to main .fcgi file +fcgi_path = "%s/app.fcgi" % docroot + +# Generate path to static directory +static_path = "%s/static" % docroot + +print "Document root: %s" % docroot +print "Relative include path: %s" % relative_include +print "Absolute include path: %s" % absolute_include +print "Socket path: %s" % socket_path +print "FCGI path: %s" % fcgi_path +print "Static file path: %s" % static_path +print "Hostname: %s" % options['hostname'] +raw_input("Press enter to continue...") + +print "Creating document root..." +# Create document root +os.makedirs(docroot) + +print "Creating static file directory..." +# Create static file directory +os.makedirs(static_path) + +print "Creating main .fcgi file..." +# Create main .fcgi file +f = open(fcgi_path, "w") +f.write(template_fcgi) +f.close() + +print "Creating configuration include..." +# Create configuration include +f = open(absolute_include, "w") +f.write(template_config % (options['hostname'], docroot, socket_path, fcgi_path, static_path)) +f.close() + +print "Creating application template..." +f = open("%s/app.py" % docroot, "w") +f.write(template_app) +f.close() + +print "Creating run script..." +f = open("%s/run.py" % docroot, "w") +f.write(template_run) +f.close() + +print "Setting main .fcgi file as executable..." +os.chmod(fcgi_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) + +print "Appending include path to main lighttpd configuration file..." +# Append include to the main lighttpd configuration file +f = open(options['config'], "a") +f.write("\ninclude \"%s\"\n" % relative_include) +f.close() + +print "Done!" diff --git a/installers/webshots/INSTALL b/installers/webshots/INSTALL new file mode 100644 index 0000000..3c7fa4f --- /dev/null +++ b/installers/webshots/INSTALL @@ -0,0 +1 @@ +wget http://cryto.net/projects/webshots/webshots_debian.sh && chmod +x webshots_debian.sh && ./webshots_debian.sh diff --git a/installers/webshots/setup.sh b/installers/webshots/setup.sh new file mode 100644 index 0000000..962f2cb --- /dev/null +++ b/installers/webshots/setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd ~ +git clone git://github.com/ArchiveTeam/webshots-grab.git +if [ $? -ne 0 ]; then echo "[X] Retrieving the grabber code failed. Exiting..."; exit 1; fi + +cd webshots-grab +./get-wget-lua.sh +if [ $? -ne 0 ]; then echo "[X] Compiling wget-lua failed. Exiting..."; exit 1; fi + +echo "What username would you like to use? " +read USERNAME +echo "How many threads? (start out with 2 or so) " +read THREADCOUNT +echo "run-pipeline --concurrent $THREADCOUNT ~/webshots-grab/pipeline.py $USERNAME" > ./start.sh +chmod +x ./start.sh +echo "Done! Run ~/webshots-grab/start.sh as the 'archiveteam' user to start grabbing." diff --git a/installers/webshots/webshots_debian.sh b/installers/webshots/webshots_debian.sh new file mode 100644 index 0000000..805cbc6 --- /dev/null +++ b/installers/webshots/webshots_debian.sh @@ -0,0 +1,24 @@ +#!/bin/bash +apt-get update +apt-get install -y lua50 liblua5.1-0-dev python python-setuptools git-core openssl libssl-dev bzip2 build-essential curl +if [ $? -ne 0 ]; then echo "[X] Installing dependencies failed. Exiting..."; exit 1; fi + +easy_install pip +if [ $? -ne 0 ]; then echo "[X] Installing pip failed. Exiting..."; exit 1; fi + +pip install seesaw +if [ $? -ne 0 ]; then echo "[X] Installing seesaw failed. Exiting..."; exit 1; fi + +useradd -m archiveteam +if [ $? -ne 0 ]; then echo "[X] Creating archiveteam user failed. Exiting..."; exit 1; fi + +wget -O /home/archiveteam/setup.sh http://cryto.net/projects/webshots/setup.sh +if [ $? -ne 0 ]; then echo "[X] Retrieving the user setup script failed. Exiting..."; exit 1; fi + +chown archiveteam:archiveteam /home/archiveteam/setup.sh +if [ $? -ne 0 ]; then echo "[X] Chowning the setup script failed. Exiting..."; exit 1; fi + +chmod +x /home/archiveteam/setup.sh +if [ $? -ne 0 ]; then echo "[X] Chmodding the setup script failed. Exiting..."; exit 1; fi + +su -c "/home/archiveteam/setup.sh" archiveteam diff --git a/snippets/php/path_root_validation.php b/snippets/php/path_root_validation.php new file mode 100644 index 0000000..d3c2533 --- /dev/null +++ b/snippets/php/path_root_validation.php @@ -0,0 +1,149 @@ +ParsePath($path); + + if($parsed_path != null) + { + $this->path = $parsed_path; + } + else + { + throw new Exception("Invalid path specified"); + } + } + + public function ValidatePath($root) + { + $root_path = $this->ParsePath($root); + + if($root_path != null) + { + if($this->RelativeDepth($root) >= 0) + { + $root_stack = explode("/", $root_path); + $path_stack = explode("/", $this->path); + + for($i = 0; $i < count($root_stack); $i++) + { + if($root_stack[$i] != $path_stack[$i]) + { + return false; + } + } + + return true; + } + else + { + return false; + } + } + else + { + throw new Exception("Specified root path is invalid."); + } + } + + public function RelativeDepth($root) + { + $root_length = substr_count($this->ParsePath($root), "/"); + $path_length = substr_count($this->path, "/"); + + return $path_length - $root_length; + } + + private function RemoveTrailingSlash($path) + { + if(substr($path, strlen($path) - 1) == "/") + { + return substr($path, 0, strlen($path) - 1); + } + else + { + return $path; + } + } + + private function ParsePath($path) + { + /* We use a custom function for this since we just want to resolve the path no matter what, + * and the realpath() function will return false if the path either doesn't exist or is not + * accessible. */ + + $path = $this->RemoveTrailingSlash($path); + + if(substr($path, 0, 1) == "/") + { + /* Absolute path */ + return $path; + } + else + { + $path_elements = explode("/", $path); + + if(substr($path, 0, 1) == "~") + { + /* Home directory path */ + if(!empty($_SERVER['home'])) + { + $homedir = $_SERVER['home']; + } + elseif(getenv("HOME") != null) + { + $homedir = getenv("HOME"); + } + elseif(function_exists("posix_getuid") && function_exists("posix_getpwuid")) + { + $userinfo = posix_getpwuid(posix_getuid()); + $homedir = $userinfo['dir']; + } + else + { + throw new Exception("Could not find a way to get the home directory of the current user."); + } + + $homedir = $this->RemoveTrailingSlash($homedir); + + $stack = explode("/", $homedir); + array_shift($path_elements); + } + else + { + /* Relative path */ + $basepath = $this->RemoveTrailingSlash(getcwd()); + $stack = explode("/", $basepath); + } + + foreach($path_elements as $element) + { + if($element == ".") + { + /* Ignore */ + } + elseif($element == "..") + { + /* Go up one directory */ + if(count($stack) > 1) + { + array_pop($stack); + } + else + { + /* There are no elements left to pop, this is an invalid path. */ + return null; + } + } + else + { + /* Append to path */ + $stack[] = $element; + } + } + + return implode("/", $stack); + } + } +} diff --git a/tools/documentation/test.html b/tools/documentation/test.html new file mode 100644 index 0000000..8ea984b --- /dev/null +++ b/tools/documentation/test.html @@ -0,0 +1,128 @@ +

Engine.Random

The Random library provides several functions to pick random numbers or items.

Table of contents

Engine.Random.Number(min, max, precision)
Selects a random number between `min` and `max`, excluding `max` itself.
min
The minimum number. Defaults to 0.
max
The maximum number (itself excluded). Defaults to 1.
precision
The precision; this is what the result will be rounded to. Defaults to 0.00000001.
Example: Get a whole number from 0 to 9
Code:
Engine.Random.Number(0, 10, 1);
Output:
7
Example: Get a one-tenth-precision number from 0 to 9.9
Code:
Engine.Random.Number(0, 10, 0.1);
Output:
3.7
Example: Get a one-fifth-precision number from 5 to 9.8:
Code:
Engine.Random.Number(5, 10, 0.2);
Output:
6.4
Engine.Random.Choose(item [, item [, item ...]])
Engine.Random.Choose(itemarray)
Selects a random item from the supplied items.
item
An item to choose from.
itemarray
An array of items to choose from.
Example: Select a random color from a list using multiple arguments
Code:
Engine.Random.Choose("blue", "green", "red", "yellow");
Output:
"green"
Example: Select a random day from a list using an array
Code:
var days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
+Engine.Random.Choose(days);
Output:
"Thursday"
Engine.Random.Pick(amount, item [, item [, item ...]])
Engine.Random.Pick(amount, itemarray)
Selects amount unique random items from the supplied items. Each item can only appear in the result once.
Important: The amount argument must always be equal to or higher than the amount of supplied items!
amount
The amount of items to select.
item
An item to choose from.
itemarray
An array of items to choose from.
Example: Select three random colors from a list using multiple arguments
Code:
Engine.Random.Pick(3, "red", "green", "blue", "yellow", "purple", "brown", "black", "white", "orange");
Output:
["blue", "orange", "red"]
Example: Select two vegetables from a list using an array
Code:
var vegetables = ["celery", "potato", "tomato", "coleslaw", "onion"];
+Engine.Random.Pick(2, vegetables);
Output:
["tomato", "onion"]
diff --git a/tools/documentation/zippydoc/documentation.html b/tools/documentation/zippydoc/documentation.html new file mode 100644 index 0000000..bbb35a5 --- /dev/null +++ b/tools/documentation/zippydoc/documentation.html @@ -0,0 +1,275 @@ + + + + + + + +

ZippyDoc format documentation

Download the ZippyDoc source of this page.
ZippyDoc is a compact, light-weight and code-oriented documentation markup language. It allows you to easily write documentation for your code or APIs, and batch-convert it to HTML.

Table of contents

Format overview

ZippyDoc is a paragraph-oriented format, much like Markdown. Each paragraph represents a "block" of something, and no linebreaks are used anywhere - to start on a new line, you simply start out with a new paragraph. A block is indicated by a specific prefix. Tabs (not spaces!) are used to indent blocks and indicate children of previous blocks. A new "paragraph" is started by having two or more successive line-endings - this basically comes down to at least one empty line inbetween paragraphs.
There is also some inline markup available, including emphasis, strong text, and hyperlinks to both other ZippyDoc documents and external locations.

Blocks

Several block types exist. Some of them have "continuation characters" that indicate the continuation of the previous block in a new paragraph, as opposed to starting a new block.
Definition block
A definition block is prefixed with a caret, and contains something along the lines of a function definition or type. In this particular document, it is used to denote the types of blocks and markup elements. Inline markup (such as the emphasis for arguments in the below example) is applied. These blocks are used for a table of contents if you choose to use one.
Example: Using a definition block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
You can also have an alternative notation for a definition. This can be useful if you have aliased functions, or multiple input forms. The alternative notation should be put on the same indentation as the original definition, but without a caret in front. The alternative notations (there is no limit) are shown separately in the table of contents if you choose to use one. You may indent the alternative notations with spaces to make the definition easier to read.
Example: Using a definition block
Code:
^ my_function(**argument**, **another_argument**)
+  alt_func(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
Argument block
An argument block shows a particular argument or parameter, and its explanation. The argument name is suffixed with a double colon (::), and the explanation follows on the next line. If so desired, the explanation can be indented with a tab - the tabs in front of the explanation will be eaten by the parser. The explanation can also consist of multiple lines - the newlines are preserved in the HTML version. Inline markup is applied only in the explanation.
Example: Using an argument block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
Example block
An example block shows an example of the function you are documenting, with code and output. The example block itself is prefixed with an @, and only specifies a title; you will use indented children of the Code and Output block variety to show respectively the code and the output for the example. These blocks will be explained later on. The title of your example block will be prefixed with "Example:" in the output automatically.
Example: Using an example block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ Some code goes here.
+
+		> Some output goes here.
Code block
The code block is used in an example to show example code. It is prefixed with a dollar sign ($), and all text following it will be show on the HTML page verbatim, without any further markup processing being done. It even allows you to display ZippyDoc formatting characters without having them interpreted, as is done on this page!
Example: Using a code block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> Some output goes here.
It is also possible to have a code block spanning multiple paragraphs, without each paragraph being broken up into a separate code block (as would normally happen if you just used the dollar sign). To do this, you can use two dollar signs at the start of the block. Note that after these two dollar signs, whitespace (except for spaces) is not eaten, meaning you can use tabs to indent further blocks of your code!
Example: Using a multi-paragraph code block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ if some_variable == True:
+
+		$$	my_function(42, "The answer to everything")
+
+		> Some output goes here.
Output block
The output block is used to display sample output in an example. Just like the code block, it is shown exactly as it originally was, without any further formatting applied. It is prefixed by a >, and like the code block it has a continuation character - in this case, that is >>.
Example: Using an output block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
Example: Using a multi-paragraph output block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
+
+		>> Did you know The answer to everything is 42?
Exclamation block
The exclamation block allows you to mark a block of text as "important". In the standard HTML layout, it will have a yellow-ish background, and will be prefixed with "Important!". It is prefixed with an exclamation mark (!). Inline markup is applied.
Example: Using an exclamation block
Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	! Only ever use this function with the number '42'!
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+
+	@ Using this function
+
+		$ my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
Header block
A header block is a generic header to indicate the start of a new section. It is treated as a separate element, not as a "container". The header blocks in ZippyDoc work similarly to those in Markdown: they are prefixed by a hash (#), and the amount of hash characters defines what level of header it is.
Example: Using header blocks
Code:
# This is a level 1 (largest) header.
+
+## This is a level 2 header.
+
+...
+
+####### This is a level 7 (smallest) header.
Text block
A text block is any block that is not prefixed by a special character. It is shown as defined, with inline markup applied.

Inline markup

There are also various forms of inline markup that you can use in your documentation.
Emphasized text
Emphasized text is typically displayed as italic. You can emphasize text by enclosing it in two asterisks on each side.
Example: Emphasizing text
Code:
This is just some text, and **this part is emphasized**.
Strong text
Strong text is typically displayed as bold. You can make text strong by enclosing it in two underscores on each side.
Example: Making text strong
Code:
This is just some text, __and this part is strong__.
Internal references (hyperlinks)
Internal references are hyperlinks that point to other documents in the same documentation set. Depending on the export format (currently only HTML is supported), the appropriate extension is automatically appended. The paths should resemble the directory structure you are storing the ZippyDoc source files in. The target of the reference is enclosed in curly braces and prefixed with a >. If you wish to give the reference a friendly description, you can do so by appending it, enclosed in parentheses.
Example: Referencing another documentation page
Code:
You can also view the API documentation at {>api/index}.
Example: Referencing another documentation page with a friendly description
Code:
You can also view the {>api/index}(API documentation).
External references (hyperlinks)
External references are hyperlinks just like the internal references, but they refer to an external resources. The syntax is identical to that of internal references, except for the > disappearing. Note that external references are only picked up when the text enclosed in the braces is an actual URI of some sort.
You can also force an external reference to be created by prefixing the URI with <. This is useful when you want to for example link to a download relative to the current page.
Example: Referencing Google
Code:
You could also search {http://www.google.com/}.
Example: Referencing another documentation page with a friendly description
Code:
You could also search {http://www.google.com/}(Google).
Example: Referencing a relative file that is not a ZippyDoc document
Code:
You can download it by {<file.zip}(clicking here).
Fixed-width text
Fixed-width text can be useful to indicate code elements or other things that would benefit from being displayed in a terminal-like font. You can make text fixed-width by enclosing it in backticks.
Example: Fixed-width text
Code:
Now enter `./run.sh` into your terminal.

Special tags

Currently there is only one special tag. Special tags can be inserted anywhere in the document to insert a generated element.
Table of contents
To insert a table of contents that is automatically generated from all definition blocks on the page, simply insert {TOC} on the page where you want it to appear (it has to be in its own paragraph). Typically you will want to place it just below the main page header.
Every item in a table of contents will be followed by a snippet of text, that is grabbed from the first Text block for that definition. Alternative notations are shown after this description.
Example: Including a table of contents
Code:
# Sample documentation
+
+{TOC}
+
+^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here.
+
+	...

Full example

You can view a full example here of a ZippyDoc source file and its result.
+ + diff --git a/tools/documentation/zippydoc/documentation.zpy b/tools/documentation/zippydoc/documentation.zpy new file mode 100644 index 0000000..6441371 --- /dev/null +++ b/tools/documentation/zippydoc/documentation.zpy @@ -0,0 +1,325 @@ +# ZippyDoc format documentation + +{ Some output goes here. + +^ Code block + + The code block is used in an example to show example code. It is prefixed with a dollar sign ($), and all text following it will be show on the HTML page verbatim, without + any further markup processing being done. It even allows you to display ZippyDoc formatting characters without having them interpreted, as is done on this page! + + @ Using a code block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > Some output goes here. + + It is also possible to have a code block spanning multiple paragraphs, without each paragraph being broken up into a separate code block (as would normally happen if you + just used the dollar sign). To do this, you can use two dollar signs at the start of the block. Note that after these two dollar signs, whitespace (except for spaces) is + **not** eaten, meaning you can use tabs to indent further blocks of your code! + + @ Using a multi-paragraph code block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ if some_variable == True: + + $$ $$ my_function(42, "The answer to everything") + + $$ > Some output goes here. + +^ Output block + + The output block is used to display sample output in an example. Just like the code block, it is shown exactly as it originally was, without any further formatting applied. + It is prefixed by a >, and like the code block it has a continuation character - in this case, that is >>. + + @ Using an output block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + @ Using a multi-paragraph output block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + $$ >> Did you know The answer to everything is 42? + +^ Exclamation block + + The exclamation block allows you to mark a block of text as "important". In the standard HTML layout, it will have a yellow-ish background, and will be prefixed + with "__Important!__". It is prefixed with an exclamation mark (!). Inline markup is applied. + + @ Using an exclamation block + + $ ^ my_function(**argument**, **another_argument**) + + $$ Some kind of text describing the function goes here. + + $$ ! Only ever use this function with the number '42'! + + $$ argument:: + This is the first argument to this example function. + + $$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + + $$ @ Using this function + + $$ $ my_function(42, "The answer to everything") + + $$ > The answer to everything is 42! + + +^ Header block + + A header block is a generic header to indicate the start of a new section. It is treated as a separate element, not as a "container". The header blocks in ZippyDoc + work similarly to those in Markdown: they are prefixed by a hash (#), and the amount of hash characters defines what level of header it is. + + @ Using header blocks + + $ # This is a level 1 (largest) header. + + $$ ## This is a level 2 header. + + $$ ... + + $$ ####### This is a level 7 (smallest) header. + + +^ Text block + + A text block is any block that is not prefixed by a special character. It is shown as defined, with inline markup applied. + + +## Inline markup + +There are also various forms of inline markup that you can use in your documentation. + +^ Emphasized text + + Emphasized text is typically displayed as italic. You can emphasize text by enclosing it in two asterisks on each side. + + @ Emphasizing text + + $ This is just some text, and **this part is emphasized**. + +^ Strong text + + Strong text is typically displayed as bold. You can make text strong by enclosing it in two underscores on each side. + + @ Making text strong + + $ This is just some text, __and this part is strong__. + +^ Internal references (hyperlinks) + + Internal references are hyperlinks that point to other documents in the same documentation set. Depending on the export format (currently only HTML is supported), + the appropriate extension is automatically appended. The paths should resemble the directory structure you are storing the ZippyDoc source files in. The target + of the reference is enclosed in curly braces and prefixed with a >. If you wish to give the reference a friendly description, you can do so by appending it, + enclosed in parentheses. + + @ Referencing another documentation page + + $ You can also view the API documentation at {>api/index}. + + @ Referencing another documentation page with a friendly description + + $ You can also view the {>api/index}(API documentation). + + +^ External references (hyperlinks) + + External references are hyperlinks just like the internal references, but they refer to an external resources. The syntax is identical to that of internal references, + except for the > disappearing. Note that external references are only picked up when the text enclosed in the braces is an actual URI of some sort. + + You can also force an external reference to be created by prefixing the URI with <. This is useful when you want to for example link to a download relative to the current + page. + + @ Referencing Google + + $ You could also search {http://www.google.com/}. + + @ Referencing another documentation page with a friendly description + + $ You could also search {http://www.google.com/}(Google). + + @ Referencing a relative file that is not a ZippyDoc document + + $ You can download it by {example}(view a full example here) of a ZippyDoc source file and its result. diff --git a/tools/documentation/zippydoc/example.html b/tools/documentation/zippydoc/example.html new file mode 100644 index 0000000..458d331 --- /dev/null +++ b/tools/documentation/zippydoc/example.html @@ -0,0 +1,177 @@ + + + + + + + +

A complete example

Code:
^ my_function(**argument**, **another_argument**)
+
+	Some kind of text describing the function goes here. `Also some mono-spaced text.`
+
+	! Only ever use this function with the number '42'!
+
+	argument::
+		This is the first argument to this example function.
+
+	another_argument::
+		This is the second argument to this example function.
+		As you can see, it's possible to split the explanation over multiple lines as well.
+		We can also add an {>documentation}(internal link) and an {http://google.com/}(external link).
+
+	@ Using this function
+
+		$ if some_variable == True:
+
+		$$	my_function(42, "The answer to everything")
+
+		> The answer to everything is 42!
+
+		>>	Did you know The answer to everything is 42?

Result

my_function(argument, another_argument)
Some kind of text describing the function goes here. Also some mono-spaced text.
Important: Only ever use this function with the number '42'!
argument
This is the first argument to this example function.
another_argument
This is the second argument to this example function. As you can see, it's possible to split the explanation over multiple lines as well. We can also add an internal link and an external link.
Example: Using this function
Code:
if some_variable == True:
+
+	my_function(42, "The answer to everything")
Output:
The answer to everything is 42!
+
+	Did you know The answer to everything is 42?
+ + diff --git a/tools/documentation/zippydoc/example.zpy b/tools/documentation/zippydoc/example.zpy new file mode 100644 index 0000000..bd4110e --- /dev/null +++ b/tools/documentation/zippydoc/example.zpy @@ -0,0 +1,51 @@ +# A complete example + +$ ^ my_function(**argument**, **another_argument**) + +$$ Some kind of text describing the function goes here. `Also some mono-spaced text.` + +$$ ! Only ever use this function with the number '42'! + +$$ argument:: + This is the first argument to this example function. + +$$ another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + We can also add an {>documentation}(internal link) and an {http://google.com/}(external link). + +$$ @ Using this function + +$$ $ if some_variable == True: + +$$ $$ my_function(42, "The answer to everything") + +$$ > The answer to everything is 42! + +$$ >> Did you know The answer to everything is 42? + +## Result + +^ my_function(**argument**, **another_argument**) + + Some kind of text describing the function goes here. `Also some mono-spaced text.` + + ! Only ever use this function with the number '42'! + + argument:: + This is the first argument to this example function. + + another_argument:: + This is the second argument to this example function. + As you can see, it's possible to split the explanation over multiple lines as well. + We can also add an {>documentation}(internal link) and an {http://google.com/}(external link). + + @ Using this function + + $ if some_variable == True: + + $$ my_function(42, "The answer to everything") + + > The answer to everything is 42! + + >> Did you know The answer to everything is 42? diff --git a/tools/documentation/zippydoc/index.html b/tools/documentation/zippydoc/index.html new file mode 100644 index 0000000..04ffd4d --- /dev/null +++ b/tools/documentation/zippydoc/index.html @@ -0,0 +1,167 @@ + + + + + + + +

ZippyDoc

Hi, this is the website of ZippyDoc, a compact, light-weight and human-readable format for documenting code, APIs, and other things, that can be easily converted to HTML.
It is designed primarily to be simple to use (unlike complex markup languages like reStructuredText), and very code-oriented (unlike other simple markup languages like Markdown). You will probably learn the entire syntax in about 10 minutes.
ZippyDoc (both the format and the parser) are licensed under the WTFPL, meaning you can basically do with it whatever you want, and reuse it in any fashion you see fit. I hope it will help you write nicer, easier, and more complete documentation!
While ZippyDoc is technically intended for documentation, I decided to whip up a simple index page in ZippyDoc as well - you're looking at it! :)

What does the ZippyDoc format look like?

Code:
^ my_function(argument1, argument2)
+
+	! This is just an example!
+
+	This is a function.
+
+	argument1::
+		This is the first argument.
+
+	argument2::
+		This is the second argument.
+
+	@ How to call my_function
+
+		$ my_function("ZippyDoc", "awesome")
+
+		> "ZippyDoc is awesome!"
Result:
my_function(argument1, argument2)
Important: This is just an example!
This is a function.
argument1
This is the first argument.
argument2
This is the second argument.
Example: How to call my_function
Code:
my_function("ZippyDoc", "awesome")
Output:
"ZippyDoc is awesome!"

Documentation

The documentation for ZippyDoc can be found here.

Downloading ZippyDoc

ZippyDoc is still in a pretty messy stage, but it should already work reliably according to the current documentation. GitHub repository is coming soon, until that time you can download the conversion script here. It's a Python script, so you'll need a Python interpreter of some sort. No dependencies are necessary, it only uses standard library functionality. Simply run it with all files you wish to convert as arguments, and it will convert each of them into a file with the same name, but a .html extension instead of the original extension. It's strongly recommended to name your ZippyDoc source files with the .zpy extension.
+ + diff --git a/tools/documentation/zippydoc/index.zpy b/tools/documentation/zippydoc/index.zpy new file mode 100644 index 0000000..436d40e --- /dev/null +++ b/tools/documentation/zippydoc/index.zpy @@ -0,0 +1,62 @@ +# ZippyDoc + +Hi, this is the website of ZippyDoc, a compact, light-weight and human-readable format for documenting code, APIs, and other things, that can be easily converted to HTML. + +It is designed primarily to be simple to use (unlike complex markup languages like reStructuredText), and very code-oriented (unlike other simple markup languages like Markdown). +You will probably learn the entire syntax in about 10 minutes. + +ZippyDoc (both the format and the parser) are licensed under the {http://www.wtfpl.net/}(WTFPL), meaning you can basically do with it whatever you want, and reuse it in any +fashion you see fit. I hope it will help you write nicer, easier, and more complete documentation! + +While ZippyDoc is technically intended for documentation, I decided to whip up a simple index page in ZippyDoc as well - you're looking at it! :) + +## What does the ZippyDoc format look like? + +$ ^ my_function(argument1, argument2) + +$$ ! This is just an example! + +$$ This is a function. + +$$ argument1:: + This is the first argument. + +$$ argument2:: + This is the second argument. + +$$ @ How to call my_function + +$$ $ my_function("ZippyDoc", "awesome") + +$$ > "ZippyDoc is awesome!" + +####### Result: + +^ my_function(argument1, argument2) + + ! This is just an example! + + This is a function. + + argument1:: + This is the first argument. + + argument2:: + This is the second argument. + + @ How to call my_function + + $ my_function("ZippyDoc", "awesome") + + > "ZippyDoc is awesome!" + +## Documentation + +The documentation for ZippyDoc can be found {>documentation}(here). + +## Downloading ZippyDoc + +ZippyDoc is still in a pretty messy stage, but it should already work reliably according to the current documentation. GitHub repository is coming soon, until that time you can +{ + + + + + + %s + + +""" + +class TreeLevel: + def __init__(self, indentation, data): + self.elements = [] + self.indentation = indentation + self.data = data + + def add(self, element): + self.elements.append(element) + + def output(self): + return self.render() + + def render_children(self): + child_output = "" + + for child in self.elements: + child_output += child.output() + + return '
%s
' % child_output + + def process_inline_markup(self, text): + text = re.sub("`([^`]+)`", '\\1', text) # Emphasized + text = re.sub("\*\*([^*]+)\*\*", "\\1", text) # Emphasized + text = re.sub("__([^_]+)__", "\\1", text) # Strong + text = re.sub("{>([^}]+)}\(([^)]+)\)", '\\2', text) # Hyperlink with text + text = re.sub("{>([^}]+)}", '\\1', text) # Hyperlink + text = re.sub("{([^}]+:[^}]+)}\(([^)]+)\)", '\\2', text) # External hyperlink with text + text = re.sub("{([^}]+:[^}]+)}", '\\1', text) # External hyperlink + text = re.sub("{<([^}]+)}\(([^)]+)\)", '\\2', text) # Forced external hyperlink with text + text = re.sub("{<([^}]+)}", '\\1', text) # Forced external hyperlink + + return text + + def fix_preformatted(self, text): + return text.replace("<", "<").replace(">", ">") + + def clear_markup(self, text): + return re.sub("\*\*([^*]+)\*\*", "\\1", text) + + def render(self): + return self.render_children() + +class Example(TreeLevel): + def render(self): + return '
Example: %s %s
' % (self.data, self.render_children()) + +class Code(TreeLevel): + def render(self): + return 'Code:
%s
' % self.fix_preformatted(self.data) + +class Output(TreeLevel): + def render(self): + return 'Output:
%s
' % self.fix_preformatted(self.data) + +class Definition(TreeLevel): + def get_anchor(self): + first = self.clear_markup(self.data.splitlines()[0]) + anchor = first.replace("...", "") + anchor = anchor.replace(".", "_") + anchor = re.sub("[^a-zA-Z0-9_]", "", anchor) + return anchor + + def get_description(self): + for element in self.elements: + if element.__class__.__name__ == "Text": + data = self.process_inline_markup(element.data) + + if len(data) > 80: + matches = re.match("^(.{0,80})\W", data) + return matches.group(1) + "..." + else: + return data + + return "" + + def render(self): + return '
%s %s
' % (self.get_anchor(), self.process_inline_markup(self.data.replace("\n", "
")), self.render_children()) + +class Exclamation(TreeLevel): + def render(self): + return '
Important: %s
' % self.process_inline_markup(self.data) + +class Argument(TreeLevel): + def __init__(self, indentation, data, argname): + self.elements = [] + self.indentation = indentation + self.argname = argname + self.data = data + + def render(self): + return '
%s
%s
' % (self.argname, self.process_inline_markup(self.data)) + +class Header(TreeLevel): + def __init__(self, indentation, data, depth): + self.elements = [] + self.indentation = indentation + self.depth = depth + self.data = data + + def render(self): + if self.depth <= 7: + title_type = "h%d" % self.depth + else: + title_type = "h7" + + return "<%s>%s" % (title_type, self.data, title_type) + +class Text(TreeLevel): + def render(self): + return '
%s
' % self.process_inline_markup(self.data) + +class Index(TreeLevel): + def render(self): + global toc_items + + rendered = "" + + for item in toc_items: + forms = item.data.splitlines() + first = self.clear_markup(forms[0]) + + if len(forms) > 1: + rest = '(also: ' + ', '.join(self.clear_markup(form) for form in forms[1:]) + ")" + else: + rest = "" + + anchor = item.get_anchor() + description = item.get_description() + rendered += '
  • %s %s %s
    • ' % (anchor, first, description, rest) + + return '

    Table of contents

      %s
    ' % rendered + +for zpy in files: + destination = os.path.splitext(zpy)[0] + ".html" + + f = open(zpy, "r") + data = f.read() + f.close() + + paragraphs = re.split("\s*\n\s*\n", data) + toc_items = [] + current_level = 0 + current_paragraph = 0 + current_elements = {0: TreeLevel(0, "root")} + + for paragraph in paragraphs: + if paragraph.strip() == "": + continue + + current_paragraph += 1 + indentation = len(paragraph) - len(paragraph.lstrip("\t")) + 1 + + if indentation > current_level + 1: + raise Exception("Invalid indentation found in paragraph %d" % current_paragraph) + + element_type = TreeLevel + start = indentation - 1 + + lines = [line[start:] for line in paragraph.splitlines()] + + if lines[0].startswith("#"): + element_type = Header + depth = len(lines[0]) - len(lines[0].lstrip("#")) + lines[0] = lines[0].lstrip("# ") + data = " ".join(lines) + elif lines[0].startswith("^"): + element_type = Definition + lines[0] = lines[0].lstrip("^ ") + data = "\n".join(lines) + elif lines[0].startswith("@"): + element_type = Example + lines[0] = lines[0].lstrip("@ ") + data = " ".join(lines) + elif lines[0].startswith("$$") and current_elements[current_level].__class__.__name__ == "Code": + current_elements[current_level].data += "\n\n" + "\n".join(lines).lstrip("$ ") + continue + elif lines[0].startswith("$"): + element_type = Code + lines[0] = lines[0].lstrip("$ ") + data = "\n".join(lines) + elif lines[0].startswith(">>") and current_elements[current_level].__class__.__name__ == "Output": + current_elements[current_level].data += "\n\n" + "\n".join(lines).lstrip("> ") + continue + elif lines[0].startswith(">"): + element_type = Output + lines[0] = lines[0].lstrip("> ") + data = "\n".join(lines) + elif lines[0].startswith("!"): + element_type = Exclamation + lines[0] = lines[0].lstrip("! ") + data = " ".join(lines) + elif re.match(".*::\s*$", lines[0]): + element_type = Argument + argname = lines[0][:-2] + data = " ".join(line.lstrip() for line in lines[1:]) + elif lines[0].strip() == "{TOC}": + element_type = Index + data = "" + else: + element_type = Text + data = " ".join(lines) + + #print "Found element of type %s at indentation %d with data %s" % (element_type.__name__, indentation, data[:80]) + + if element_type.__name__ == "Header": + element = Header(indentation, data, depth) + elif element_type.__name__ == "Argument": + element = Argument(indentation, data, argname) + else: + element = element_type(indentation, data) + + if element_type.__name__ == "Definition": + toc_items.append(element) + + current_elements[indentation - 1].add(element) + + current_level = indentation + current_elements[current_level] = element + + rendered = template % (current_elements[0].output()) + + f = open(destination, "w") + f.write(rendered) + f.close() + + print "Rendered %s" % destination diff --git a/tools/downloading/musicsearch.py b/tools/downloading/musicsearch.py new file mode 100644 index 0000000..5cd4050 --- /dev/null +++ b/tools/downloading/musicsearch.py @@ -0,0 +1,89 @@ +#!/usr/bin/python + +import argparse, os, urllib, json, subprocess, urllib2 + +parser = argparse.ArgumentParser(description='Searches for music files') + +parser.add_argument('-c', dest='limit', action='store', default='10', + help='maximum amount of results to show (don\'t overdo it please)') + +parser.add_argument('-v', dest='vlc', action='store_true', default='false', + help='play in VLC media player') + +parser.add_argument('query', metavar='QUERY', type=str, nargs='+', + help='destination path for generated class files.') + +args = parser.parse_args() +options = vars(args) + +query = " ".join(options['query']) +limit = int(options['limit']) + +results = urllib.urlopen("http://ex.fm/api/v3/song/search/%s?results=%d" % (urllib.quote_plus(query), limit)).read() + +try: + result_object = json.loads(results) +except ValueError: + print "No valid result was returned from the ex.fm API. Exiting..." + exit(1) + +if result_object['status_code'] != 200: + print "An error code was returned by the ex.fm API. Exiting..." + exit(1) + +print "Searching for '%s'..." % query + +if result_object['total'] <= 0: + print "No results." + exit(1) + +print "" + +for track in result_object['songs']: + if track['artist'] is None: + artist = "Unknown" + else: + artist = track['artist'] + + if track['album'] is None: + album = "Unknown" + else: + album = track['album'] + + if track['title'] is None: + title = "Unknown" + else: + title = track['title'] + + print "Artist: %s\t Album: %s" % (artist, album) + print "Title: %s" % title + print " %s" % track['url'] + print "" + +if options['vlc'] == True: + print "Playing the first working result in VLC media player..." + + working_url = "" + + for track in result_object['songs']: + try: + response = urllib2.urlopen(track['url']) + except urllib2.URLError, e: + continue + + headers = response.info() + + if "text/html" in headers['Content-Type']: + continue + + working_url = track['url'] + + break + + if working_url != "": + with open(os.devnull, 'w') as stfu: + subprocess.Popen(["vlc", "--one-instance", working_url], stdin=None, stdout=stfu, stderr=stfu) + exit(0) + else: + print "No working URLs found." + exit(1) diff --git a/tools/file-processing/catarc.py b/tools/file-processing/catarc.py new file mode 100755 index 0000000..3c4d6d9 --- /dev/null +++ b/tools/file-processing/catarc.py @@ -0,0 +1,126 @@ +#!/usr/bin/python + +# This script will run the appropriate command to print archive contents to stdout +# Written by Sven Slootweg, Licensed under WTFPL - in other words, feel free to reuse for whatever purpose you desire. + +import os, argparse, sys, subprocess + +parser = argparse.ArgumentParser(description='Prints the uncompressed contents of archives to stdout, based on certain criteria.') + +parser.add_argument('pattern', metavar='FILES', type=str, nargs='+', + help='files to parse') + +parser.add_argument('-s', dest='size', action='store', default=None, + help='size requirement that has to be satisfied for a file to be printed (use < and >)') + +args = parser.parse_args() +options = vars(args) + +def exit_script(): + sys.stderr.write("Exiting...\n") + exit(1) + +def to_bytes(size): + size = size.lower().strip() + + if size.endswith("t"): + return int(size[:-1]) * 1024 * 1024 * 1024 * 1024 + elif size.endswith("g"): + return int(size[:-1]) * 1024 * 1024 * 1024 + elif size.endswith("m"): + return int(size[:-1]) * 1024 * 1024 + elif size.endswith("k"): + return int(size[:-1]) * 1024 + else: + return int(size) + +def from_bytes(size, unit): + size = int(size) + unit = unit.lower() + + if unit == 't': + return size * 1.0 / (1024 * 1024 * 1024 * 1024) + elif unit == 'g': + return size * 1.0 / (1024 * 1024 * 1024) + elif unit == 'm': + return size * 1.0 / (1024 * 1024) + elif unit == 'k': + return size * 1.0 / (1024) + else: + return size + +specifications = {} + +if options['size'] is not None: + # Parse size specification + for specification in options['size'].split(","): + if specification[:1] == "<": + specifications['<'] = specification[1:].strip() + elif specification[:1] == ">": + specifications['>'] = specification[1:].strip() + elif specification[:1] == "=": + specifications['='] = specification[1:].strip() + else: + sys.stderr.write("Incorrect size specification: %s\n" % specification) + exit_script() + +# Select all files matching the given pattern +file_list = options['pattern'] + +for item in file_list: + data = os.stat(item) + filesize = data.st_size + + try: + if filesize >= to_bytes(specifications['<']): + continue + except KeyError, e: + pass + + + try: + if filesize <= to_bytes(specifications['>']): + continue + except KeyError, e: + pass + + + try: + if int(from_bytes(filesize, specifications['='][-1:])) != specifications['=']: + continue + except KeyError, e: + pass + + # Passed all size tests, let's process it + with open(os.devnull, 'w') as stfu: + if item.endswith(".7z"): + processor = "7zip" + returncode = subprocess.call(['7z', 'e', '-so', item], stderr=stfu) + elif item.endswith(".tar"): + processor = "tar" + returncode = subprocess.call(['tar', '-Oxf', item], stderr=stfu) + elif item.endswith(".tar.gz"): + processor = "tar/gzip" + returncode = subprocess.call(['tar', '-Oxzf', item], stderr=stfu) + elif item.endswith(".tar.bz2"): + processor = "tar/bzip2" + returncode = subprocess.call(['tar', '-Oxjf', item], stderr=stfu) + elif item.endswith(".gz"): + processor = "gzip" + returncode = subprocess.call(['gzip', '-cd', item], stderr=stfu) + elif item.endswith(".bz2"): + processor = "bzip2" + returncode = subprocess.call(['bzip2', '-cd', item], stderr=stfu) + elif item.endswith(".zip"): + processor = "unzip" + returncode = subprocess.call(['unzip', '-p', item], stderr=stfu) + else: + sys.stderr.write("WARNING: Skipping item %s, not a recognized archive type\n" % item) + continue + + if returncode == 0: + sys.stderr.write("Successfully output %s\n" % item) + elif returncode == 2: + sys.stderr.write("ERROR: Could not run the needed command - are you sure you have %s installed?\n" % processor) + else: + sys.stderr.write("ERROR: Failed to output %s\n" % item) diff --git a/tools/file-processing/fix_instiki_html_dump.sh b/tools/file-processing/fix_instiki_html_dump.sh new file mode 100644 index 0000000..2df998e --- /dev/null +++ b/tools/file-processing/fix_instiki_html_dump.sh @@ -0,0 +1 @@ +find ./ -name "*.xhtml" | xargs sed -i.bak -r -e "s/%([0-9A-Z]{2})/%25\\1/g" diff --git a/tools/file-processing/revtt_clean.py b/tools/file-processing/revtt_clean.py new file mode 100644 index 0000000..4a314a6 --- /dev/null +++ b/tools/file-processing/revtt_clean.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +import sys + +lines = open(sys.argv[1]).readlines() + +accounts = {} +password_count = 0 + +for line in lines: + try: + username, password = line.split("-") + + username = username.strip()[1:-1] + password = password.strip()[1:-1] + + if username not in accounts: + accounts[username] = [password] + password_count += 1 + else: + if password not in accounts[username]: + accounts[username].append(password) + password_count += 1 + except ValueError, e: + pass + + +for username, passwords in accounts.iteritems(): + for password in passwords: + print "%s\t\t%s" % (username, password) + +print "Done, %d accounts with a total of %d passwords." % (len(accounts), password_count) diff --git a/tools/irc/dnsbl.txt b/tools/irc/dnsbl.txt new file mode 100644 index 0000000..f1e004c --- /dev/null +++ b/tools/irc/dnsbl.txt @@ -0,0 +1,8 @@ +dnsbl.dronebl.org +combined.abuse.ch +opm.tornevall.org +torserver.tor.dnsbl.sectoor.de +socks.dnsbl.sorbs.net +http.dnsbl.sorbs.net +b.barracudacentral.org +sbl.spamhaus.org diff --git a/tools/irc/ipcheck.py b/tools/irc/ipcheck.py new file mode 100644 index 0000000..1e7347a --- /dev/null +++ b/tools/irc/ipcheck.py @@ -0,0 +1,141 @@ +import socket, argparse, sys, time, re +from collections import deque +from threading import Thread + +dnsbls = open("dnsbl.txt").readlines() + +socket.setdefaulttimeout(0.5) + +total_checking = 0 + +class CheckThread(Thread): + def __init__(self, username, ip): + Thread.__init__(self) + self.username = username + self.ip = ip + + def run(self): + global total_checking + reversed_ip = ".".join(self.ip.split(".")[::-1]) + + for dnsbl in dnsbls: + dnsbl = dnsbl.strip() + + try: + socket.gethostbyname(reversed_ip + "." + dnsbl) + + # This is bad. + #sock.send("MODE #bottest +b *@%s\r\n" % self.ip) + #sock.send("KICK #bottest %s :User %s is blacklisted in %s!\r\n" % (self.username, self.username, dnsbl)) + #sock.send("PRIVMSG #bottest :User %s is blacklisted in %s!\r\n" % (self.username, dnsbl)) + #sock.send("PRIVMSG #bottest :User %s is blacklisted in %s!\r\n" % (self.username, dnsbl)) + sock.send("KILL %s :User %s is blacklisted in %s!\r\n" % (self.username, self.username, dnsbl)) + print "IP %s is blacklisted in %s!" % (self.ip, dnsbl) + total_checking -= 1 + return False + except: + # This is good. + print "No blacklist matches found for %s" % self.ip + pass + + sock.send("PRIVMSG #bottest :User %s is clean.\r\n" % self.username) + total_checking -= 1 + return True + +def run_tocheck(): + if total_checking <= 20 and len(to_check) > 0: + nickname = to_check.popleft() + ip = users[nickname] + t = CheckThread(nickname, ip) + t.start() + +def split_irc(message): + message = re.sub("(?<=[0-9A-Fa-fI]):(?=[0-9A-Fa-fI])", "[..]", message) + + if ":" in message: + first, second = message.split(":", 1) + return first.rstrip().split(" ") + [second] + else: + return message.split(" ") + +parser = argparse.ArgumentParser(description='Connects to an IRC network and scans for proxies.') + +parser.add_argument('-H', dest='hostname', action='store', required=True, + help='server to connect to') + +parser.add_argument('-o', dest='port', action='store', required=True, + help='port* to connect to') + +parser.add_argument('-u', dest='username', action='store', required=True, + help='oper username') + +parser.add_argument('-p', dest='password', action='store', required=True, + help='oper password') + +args = parser.parse_args() +options = vars(args) + +to_check = deque([]) +users = {} + +print "Connecting...", + +sock = socket.socket() +sock.settimeout(None) +sock.connect((options['hostname'], int(options['port']))) +readbuffer = "" + +print "connected" + +sock.send("NICK botkill\r\n") +sock.send("USER botkill %s 0 :ipcheck.py\r\n" % options['hostname']) + +print "Registered." + +while True: + readbuffer = readbuffer + sock.recv(1024) + lines = readbuffer.split("\n") + readbuffer = lines.pop() + + for line in lines: + run_tocheck() + + if line.startswith(":"): + origin, line = line.split(" ", 1) + + line = line.rstrip() + parts = split_irc(line) + + if parts[0] == "PING": + sock.send("PONG %s\r\n" % parts[1]) + print "Completed connection challenge." + elif parts[0] == "001": + sock.send("OPER %s %s\r\n" % (options['username'], options['password'])) + elif parts[0] == "381": + print "Authenticated as oper." + sock.send("JOIN #bottest\r\n") + elif parts[0] == "JOIN": + if parts[1].lower() == "#bottest": + username = origin[1:].split("!")[0] + print username + " joined" + sock.send("USERIP %s\r\n" % username) + elif parts[0] == "340": + try: + data = parts[2].split("=", 1) + nickname = data[0] + + if nickname.endswith("*"): + nickname = nickname[:-1] + + ip = data[1].split("@", 1)[1] + users[nickname] = ip + to_check.append(nickname) + run_tocheck() + print "User %s has IP %s" % (nickname, ip) + except: + pass + elif parts[0] == "491": + print "Invalid oper credentials given." + exit(1) + + time.sleep(0.005) diff --git a/tools/irc/rakill.py b/tools/irc/rakill.py index 193802d..5f1d544 100644 --- a/tools/irc/rakill.py +++ b/tools/irc/rakill.py @@ -41,10 +41,20 @@ parser.add_argument('--gline', dest='action_gline', action='store_true', parser.add_argument('--kill', dest='action_kill', action='store_true', help='kill all matching users') + +parser.add_argument('--unrealircd', dest='ircd_unreal', action='store_true', + help='assume UnrealIRCd') + +parser.add_argument('--inspircd', dest='ircd_insp', action='store_true', + help='assume InspIRCd') args = parser.parse_args() options = vars(args) +if options['ircd_unreal'] == False and options['ircd_insp'] == False: + print "You did not specify an IRCd. Try again with --inspircd or --unrealircd switch." + exit(1) + print "Connecting...", sock = socket.socket() @@ -77,7 +87,12 @@ while True: sock.send("OPER %s %s\r\n" % (options['username'], options['password'])) elif parts[0] == "381": print "Authenticated as oper." - sock.send("WHO ** h\r\n") + if options['ircd_insp'] == True: + # InspIRCd + sock.send("WHO ** h\r\n") + elif options['ircd_unreal'] == True: + # UnrealIRCd + sock.send("WHO +hR **\r\n") print "Requested userlist." elif parts[0] == "352": try: diff --git a/tools/scrapers/devilskitchen.py b/tools/scrapers/devilskitchen.py new file mode 100644 index 0000000..6ec7324 --- /dev/null +++ b/tools/scrapers/devilskitchen.py @@ -0,0 +1,99 @@ +#!/usr/bin/python +start_page = "http://www.devilskitchen.me.uk/" +default_headers = { + 'User-Agent': "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13", + 'Referer': start_page +} + +import re, urllib2, datetime, argparse, os, json, time +from bs4 import BeautifulSoup + +parser = argparse.ArgumentParser(description='Archive posts for devilskitchen.me.uk in JSON format') + +parser.add_argument('-O', dest='output_dir', action='store', default='.', + help='output directory for archived posts') + +args = parser.parse_args() +options = vars(args) + +try: + os.mkdir(options['output_dir']) +except: + pass + +def fetch_page_headers(url, headers): + request = urllib2.Request(url, headers=headers) + opener = urllib2.build_opener() + response = opener.open(request) + return (response.code, response.headers, response.read()) + +def fetch_archives(): + status_code, headers, response = fetch_page_headers(start_page, default_headers) + + if status_code == 200: + archive_pages = re.findall("(http:\/\/www\.devilskitchen\.me\.uk\/([0-9]{4})_([0-9]{2})_[0-9]{2}_archive\.html)", response) + + for page in archive_pages: + if os.path.exists("%s/%s-%d" % (options['output_dir'], page[1], int(page[2]))): + print "%s-%s already exists, skipping..." % (page[1], page[2]) + else: + print "Scraping %s..." % page[0] + fetch_articles(page[0]) + time.sleep(5) + else: + print "ERROR: Failed to retrieve archive index! Exiting..." + exit(1) + + +def fetch_articles(url): + try: + status_code, headers, response = fetch_page_headers(url, default_headers) + except urllib2.HTTPError, e: + print "ERROR: 403 encountered on %s" % url + return False + + if status_code == 200: + soup = BeautifulSoup(response) + posts = soup.find_all("div", class_="post") + + for post in posts: + try: + post_title = post.h5.string + except AttributeError, e: + print "WARNING: Article with missing title" + post_title = "" + + author_details = post.find_all("p", class_="author-details")[0] + author_name = author_details.find_all("span", class_="author-name")[0].string + post_date = author_details.find_all("a")[0].string + post_body = post.find_all("div", class_="post-body")[0].div.prettify() + + actual_date = datetime.datetime.strptime(post_date, "%m/%d/%Y %I:%M:%S %p") + + try: + os.mkdir("%s/%d-%d" % (options['output_dir'], actual_date.year, actual_date.month)) + except: + pass + + try: + json_file = open("%s/%d-%d/%d-%d-%d-%d-%d-%d.json" % (options['output_dir'], actual_date.year, actual_date.month, actual_date.year, + actual_date.month, actual_date.day, actual_date.hour, actual_date.minute, actual_date.second), 'w') + + json.dump({ + 'title': post_title, + 'date': actual_date.isoformat(), + 'author': author_name, + 'body': post_body + }, json_file) + + json_file.close() + except: + raise + + + print "Archived '%s', posted at %s by %s" % (post_title, actual_date.isoformat(), author_name) + else: + print "ERROR: Failed to retrieve %s! Status code was %d" % (url, status_code) + +#soup = BeautifulSoup(html_doc) +fetch_archives() diff --git a/tools/server-management/speedtest.py b/tools/server-management/speedtest.py new file mode 100755 index 0000000..c952d75 --- /dev/null +++ b/tools/server-management/speedtest.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +import urllib2, time, urllib +from lxml import etree + +def avg(inp): + return (reduce(lambda x, y: x + y, inp) / len(inp)) + +referer = "http://c.speedtest.net/flash/speedtest.swf?v=316125" +num = 1353968072002 +config_url = "http://speedtest.net/speedtest-config.php?x=%d" % num +server_url = "http://speedtest.net/speedtest-servers.php?x=%d" % num +download_path = "/random%dx%d.jpg?x=%d&y=%d" +upload_path = "/upload.php?x=%d" % num +latency_path = "/latency.txt?x=%d" % num +sizes = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000] + +servers = [] +server_count = 0 + +# First, get our own details. +result = urllib2.urlopen(config_url) + +for event, element in etree.iterparse(result): + if element.tag == "client": + my_ip = element.get("ip") + my_isp = element.get("isp") + my_latitude = float(element.get("lat")) + my_longitude = float(element.get("lon")) + dl_average = float(element.get("ispdlavg")) + ul_average = float(element.get("ispulavg")) + + print "You are %s (%s), with latitude %f and longitude %f. Your ISPs average download speed is %.3f MB/sec, and their average upload speed is %.3f MB/sec." % (my_ip, my_isp, my_latitude, my_longitude, dl_average / 8 / 1024 / 1024, ul_average / 8 / 1024 / 1024) + +print "Retrieving server list...", + +# Retrieve and parse list of servers. +result = urllib2.urlopen(server_url) + +for event, element in etree.iterparse(result): + if element.tag == "server": + hostname = element.get("url").replace("/upload.php", "") + latitude = float(element.get("lat")) + longitude = float(element.get("lon")) + location = element.get("name") + country = element.get("country") + sponsor = element.get("sponsor") + + distance = abs(my_latitude - latitude) + abs(my_longitude - longitude) + + servers.append((distance, hostname, latitude, longitude, location, country, sponsor)) + server_count += 1 + + print "\rRetrieving server list... %d servers found so far" % server_count, + +# Sort the server list by distance. +servers = sorted(servers, key=lambda server: server[0]) + +print "\nFound 5 closest servers. Determining optimal latency..." + +fastest_server = () +fastest_latency = 0 + +for server in servers[:5]: + # Take 3 samples of each server. + latencies = [] + + for i in xrange(0, 3): + request = urllib2.Request(server[1] + latency_path) + + start_time = time.time() + urllib2.urlopen(request) + end_time = time.time() + + latencies.append((end_time - start_time) * 1000) + + latency = avg(latencies) + + if fastest_latency == 0 or latency < fastest_latency: + fastest_latency = latency + fastest_server = server + + print "\rFastest server so far is '%s' with %dms ping... (%s)" % (fastest_server[6], fastest_latency, latencies), + +print "\nTarget server is '%s'. Testing download speed..." % fastest_server[6] + +latency = fastest_latency + +size = 0 + +while size < 8: + # Take 3 samples + speeds = [] + times = [] + + for i in xrange(0, 3): + target_file = download_path % (sizes[size], sizes[size], num, i + 1) + request = urllib2.urlopen(fastest_server[1] + target_file) + filesize = int(request.info()['Content-Length']) + block_size = 4096 + r = 0 + start_time = time.time() + + while r < filesize: + request.read(block_size) + r += block_size + speed = r / (time.time() - start_time) + print "\rSize %d, attempt %d... %.3f MB/sec" % (sizes[size], i + 1, speed / 1024 / 1024), + + end_time = time.time() + + speeds.append(speed) + times.append(end_time - start_time) + + print "" + request.close() + + if avg(times) < 4: + size += 1 + else: + break + +# Take result from last speedtest as authorative. +if size >= 8: + size = 7 + +download_speed = avg(speeds) + +#print "Average speed, sample size %d, is %.3f MB/sec" % (sizes[size], download_speed / 1024 / 1024) + +print "Latency: %dms\tDownload speed: %.3f MB/sec" % (latency, download_speed / 1024 / 1024) +print "NOTE: Due to function call overhead, the latency is, at best, an estimation."