From 2c60dca0d1d8a222829c6f3e547b29d206d7f92d Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 16 Mar 2013 01:04:38 +0100 Subject: [PATCH 01/14] Write first version of command daemon --- command_daemon/command_daemon.py | 109 +++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 command_daemon/command_daemon.py diff --git a/command_daemon/command_daemon.py b/command_daemon/command_daemon.py new file mode 100644 index 0000000..142e735 --- /dev/null +++ b/command_daemon/command_daemon.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +import sys, os +import subprocess +import json, urlparse +import SocketServer, SimpleHTTPServer +from optparse import OptionParser + +parser = OptionParser() +(options, cmdargs) = parser.parse_args() + +try: + f = open("session_key", "r") + session_key = f.read().strip() + f.close() +except IOError, e: + sys.stderr.write("You must specify a session key.\n") + exit(1) + +os.remove("session_key") + +class CommandHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def do_GET(self): + global session_key + + req = urlparse.urlparse(self.path) + get_params = urlparse.parse_qs(req.query) + path = req.path + + if path=='/': + try: + command = json.loads(get_params['command'][0]) + except KeyError, e: + self.send_404() + return + except IndexError, e: + self.send_404() + return + except ValueError, e: + self.send_404() + return + + try: + key = get_params['key'][0] + except KeyError, e: + self.send_403() + return + except IndexError, e: + self.send_403() + return + + if key != session_key: + self.send_403() + return + + try: + result = json.dumps(self.run_command(command)) + except Exception, e: + print e + self.send_404() + return + + self.send_response(200) + self.send_header('Content-type','text/json') + self.end_headers() + self.wfile.write(result) + else: + self.send_404() + return + + def send_404(self): + self.send_response(404) + self.send_header('Content-type','text/plain') + self.end_headers() + self.wfile.write("404 Not Found") + + def send_403(self): + self.send_response(403) + self.send_header('Content-type','text/plain') + self.end_headers() + self.wfile.write("403 Forbidden") + + def run_command(self, args): + pr = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + data = pr.communicate() + pr.wait() + + return { + 'stdout': data[0], + 'stderr': data[1], + 'returncode': pr.returncode + } + +if os.fork(): exit(0) +os.umask(0) +os.setsid() +if os.fork(): exit(0) + +sys.stdout.flush() +sys.stderr.flush() +si = file('/dev/null', 'r') +so = file('/dev/null', 'a+') +se = file('/dev/null', 'a+', 0) +os.dup2(si.fileno(), sys.stdin.fileno()) +os.dup2(so.fileno(), sys.stdout.fileno()) +os.dup2(se.fileno(), sys.stderr.fileno()) + +httpd = SocketServer.ThreadingTCPServer(("localhost", 3434), CommandHandler) +httpd.serve_forever() From d3f070850f3e5e4c3882b923265b43f72e553061 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 16 Mar 2013 13:17:59 +0100 Subject: [PATCH 02/14] Add script to establish SSH daemon tunnels --- command_daemon/start.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 command_daemon/start.py diff --git a/command_daemon/start.py b/command_daemon/start.py new file mode 100644 index 0000000..79e8d16 --- /dev/null +++ b/command_daemon/start.py @@ -0,0 +1,23 @@ +import sys, subprocess + +stfu = open("/dev/null", "w") + +def run_command(args): + pr = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + data = pr.communicate() + pr.wait() + return pr.returncode + +host = sys.argv[1] +user = sys.argv[2] +port = int(sys.argv[3]) +keyfile = sys.argv[4] +session_key = sys.argv[5] + +if run_command(["ssh", "%s@%s" % (user, host), "-i", keyfile, "cd /etc/cvm/command_daemon; echo '%s' > session_key && ./command_daemon" % session_key]) == 0: + if run_command(["autossh", "-f", "-i", keyfile, "-M", str(port + 1), "%s@%s" % (user, host), "-L", "%s:localhost:3434" % port, "-N"]) == 0: + exit(0) + else: + sys.stderr.write("Failed to establish tunnel.\n") +else: + sys.stderr.write("Failed to start daemon.\n") From 387ced5e851499a5566b6003593580dd4dce635b Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 16 Mar 2013 13:18:46 +0100 Subject: [PATCH 03/14] Make the slave installer also install the command daemon --- installer/build.sh | 1 + installer/slave/install.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/installer/build.sh b/installer/build.sh index c996681..bc81126 100755 --- a/installer/build.sh +++ b/installer/build.sh @@ -1,6 +1,7 @@ cd slave echo "Copying needed files for slave SFX..." cp ../../runhelper/runhelper . +cp ../../command_daemon/command_daemon . cp ../../console/slave/dropper . cp ../../logshell/logshell . cp ../../logshell/cvmshell . diff --git a/installer/slave/install.py b/installer/slave/install.py index a64b499..19c8f11 100644 --- a/installer/slave/install.py +++ b/installer/slave/install.py @@ -229,12 +229,17 @@ sys.stdout.write("Lock on /etc/group removed.\n") # Create the main CVM data directories setuplib.create_directory("/etc/cvm", True, 0, cvm_gid, "u+rwx g+rwx o+rx") setuplib.create_directory("/etc/cvm/log", True, 0, 0, "u+rwx") +setuplib.create_directory("/etc/cvm/command_daemon", True, 0, 0, "u+rwx") sys.stdout.write("Created directories.\n") # Copy the runhelper setuplib.copy_file("runhelper", "/home/cvm/runhelper", True, cvm_uid, cvm_gid, "u+rwx") sys.stdout.write("Installed runhelper.\n") +# Copy the command daemon +setuplib.copy_file("command_daemon", "/etc/cvm/command_daemon/command_daemon", True, cvm_uid, cvm_gid, "u+rwx") +sys.stdout.write("Installed command daemon.\n") + if enable_dropper == "y": # Copy the shell dropper setuplib.copy_file("dropper", "/home/vz/dropper", True, vz_uid, vz_gid, "u+rwx") From 9f5c713ccefb7395fd0bf346e60963bc1d0df8ad Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:18:24 +0100 Subject: [PATCH 04/14] Fix incorrect uid and gid on command daemon script --- installer/slave/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/slave/install.py b/installer/slave/install.py index 19c8f11..866d52a 100644 --- a/installer/slave/install.py +++ b/installer/slave/install.py @@ -229,7 +229,7 @@ sys.stdout.write("Lock on /etc/group removed.\n") # Create the main CVM data directories setuplib.create_directory("/etc/cvm", True, 0, cvm_gid, "u+rwx g+rwx o+rx") setuplib.create_directory("/etc/cvm/log", True, 0, 0, "u+rwx") -setuplib.create_directory("/etc/cvm/command_daemon", True, 0, 0, "u+rwx") +setuplib.create_directory("/etc/cvm/command_daemon", True, cvm_uid, cvm_gid, "u+rwx") sys.stdout.write("Created directories.\n") # Copy the runhelper From 10dcb6102cdeeb207e6658b7e298737a4f4e7fc6 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:19:04 +0100 Subject: [PATCH 05/14] Fix code to properly set up a tunnel --- command_daemon/start.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/command_daemon/start.py b/command_daemon/start.py index 79e8d16..5f22872 100644 --- a/command_daemon/start.py +++ b/command_daemon/start.py @@ -1,4 +1,4 @@ -import sys, subprocess +import sys, os, subprocess stfu = open("/dev/null", "w") @@ -14,8 +14,13 @@ port = int(sys.argv[3]) keyfile = sys.argv[4] session_key = sys.argv[5] -if run_command(["ssh", "%s@%s" % (user, host), "-i", keyfile, "cd /etc/cvm/command_daemon; echo '%s' > session_key && ./command_daemon" % session_key]) == 0: - if run_command(["autossh", "-f", "-i", keyfile, "-M", str(port + 1), "%s@%s" % (user, host), "-L", "%s:localhost:3434" % port, "-N"]) == 0: +if run_command(["ssh", "%s@%s" % (user, host), "-o", "UserKnownHostsFile=/etc/cvm/knownhosts", "-o", "StrictHostKeyChecking=no", "-i", keyfile, "cd /etc/cvm/command_daemon; echo '%s' > session_key && ./command_daemon" % session_key]) == 0: + # Make autossh verify the connection is still alive every 10 seconds. + os.environ["AUTOSSH_POLL"] = 10 + os.environ["AUTOSSH_FIRST_POLL"] = 10 + + if run_command(["autossh", "-f", "-i", keyfile, "-M", str(port + 1), "-o", "UserKnownHostsFile=/etc/cvm/knownhosts", "-o", "StrictHostKeyChecking=no", "%s@%s" % (user, host), "-L", "%s:localhost:3434" % port, "-N"]) == 0: + sys.stdout.write("Tunnel established.\n"); exit(0) else: sys.stderr.write("Failed to establish tunnel.\n") From 8c5b62730a46210e502b7871fb6c5f9fa9e5e02e Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:20:09 +0100 Subject: [PATCH 06/14] Explicitly specify 'sudo' in commands --- frontend/classes/vps.php | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/classes/vps.php b/frontend/classes/vps.php index f2c00e4..bb45e47 100644 --- a/frontend/classes/vps.php +++ b/frontend/classes/vps.php @@ -100,7 +100,7 @@ class Vps extends CPHPDatabaseRecordClass { try { - $command = array("vzctl", "status", $this->sInternalId); + $command = array("sudo", "vzctl", "status", $this->sInternalId); $result = $this->sNode->ssh->RunCommandCached($command, false); @@ -276,7 +276,7 @@ class Vps extends CPHPDatabaseRecordClass { if(is_array($options)) { - $command_elements = array("vzctl", "set", $this->sInternalId); + $command_elements = array("sudo", "vzctl", "set", $this->sInternalId); foreach($options as $key => $value) { @@ -296,12 +296,12 @@ class Vps extends CPHPDatabaseRecordClass public function RunCommand($command, $throw_exception = false) { - return $this->sNode->ssh->RunCommand(array("vzctl", "exec", $this->sInternalId, $command), $throw_exception); + return $this->sNode->ssh->RunCommand(array("sudo", "vzctl", "exec", $this->sInternalId, $command), $throw_exception); } public function RunCommandCached($command, $throw_exception = false) { - return $this->sNode->ssh->RunCommandCached(array("vzctl", "exec", $this->sInternalId, $command), $throw_exception); + return $this->sNode->ssh->RunCommandCached(array("sudo", "vzctl", "exec", $this->sInternalId, $command), $throw_exception); } public function Deploy($conf = array()) @@ -311,7 +311,7 @@ class Vps extends CPHPDatabaseRecordClass $this->uRootPassword = $sRootPassword; $this->InsertIntoDatabase(); - $command = array("vzctl", "create", $this->sInternalId, "--ostemplate", $this->sTemplate->sTemplateName); + $command = array("sudo", "vzctl", "create", $this->sInternalId, "--ostemplate", $this->sTemplate->sTemplateName); $result = $this->sNode->ssh->RunCommand($command, false); $result->returncode = 0; @@ -347,7 +347,7 @@ class Vps extends CPHPDatabaseRecordClass $sDCacheLimit = (isset($conf['sDCacheLimit'])) ? $conf['sDCacheLimit'] : (int)($sDCache * 1.1); $sAvgProc = (isset($conf['sAvgProc'])) ? $conf['sAvgProc'] : $dummy_processes / 2; - $command = array("vzctl", "set", $this->sInternalId, + $command = array("sudo", "vzctl", "set", $this->sInternalId, "--onboot", "yes", "--setmode", "restart", "--hostname", $this->sHostname, @@ -441,7 +441,7 @@ class Vps extends CPHPDatabaseRecordClass $this->Stop(); } - $command = array("vzctl", "destroy", $this->sInternalId); + $command = array("sudo", "vzctl", "destroy", $this->sInternalId); $result = $this->sNode->ssh->RunCommand($command, false); if($result->returncode == 0) @@ -487,7 +487,7 @@ class Vps extends CPHPDatabaseRecordClass } else { - $command = array("vzctl", "start", $this->sInternalId); + $command = array("sudo", "vzctl", "start", $this->sInternalId); $result = $this->sNode->ssh->RunCommand($command, false); if($result->returncode == 0) @@ -515,7 +515,7 @@ class Vps extends CPHPDatabaseRecordClass } else { - $command = array("vzctl", "stop", $this->sInternalId); + $command = array("sudo", "vzctl", "stop", $this->sInternalId); $result = $this->sNode->ssh->RunCommand($command, false); /* vzctl is retarded enough to return code 0 when the command fails because the container isn't running, @@ -577,7 +577,7 @@ class Vps extends CPHPDatabaseRecordClass public function AddIp($ip) { - $command = array("vzctl", "set", $this->sInternalId, "--ipadd", $ip, "--save"); + $command = array("sudo", "vzctl", "set", $this->sInternalId, "--ipadd", $ip, "--save"); $result = $this->sNode->ssh->RunCommand($command, false); @@ -593,7 +593,7 @@ class Vps extends CPHPDatabaseRecordClass public function RemoveIp($ip) { - $command = array("vzctl", "set", $this->sInternalId, "--ipdel", $ip, "--save"); + $command = array("sudo", "vzctl", "set", $this->sInternalId, "--ipdel", $ip, "--save"); $result = $this->sNode->ssh->RunCommand($command, false); @@ -611,7 +611,7 @@ class Vps extends CPHPDatabaseRecordClass { /* TODO: Don't rely on grep, and parse the output in this function itself. Also try to find another way to do this without relying * on the container at all. */ - $result = $this->sNode->ssh->RunCommand(array("vzctl", "exec", $this->sInternalId, "cat /proc/net/dev | grep venet0"), false); + $result = $this->sNode->ssh->RunCommand(array("sudo", "vzctl", "exec", $this->sInternalId, "cat /proc/net/dev | grep venet0"), false); if($result->returncode == 0) { @@ -668,6 +668,6 @@ class Vps extends CPHPDatabaseRecordClass public function EnableTunTap() { /* TODO: Finish EnableTunTap function, check whether tun module is available on host */ - $command = array("vzctl", "set", $this->sInternalId, "--devnodes", "net/tun:rw", "--save"); + $command = array("sudo", "vzctl", "set", $this->sInternalId, "--devnodes", "net/tun:rw", "--save"); } } From 1d3aa1f7563a4f8a51fcd1227b06c3f8dfb5b9f0 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:20:21 +0100 Subject: [PATCH 07/14] Make the panel establish and use an SSH tunnel instead of using php-ssh2 --- frontend/classes/node.php | 9 +- frontend/classes/sshconnector.php | 279 ++++++++++++++++++++++++------ 2 files changed, 233 insertions(+), 55 deletions(-) diff --git a/frontend/classes/node.php b/frontend/classes/node.php index dd5e954..f20b822 100644 --- a/frontend/classes/node.php +++ b/frontend/classes/node.php @@ -26,10 +26,12 @@ class Node extends CPHPDatabaseRecordClass 'PhysicalLocation' => "PhysicalLocation", 'PrivateKey' => "CustomPrivateKey", 'PublicKey' => "CustomPublicKey", - 'User' => "User" + 'User' => "User", + 'TunnelKey' => "TunnelKey" ), 'numeric' => array( - 'Port' => "Port" + 'Port' => "Port", + 'TunnelPort' => "TunnelPort" ), 'boolean' => array( 'HasCustomKey' => "HasCustomKey" @@ -47,6 +49,9 @@ class Node extends CPHPDatabaseRecordClass $this->ssh->host = $this->sHostname; $this->ssh->port = $this->sPort; $this->ssh->user = $this->sUser; + $this->ssh->tunnel_port = $this->sTunnelPort; + $this->ssh->tunnel_key = $this->sTunnelKey; + $this->ssh->node = $this; if($this->HasCustomKey === true) { diff --git a/frontend/classes/sshconnector.php b/frontend/classes/sshconnector.php index 643e832..8d3045b 100644 --- a/frontend/classes/sshconnector.php +++ b/frontend/classes/sshconnector.php @@ -35,15 +35,6 @@ class SshConnector extends CPHPBaseClass { try { - if($this->failed == false && ($this->connected == false || $this->authenticated == false)) - { - $this->Connect(); - } - elseif($this->failed == true) - { - throw new SshConnectException("Previous connection attempt failed."); - } - return $this->DoCommand($command, $throw_exception); } catch (SshConnectException $e) @@ -78,81 +69,263 @@ class SshConnector extends CPHPBaseClass public function Connect() { - $fp = @fsockopen($this->host, $this->port, $errno, $errstr, 3); - - if(!$fp) + /* TODO: TIME_WAIT status for a previous socket on the same port may cause issues + * when attempting to restart the command daemon. There is currently no way + * to detect this from the code, and it makes all subsequent requests fail + * (silently?) because the tunnel is available but nothing is listening on + * the other end. This kind of edge case should be detected and dealt with. + * A browser displays a 'no data received' error in this case. */ + if($this->failed) { - throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$errstr}"); + throw new SshConnectException("A previous connection attempt failed."); + } + + $sHost = escapeshellarg($this->host); + $sUser = escapeshellarg($this->user); + $sPort = $this->tunnel_port = $this->node->uTunnelPort = $this->ChoosePort(); + $sKeyFile = escapeshellarg($this->key); + $this->node->uTunnelKey = $this->tunnel_key = random_string(16); + $sSessionKey = escapeshellarg($this->node->uTunnelKey); + + $command = "python /etc/cvm/start_tunnel.py {$sHost} {$sUser} {$sPort} {$sKeyFile} {$sSessionKey}"; + + $steps = array(); + + foreach(debug_backtrace() as $step) + { + try + { + $allargs = implode(", ", $step['args']); + } + catch (Exception $e) + { + $allargs = "[unserializable]"; + } + + $steps[] = "{$step['file']}:{$step['line']} => {$step['class']}{$step['type']}{$step['function']}({$allargs})"; } - fclose($fp); + cphp_debug_snapshot(array( + "action" => "start tunnel", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey, + "trace" => $steps + )); - $options = array( - 'hostkey' => $this->keytype - ); + exec($command, $output, $returncode); - if($this->connection = @ssh2_connect($this->host, $this->port, $options)) + if($returncode === 0) { - $this->connected = true; + /* autossh returns before the SSH connection has actually been established. We'll make the + * script sleep until a connection has been established, with a timeout of 10 seconds, after + * which an exception is raised. The polling interval is 100ms. */ - if(empty($this->passphrase)) + $start_time = time(); + + while(true) { - $result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key); + if(time() > $start_time + 10) + { + throw new SshConnectException("The SSH tunnel could not be fully established within the timeout period."); + } + + if($pollsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1)) + { + /* The tunnel has been fully established. */ + + fclose($pollsock); + break; + } + + usleep(100000); } - else + + cphp_debug_snapshot(array( + "action" => "pre insert db", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey + )); + + $this->node->InsertIntoDatabase(); + + cphp_debug_snapshot(array( + "action" => "inserted db", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey + )); + + return true; + } + else + { + throw new SshConnectException("Could not establish tunnel to {$this->host}:{$this->port}."); + } + } + + private function ChoosePort() + { + try + { + $sPorts = array(); + + foreach(Node::CreateFromQuery("SELECT * FROM nodes WHERE `TunnelPort` != 0") as $node) { - $result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key, $this->passphrase); + $sPorts[] = $node->sTunnelPort; + $sPorts[] = $node->sTunnelPort + 1; + $sPorts[] = $node->sTunnelPort + 2; } - if($result === true) + /* TODO: Figure out a more intelligent way of choosing ports. */ + $start = max($sPorts) + 1; + } + catch (NotFoundException $e) + { + $start = 2000; + } + + $current = $start; + + while(true) + { + if($current > 65534) + { + throw new SshConnectException("No free tunnel ports left."); + } + + if(!$this->TestPort($current)) { - $this->authenticated = true; - return true; + if(!$this->TestPort($current + 1)) + { + if(!$this->TestPort($current + 2)) + { + break; + } + else + { + $current = $current + 3; + } + } + else + { + $current = $current + 2; + } } else { - throw new SshAuthException("Could not connect to {$this->host}:{$this->port}: Key authentication failed"); + $current = $current + 1; } } + + return $current; + } + + private function TestPort($port) + { + if($testsock = @fsockopen("localhost", $port, $errno, $errstr, 1)) + { + fclose($testsock); + return true; + } else { - throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$error}"); + return false; } - - return false; } - private function DoCommand($command, $throw_exception) + private function DoCommand($command, $throw_exception, $allow_retry = true) { - $command = base64_encode(json_encode($command)); - $command = "{$this->helper} {$command}"; + cphp_debug_snapshot(array( + "action" => "pre run command", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "command" => $command, + "allow-retry" => $allow_retry + )); - $stream = ssh2_exec($this->connection, $command); - $error_stream = ssh2_fetch_stream($stream, SSH2_STREAM_STDERR); + $cmd = urlencode(json_encode($command)); + $url = "http://localhost:{$this->tunnel_port}/?key={$this->tunnel_key}&command={$cmd}"; - stream_set_blocking($stream, true); - stream_set_blocking($error_stream, true); - - $result = stream_get_contents($stream); - $error = stream_get_contents($error_stream); - - if(strpos($error, "No such file or directory") !== false) + $context = stream_context_create(array( + 'http' => array( + 'timeout' => 2.0 + ) + )); + + $response = @file($url, 0, $context); + + cphp_debug_snapshot(array( + "action" => "post run command", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "command" => $command, + "allow-retry" => $allow_retry, + "response" => $response + )); + + if($response === false) + { + /* Determine why the connection failed, and what we need to do to fix it. */ + if($testsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1)) + { + /* The socket works fine. */ + fclose($testsock); + + /* Since the socket works but we can't make a request, there is most + * likely a serious problem with the command daemon (stuck, crashed, + * etc.) We'll throw an exception. TODO: Log error. */ + $this->failed = true; + throw new SshCommandException("The command daemon is unavailable."); + } + else + { + /* The tunnel is gone for some reason. Either the connection broke + * and autossh is busy reconnecting, or autossh broke entirely. We + * will attempt to connect to the monitoring port to see if autossh + * is still running or not. */ + if($testsock = @fsockopen("localhost", ($this->tunnel_port + 2), $errno, $errstr, 1)) + { + /* The socket works fine. */ + fclose($testsock); + + /* Most likely autossh is very busy trying to reconnect to the node. We'll throw a + * connection exception for now. TODO: Consider waiting with a specified timeout. */ + $this->failed = true; + throw new SshConnectException("The SSH connection to this node is currently unavailable."); + } + else + { + if($allow_retry) + { + $this->Connect(); + $res = $this->DoCommand($command, $throw_exception, false); + } + else + { + $this->failed = true; + throw new SshConnectException("Could not connect to node."); + /* TODO: Log error, this is probably very bad. */ + } + } + } + } + else { - throw new Exception("The runhelper is not installed on the node or an error occurred."); + $response = json_decode(implode("", $response)); } - $returndata = json_decode($result); - - $returndata->stderr = trim($returndata->stderr); - - fclose($stream); - fclose($error_stream); - - if($returndata->returncode != 0 && $throw_exception === true) + if($response->returncode != 0 && $throw_exception === true) { - throw new SshExitException("Non-zero exit code returned: {$returndata->stderr}", $returndata->returncode); + throw new SshExitException("Non-zero exit code returned: {$response->stderr}", $response->returncode); } - return $returndata; + return $response; } } From fafa7ae8b5cadfee0e04d7c274eb4f32bb7b6ad6 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:22:07 +0100 Subject: [PATCH 08/14] We don't need this debugging code anymore --- frontend/classes/sshconnector.php | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/frontend/classes/sshconnector.php b/frontend/classes/sshconnector.php index 8d3045b..405d8a5 100644 --- a/frontend/classes/sshconnector.php +++ b/frontend/classes/sshconnector.php @@ -89,29 +89,12 @@ class SshConnector extends CPHPBaseClass $command = "python /etc/cvm/start_tunnel.py {$sHost} {$sUser} {$sPort} {$sKeyFile} {$sSessionKey}"; - $steps = array(); - - foreach(debug_backtrace() as $step) - { - try - { - $allargs = implode(", ", $step['args']); - } - catch (Exception $e) - { - $allargs = "[unserializable]"; - } - - $steps[] = "{$step['file']}:{$step['line']} => {$step['class']}{$step['type']}{$step['function']}({$allargs})"; - } - cphp_debug_snapshot(array( "action" => "start tunnel", "db-tunnelkey" => $this->node->sTunnelKey, "db-utunnelkey" => $this->node->uTunnelKey, "ssh-tunnelkey" => $this->tunnel_key, - "arg-tunnelkey" => $sSessionKey, - "trace" => $steps + "arg-tunnelkey" => $sSessionKey )); exec($command, $output, $returncode); From 3739b5b35ed4cc18f90b1eeb0b1940b10c2bb628 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 23 Mar 2013 04:22:52 +0100 Subject: [PATCH 09/14] Add todo entry --- frontend/classes/sshconnector.php | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/classes/sshconnector.php b/frontend/classes/sshconnector.php index 405d8a5..35f405d 100644 --- a/frontend/classes/sshconnector.php +++ b/frontend/classes/sshconnector.php @@ -178,6 +178,7 @@ class SshConnector extends CPHPBaseClass { if($current > 65534) { + /* TODO: We really need to deal with this properly... */ throw new SshConnectException("No free tunnel ports left."); } From 83bd0610c1058001372c3475a80d128fa8a83c5a Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 23 May 2013 20:14:21 +0200 Subject: [PATCH 10/14] Instead of showing an authorization error when the user is not logged in and tries to visit /, redirect them to the login page. --- frontend/modules/client/vps/list.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/modules/client/vps/list.php b/frontend/modules/client/vps/list.php index 049b7c5..a2ddec5 100644 --- a/frontend/modules/client/vps/list.php +++ b/frontend/modules/client/vps/list.php @@ -53,5 +53,5 @@ if($sLoggedIn === true) } else { - throw new UnauthorizedException("You must be logged in to view this page."); + redirect("/login"); } From d6d1804fa259c61ac27337470f087c085ef4eeed Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 23 May 2013 20:15:32 +0200 Subject: [PATCH 11/14] Make some warnings go away --- frontend/rewrite.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/rewrite.php b/frontend/rewrite.php index 7201b4d..177d286 100644 --- a/frontend/rewrite.php +++ b/frontend/rewrite.php @@ -239,7 +239,7 @@ try if(empty($router->uVariables['raw'])) { - if($router->uVariables['menu'] == "vps" && $router->uVariables['display_menu'] === true) + if(isset($router->uVariables['menu']) && $router->uVariables['menu'] == "vps" && $router->uVariables['display_menu'] === true) { $sMainContents .= Templater::AdvancedParse("{$sTheme}/client/vps/main", $locale->strings, array( 'error' => $sError, @@ -247,7 +247,7 @@ try 'id' => $sVps->sId )); } - elseif($router->uVariables['menu'] == "admin" && $router->uVariables['display_menu'] === true) + elseif(isset($router->uVariables['menu']) && $router->uVariables['menu'] == "admin" && $router->uVariables['display_menu'] === true) { $sMainContents .= Templater::AdvancedParse("{$sTheme}/admin/main", $locale->strings, array( 'error' => $sError, From 454d21946a5dc8f22dac4cc7bb4a6fa66ce28622 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 23 May 2013 21:52:26 +0200 Subject: [PATCH 12/14] Only attempt to iterate through VPS list query results when the result isn't empty --- frontend/modules/client/vps/list.php | 51 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/frontend/modules/client/vps/list.php b/frontend/modules/client/vps/list.php index a2ddec5..9aaa799 100644 --- a/frontend/modules/client/vps/list.php +++ b/frontend/modules/client/vps/list.php @@ -15,36 +15,37 @@ if(!isset($_CVM)) { die("Unauthorized."); } if($sLoggedIn === true) { - $result = $database->CachedQuery("SELECT * FROM containers WHERE `UserId` = :UserId", array(":UserId" => $sUser->sId)); - $sVpsList = array(); - foreach($result->data as $row) + if($result = $database->CachedQuery("SELECT * FROM containers WHERE `UserId` = :UserId", array(":UserId" => $sUser->sId))) { - $sVps = new Vps($row); - - try - { - $sStatus = $sVps->sStatusText; - } - catch (SshException $e) + foreach($result->data as $row) { - $sStatus = "unknown"; + $sVps = new Vps($row); + + try + { + $sStatus = $sVps->sStatusText; + } + catch (SshException $e) + { + $sStatus = "unknown"; + } + + $sVpsList[] = array( + 'id' => $sVps->sId, + 'hostname' => $sVps->sHostname, + 'node' => $sVps->sNode->sName, + 'node-hostname' => $sVps->sNode->sHostname, + 'template' => $sVps->sTemplate->sName, + 'diskspace' => number_format($sVps->sDiskSpace / 1024), + 'diskspace-unit' => "GB", + 'guaranteed-ram' => $sVps->sGuaranteedRam, + 'guaranteed-ram-unit' => "MB", + 'status' => $sStatus, + 'virtualization-type' => $sVps->sVirtualizationType + ); } - - $sVpsList[] = array( - 'id' => $sVps->sId, - 'hostname' => $sVps->sHostname, - 'node' => $sVps->sNode->sName, - 'node-hostname' => $sVps->sNode->sHostname, - 'template' => $sVps->sTemplate->sName, - 'diskspace' => number_format($sVps->sDiskSpace / 1024), - 'diskspace-unit' => "GB", - 'guaranteed-ram' => $sVps->sGuaranteedRam, - 'guaranteed-ram-unit' => "MB", - 'status' => $sStatus, - 'virtualization-type' => $sVps->sVirtualizationType - ); } $sMainContents = Templater::AdvancedParse("{$sTheme}/client/vps/list", $locale->strings, array( From b702923505803fde3047c52f1b3c84e85c10548f Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 23 May 2013 22:20:41 +0200 Subject: [PATCH 13/14] Fix a warning --- frontend/modules/admin/node/lookup.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/modules/admin/node/lookup.php b/frontend/modules/admin/node/lookup.php index 7b4e204..1f0ad65 100644 --- a/frontend/modules/admin/node/lookup.php +++ b/frontend/modules/admin/node/lookup.php @@ -17,6 +17,8 @@ try { $sNode = new Node($router->uParameters[1]); + $sVpsList = array(); + if($result = $database->CachedQuery("SELECT * FROM containers WHERE `NodeId` = :NodeId", array(":NodeId" => $sNode->sId))) { foreach($result->data as $row) From 236f5d2c4d1c54e0d11ccfcab65143d42dc160c1 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 23 May 2013 22:22:24 +0200 Subject: [PATCH 14/14] Fix another notice --- frontend/modules/admin/vps/create.php | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/frontend/modules/admin/vps/create.php b/frontend/modules/admin/vps/create.php index dff7679..23e2b15 100644 --- a/frontend/modules/admin/vps/create.php +++ b/frontend/modules/admin/vps/create.php @@ -172,18 +172,23 @@ foreach($result->data as $row) ); } -$result = $database->CachedQuery("SELECT * FROM templates WHERE `Available` = 1"); - $sTemplates = array(); -foreach($result->data as $row) +if($result = $database->CachedQuery("SELECT * FROM templates WHERE `Available` = 1")) { - $sTemplate = new Template($row); - - $sTemplates[] = array( - 'id' => $sTemplate->sId, - 'name' => $sTemplate->sName - ); + foreach($result->data as $row) + { + $sTemplate = new Template($row); + + $sTemplates[] = array( + 'id' => $sTemplate->sId, + 'name' => $sTemplate->sName + ); + } +} +else +{ + /* TODO: Show an error when no templates are available. */ } $result = $database->CachedQuery("SELECT * FROM users WHERE `AccessLevel` > 0");