diff --git a/frontend/classes/node.php b/frontend/classes/node.php index dd5e954..f20b822 100644 --- a/frontend/classes/node.php +++ b/frontend/classes/node.php @@ -26,10 +26,12 @@ class Node extends CPHPDatabaseRecordClass 'PhysicalLocation' => "PhysicalLocation", 'PrivateKey' => "CustomPrivateKey", 'PublicKey' => "CustomPublicKey", - 'User' => "User" + 'User' => "User", + 'TunnelKey' => "TunnelKey" ), 'numeric' => array( - 'Port' => "Port" + 'Port' => "Port", + 'TunnelPort' => "TunnelPort" ), 'boolean' => array( 'HasCustomKey' => "HasCustomKey" @@ -47,6 +49,9 @@ class Node extends CPHPDatabaseRecordClass $this->ssh->host = $this->sHostname; $this->ssh->port = $this->sPort; $this->ssh->user = $this->sUser; + $this->ssh->tunnel_port = $this->sTunnelPort; + $this->ssh->tunnel_key = $this->sTunnelKey; + $this->ssh->node = $this; if($this->HasCustomKey === true) { diff --git a/frontend/classes/sshconnector.php b/frontend/classes/sshconnector.php index 643e832..8d3045b 100644 --- a/frontend/classes/sshconnector.php +++ b/frontend/classes/sshconnector.php @@ -35,15 +35,6 @@ class SshConnector extends CPHPBaseClass { try { - if($this->failed == false && ($this->connected == false || $this->authenticated == false)) - { - $this->Connect(); - } - elseif($this->failed == true) - { - throw new SshConnectException("Previous connection attempt failed."); - } - return $this->DoCommand($command, $throw_exception); } catch (SshConnectException $e) @@ -78,81 +69,263 @@ class SshConnector extends CPHPBaseClass public function Connect() { - $fp = @fsockopen($this->host, $this->port, $errno, $errstr, 3); - - if(!$fp) + /* TODO: TIME_WAIT status for a previous socket on the same port may cause issues + * when attempting to restart the command daemon. There is currently no way + * to detect this from the code, and it makes all subsequent requests fail + * (silently?) because the tunnel is available but nothing is listening on + * the other end. This kind of edge case should be detected and dealt with. + * A browser displays a 'no data received' error in this case. */ + if($this->failed) { - throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$errstr}"); + throw new SshConnectException("A previous connection attempt failed."); + } + + $sHost = escapeshellarg($this->host); + $sUser = escapeshellarg($this->user); + $sPort = $this->tunnel_port = $this->node->uTunnelPort = $this->ChoosePort(); + $sKeyFile = escapeshellarg($this->key); + $this->node->uTunnelKey = $this->tunnel_key = random_string(16); + $sSessionKey = escapeshellarg($this->node->uTunnelKey); + + $command = "python /etc/cvm/start_tunnel.py {$sHost} {$sUser} {$sPort} {$sKeyFile} {$sSessionKey}"; + + $steps = array(); + + foreach(debug_backtrace() as $step) + { + try + { + $allargs = implode(", ", $step['args']); + } + catch (Exception $e) + { + $allargs = "[unserializable]"; + } + + $steps[] = "{$step['file']}:{$step['line']} => {$step['class']}{$step['type']}{$step['function']}({$allargs})"; } - fclose($fp); + cphp_debug_snapshot(array( + "action" => "start tunnel", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey, + "trace" => $steps + )); - $options = array( - 'hostkey' => $this->keytype - ); + exec($command, $output, $returncode); - if($this->connection = @ssh2_connect($this->host, $this->port, $options)) + if($returncode === 0) { - $this->connected = true; + /* autossh returns before the SSH connection has actually been established. We'll make the + * script sleep until a connection has been established, with a timeout of 10 seconds, after + * which an exception is raised. The polling interval is 100ms. */ - if(empty($this->passphrase)) + $start_time = time(); + + while(true) { - $result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key); + if(time() > $start_time + 10) + { + throw new SshConnectException("The SSH tunnel could not be fully established within the timeout period."); + } + + if($pollsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1)) + { + /* The tunnel has been fully established. */ + + fclose($pollsock); + break; + } + + usleep(100000); } - else + + cphp_debug_snapshot(array( + "action" => "pre insert db", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey + )); + + $this->node->InsertIntoDatabase(); + + cphp_debug_snapshot(array( + "action" => "inserted db", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "arg-tunnelkey" => $sSessionKey + )); + + return true; + } + else + { + throw new SshConnectException("Could not establish tunnel to {$this->host}:{$this->port}."); + } + } + + private function ChoosePort() + { + try + { + $sPorts = array(); + + foreach(Node::CreateFromQuery("SELECT * FROM nodes WHERE `TunnelPort` != 0") as $node) { - $result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key, $this->passphrase); + $sPorts[] = $node->sTunnelPort; + $sPorts[] = $node->sTunnelPort + 1; + $sPorts[] = $node->sTunnelPort + 2; } - if($result === true) + /* TODO: Figure out a more intelligent way of choosing ports. */ + $start = max($sPorts) + 1; + } + catch (NotFoundException $e) + { + $start = 2000; + } + + $current = $start; + + while(true) + { + if($current > 65534) + { + throw new SshConnectException("No free tunnel ports left."); + } + + if(!$this->TestPort($current)) { - $this->authenticated = true; - return true; + if(!$this->TestPort($current + 1)) + { + if(!$this->TestPort($current + 2)) + { + break; + } + else + { + $current = $current + 3; + } + } + else + { + $current = $current + 2; + } } else { - throw new SshAuthException("Could not connect to {$this->host}:{$this->port}: Key authentication failed"); + $current = $current + 1; } } + + return $current; + } + + private function TestPort($port) + { + if($testsock = @fsockopen("localhost", $port, $errno, $errstr, 1)) + { + fclose($testsock); + return true; + } else { - throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$error}"); + return false; } - - return false; } - private function DoCommand($command, $throw_exception) + private function DoCommand($command, $throw_exception, $allow_retry = true) { - $command = base64_encode(json_encode($command)); - $command = "{$this->helper} {$command}"; + cphp_debug_snapshot(array( + "action" => "pre run command", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "command" => $command, + "allow-retry" => $allow_retry + )); - $stream = ssh2_exec($this->connection, $command); - $error_stream = ssh2_fetch_stream($stream, SSH2_STREAM_STDERR); + $cmd = urlencode(json_encode($command)); + $url = "http://localhost:{$this->tunnel_port}/?key={$this->tunnel_key}&command={$cmd}"; - stream_set_blocking($stream, true); - stream_set_blocking($error_stream, true); - - $result = stream_get_contents($stream); - $error = stream_get_contents($error_stream); - - if(strpos($error, "No such file or directory") !== false) + $context = stream_context_create(array( + 'http' => array( + 'timeout' => 2.0 + ) + )); + + $response = @file($url, 0, $context); + + cphp_debug_snapshot(array( + "action" => "post run command", + "db-tunnelkey" => $this->node->sTunnelKey, + "db-utunnelkey" => $this->node->uTunnelKey, + "ssh-tunnelkey" => $this->tunnel_key, + "command" => $command, + "allow-retry" => $allow_retry, + "response" => $response + )); + + if($response === false) + { + /* Determine why the connection failed, and what we need to do to fix it. */ + if($testsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1)) + { + /* The socket works fine. */ + fclose($testsock); + + /* Since the socket works but we can't make a request, there is most + * likely a serious problem with the command daemon (stuck, crashed, + * etc.) We'll throw an exception. TODO: Log error. */ + $this->failed = true; + throw new SshCommandException("The command daemon is unavailable."); + } + else + { + /* The tunnel is gone for some reason. Either the connection broke + * and autossh is busy reconnecting, or autossh broke entirely. We + * will attempt to connect to the monitoring port to see if autossh + * is still running or not. */ + if($testsock = @fsockopen("localhost", ($this->tunnel_port + 2), $errno, $errstr, 1)) + { + /* The socket works fine. */ + fclose($testsock); + + /* Most likely autossh is very busy trying to reconnect to the node. We'll throw a + * connection exception for now. TODO: Consider waiting with a specified timeout. */ + $this->failed = true; + throw new SshConnectException("The SSH connection to this node is currently unavailable."); + } + else + { + if($allow_retry) + { + $this->Connect(); + $res = $this->DoCommand($command, $throw_exception, false); + } + else + { + $this->failed = true; + throw new SshConnectException("Could not connect to node."); + /* TODO: Log error, this is probably very bad. */ + } + } + } + } + else { - throw new Exception("The runhelper is not installed on the node or an error occurred."); + $response = json_decode(implode("", $response)); } - $returndata = json_decode($result); - - $returndata->stderr = trim($returndata->stderr); - - fclose($stream); - fclose($error_stream); - - if($returndata->returncode != 0 && $throw_exception === true) + if($response->returncode != 0 && $throw_exception === true) { - throw new SshExitException("Non-zero exit code returned: {$returndata->stderr}", $returndata->returncode); + throw new SshExitException("Non-zero exit code returned: {$response->stderr}", $response->returncode); } - return $returndata; + return $response; } }