|
|
|
@ -35,15 +35,6 @@ class SshConnector extends CPHPBaseClass
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
if($this->failed == false && ($this->connected == false || $this->authenticated == false))
|
|
|
|
|
{
|
|
|
|
|
$this->Connect();
|
|
|
|
|
}
|
|
|
|
|
elseif($this->failed == true)
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("Previous connection attempt failed.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $this->DoCommand($command, $throw_exception);
|
|
|
|
|
}
|
|
|
|
|
catch (SshConnectException $e)
|
|
|
|
@ -78,81 +69,263 @@ class SshConnector extends CPHPBaseClass
|
|
|
|
|
|
|
|
|
|
public function Connect()
|
|
|
|
|
{
|
|
|
|
|
$fp = @fsockopen($this->host, $this->port, $errno, $errstr, 3);
|
|
|
|
|
|
|
|
|
|
if(!$fp)
|
|
|
|
|
/* TODO: TIME_WAIT status for a previous socket on the same port may cause issues
|
|
|
|
|
* when attempting to restart the command daemon. There is currently no way
|
|
|
|
|
* to detect this from the code, and it makes all subsequent requests fail
|
|
|
|
|
* (silently?) because the tunnel is available but nothing is listening on
|
|
|
|
|
* the other end. This kind of edge case should be detected and dealt with.
|
|
|
|
|
* A browser displays a 'no data received' error in this case. */
|
|
|
|
|
if($this->failed)
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$errstr}");
|
|
|
|
|
throw new SshConnectException("A previous connection attempt failed.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$sHost = escapeshellarg($this->host);
|
|
|
|
|
$sUser = escapeshellarg($this->user);
|
|
|
|
|
$sPort = $this->tunnel_port = $this->node->uTunnelPort = $this->ChoosePort();
|
|
|
|
|
$sKeyFile = escapeshellarg($this->key);
|
|
|
|
|
$this->node->uTunnelKey = $this->tunnel_key = random_string(16);
|
|
|
|
|
$sSessionKey = escapeshellarg($this->node->uTunnelKey);
|
|
|
|
|
|
|
|
|
|
$command = "python /etc/cvm/start_tunnel.py {$sHost} {$sUser} {$sPort} {$sKeyFile} {$sSessionKey}";
|
|
|
|
|
|
|
|
|
|
$steps = array();
|
|
|
|
|
|
|
|
|
|
foreach(debug_backtrace() as $step)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
$allargs = implode(", ", $step['args']);
|
|
|
|
|
}
|
|
|
|
|
catch (Exception $e)
|
|
|
|
|
{
|
|
|
|
|
$allargs = "[unserializable]";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$steps[] = "{$step['file']}:{$step['line']} => {$step['class']}{$step['type']}{$step['function']}({$allargs})";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fclose($fp);
|
|
|
|
|
cphp_debug_snapshot(array(
|
|
|
|
|
"action" => "start tunnel",
|
|
|
|
|
"db-tunnelkey" => $this->node->sTunnelKey,
|
|
|
|
|
"db-utunnelkey" => $this->node->uTunnelKey,
|
|
|
|
|
"ssh-tunnelkey" => $this->tunnel_key,
|
|
|
|
|
"arg-tunnelkey" => $sSessionKey,
|
|
|
|
|
"trace" => $steps
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
$options = array(
|
|
|
|
|
'hostkey' => $this->keytype
|
|
|
|
|
);
|
|
|
|
|
exec($command, $output, $returncode);
|
|
|
|
|
|
|
|
|
|
if($this->connection = @ssh2_connect($this->host, $this->port, $options))
|
|
|
|
|
if($returncode === 0)
|
|
|
|
|
{
|
|
|
|
|
$this->connected = true;
|
|
|
|
|
/* autossh returns before the SSH connection has actually been established. We'll make the
|
|
|
|
|
* script sleep until a connection has been established, with a timeout of 10 seconds, after
|
|
|
|
|
* which an exception is raised. The polling interval is 100ms. */
|
|
|
|
|
|
|
|
|
|
if(empty($this->passphrase))
|
|
|
|
|
$start_time = time();
|
|
|
|
|
|
|
|
|
|
while(true)
|
|
|
|
|
{
|
|
|
|
|
$result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key);
|
|
|
|
|
if(time() > $start_time + 10)
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("The SSH tunnel could not be fully established within the timeout period.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if($pollsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1))
|
|
|
|
|
{
|
|
|
|
|
/* The tunnel has been fully established. */
|
|
|
|
|
|
|
|
|
|
fclose($pollsock);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
usleep(100000);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
|
|
|
|
|
cphp_debug_snapshot(array(
|
|
|
|
|
"action" => "pre insert db",
|
|
|
|
|
"db-tunnelkey" => $this->node->sTunnelKey,
|
|
|
|
|
"db-utunnelkey" => $this->node->uTunnelKey,
|
|
|
|
|
"ssh-tunnelkey" => $this->tunnel_key,
|
|
|
|
|
"arg-tunnelkey" => $sSessionKey
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
$this->node->InsertIntoDatabase();
|
|
|
|
|
|
|
|
|
|
cphp_debug_snapshot(array(
|
|
|
|
|
"action" => "inserted db",
|
|
|
|
|
"db-tunnelkey" => $this->node->sTunnelKey,
|
|
|
|
|
"db-utunnelkey" => $this->node->uTunnelKey,
|
|
|
|
|
"ssh-tunnelkey" => $this->tunnel_key,
|
|
|
|
|
"arg-tunnelkey" => $sSessionKey
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("Could not establish tunnel to {$this->host}:{$this->port}.");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private function ChoosePort()
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
$sPorts = array();
|
|
|
|
|
|
|
|
|
|
foreach(Node::CreateFromQuery("SELECT * FROM nodes WHERE `TunnelPort` != 0") as $node)
|
|
|
|
|
{
|
|
|
|
|
$result = @ssh2_auth_pubkey_file($this->connection, $this->user, $this->pubkey, $this->key, $this->passphrase);
|
|
|
|
|
$sPorts[] = $node->sTunnelPort;
|
|
|
|
|
$sPorts[] = $node->sTunnelPort + 1;
|
|
|
|
|
$sPorts[] = $node->sTunnelPort + 2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if($result === true)
|
|
|
|
|
/* TODO: Figure out a more intelligent way of choosing ports. */
|
|
|
|
|
$start = max($sPorts) + 1;
|
|
|
|
|
}
|
|
|
|
|
catch (NotFoundException $e)
|
|
|
|
|
{
|
|
|
|
|
$start = 2000;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$current = $start;
|
|
|
|
|
|
|
|
|
|
while(true)
|
|
|
|
|
{
|
|
|
|
|
if($current > 65534)
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("No free tunnel ports left.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if(!$this->TestPort($current))
|
|
|
|
|
{
|
|
|
|
|
$this->authenticated = true;
|
|
|
|
|
return true;
|
|
|
|
|
if(!$this->TestPort($current + 1))
|
|
|
|
|
{
|
|
|
|
|
if(!$this->TestPort($current + 2))
|
|
|
|
|
{
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
$current = $current + 3;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
$current = $current + 2;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
throw new SshAuthException("Could not connect to {$this->host}:{$this->port}: Key authentication failed");
|
|
|
|
|
$current = $current + 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $current;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private function TestPort($port)
|
|
|
|
|
{
|
|
|
|
|
if($testsock = @fsockopen("localhost", $port, $errno, $errstr, 1))
|
|
|
|
|
{
|
|
|
|
|
fclose($testsock);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
throw new SshConnectException("Could not connect to {$this->host}:{$this->port}: {$error}");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private function DoCommand($command, $throw_exception)
|
|
|
|
|
private function DoCommand($command, $throw_exception, $allow_retry = true)
|
|
|
|
|
{
|
|
|
|
|
$command = base64_encode(json_encode($command));
|
|
|
|
|
$command = "{$this->helper} {$command}";
|
|
|
|
|
cphp_debug_snapshot(array(
|
|
|
|
|
"action" => "pre run command",
|
|
|
|
|
"db-tunnelkey" => $this->node->sTunnelKey,
|
|
|
|
|
"db-utunnelkey" => $this->node->uTunnelKey,
|
|
|
|
|
"ssh-tunnelkey" => $this->tunnel_key,
|
|
|
|
|
"command" => $command,
|
|
|
|
|
"allow-retry" => $allow_retry
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
$stream = ssh2_exec($this->connection, $command);
|
|
|
|
|
$error_stream = ssh2_fetch_stream($stream, SSH2_STREAM_STDERR);
|
|
|
|
|
$cmd = urlencode(json_encode($command));
|
|
|
|
|
$url = "http://localhost:{$this->tunnel_port}/?key={$this->tunnel_key}&command={$cmd}";
|
|
|
|
|
|
|
|
|
|
stream_set_blocking($stream, true);
|
|
|
|
|
stream_set_blocking($error_stream, true);
|
|
|
|
|
|
|
|
|
|
$result = stream_get_contents($stream);
|
|
|
|
|
$error = stream_get_contents($error_stream);
|
|
|
|
|
|
|
|
|
|
if(strpos($error, "No such file or directory") !== false)
|
|
|
|
|
$context = stream_context_create(array(
|
|
|
|
|
'http' => array(
|
|
|
|
|
'timeout' => 2.0
|
|
|
|
|
)
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
$response = @file($url, 0, $context);
|
|
|
|
|
|
|
|
|
|
cphp_debug_snapshot(array(
|
|
|
|
|
"action" => "post run command",
|
|
|
|
|
"db-tunnelkey" => $this->node->sTunnelKey,
|
|
|
|
|
"db-utunnelkey" => $this->node->uTunnelKey,
|
|
|
|
|
"ssh-tunnelkey" => $this->tunnel_key,
|
|
|
|
|
"command" => $command,
|
|
|
|
|
"allow-retry" => $allow_retry,
|
|
|
|
|
"response" => $response
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
if($response === false)
|
|
|
|
|
{
|
|
|
|
|
/* Determine why the connection failed, and what we need to do to fix it. */
|
|
|
|
|
if($testsock = @fsockopen("localhost", $this->tunnel_port, $errno, $errstr, 1))
|
|
|
|
|
{
|
|
|
|
|
/* The socket works fine. */
|
|
|
|
|
fclose($testsock);
|
|
|
|
|
|
|
|
|
|
/* Since the socket works but we can't make a request, there is most
|
|
|
|
|
* likely a serious problem with the command daemon (stuck, crashed,
|
|
|
|
|
* etc.) We'll throw an exception. TODO: Log error. */
|
|
|
|
|
$this->failed = true;
|
|
|
|
|
throw new SshCommandException("The command daemon is unavailable.");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
/* The tunnel is gone for some reason. Either the connection broke
|
|
|
|
|
* and autossh is busy reconnecting, or autossh broke entirely. We
|
|
|
|
|
* will attempt to connect to the monitoring port to see if autossh
|
|
|
|
|
* is still running or not. */
|
|
|
|
|
if($testsock = @fsockopen("localhost", ($this->tunnel_port + 2), $errno, $errstr, 1))
|
|
|
|
|
{
|
|
|
|
|
/* The socket works fine. */
|
|
|
|
|
fclose($testsock);
|
|
|
|
|
|
|
|
|
|
/* Most likely autossh is very busy trying to reconnect to the node. We'll throw a
|
|
|
|
|
* connection exception for now. TODO: Consider waiting with a specified timeout. */
|
|
|
|
|
$this->failed = true;
|
|
|
|
|
throw new SshConnectException("The SSH connection to this node is currently unavailable.");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
if($allow_retry)
|
|
|
|
|
{
|
|
|
|
|
$this->Connect();
|
|
|
|
|
$res = $this->DoCommand($command, $throw_exception, false);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
$this->failed = true;
|
|
|
|
|
throw new SshConnectException("Could not connect to node.");
|
|
|
|
|
/* TODO: Log error, this is probably very bad. */
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
throw new Exception("The runhelper is not installed on the node or an error occurred.");
|
|
|
|
|
$response = json_decode(implode("", $response));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$returndata = json_decode($result);
|
|
|
|
|
|
|
|
|
|
$returndata->stderr = trim($returndata->stderr);
|
|
|
|
|
|
|
|
|
|
fclose($stream);
|
|
|
|
|
fclose($error_stream);
|
|
|
|
|
|
|
|
|
|
if($returndata->returncode != 0 && $throw_exception === true)
|
|
|
|
|
if($response->returncode != 0 && $throw_exception === true)
|
|
|
|
|
{
|
|
|
|
|
throw new SshExitException("Non-zero exit code returned: {$returndata->stderr}", $returndata->returncode);
|
|
|
|
|
throw new SshExitException("Non-zero exit code returned: {$response->stderr}", $response->returncode);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $returndata;
|
|
|
|
|
return $response;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|