[RESOLVED] Retrieving info from a remote page using Snoopy and CURL

padanaram · May 1, 2007

I am trying to use snoopy to log into a remote web site that is on SSL, but my host does not give me access to the CURL binary on the server and snoopy needs it to work. Snoopy does not use the CURL function because when snoopy was written the CURL function was unstable.

For me to be able to acces the remote site I have to go through my web hosts proxy. At the remote site there is a form that gets submitted that brings up the results, but it is a $_GET method so I can just go to the page with the URL the form produces and get the info.

I may have to follow some links as well to retrieve certain info from those pages.

Does anyone know of another class or have some code or suggestions that may help?

Here is the code for the SNOOPY function (from the class) that does the connection through https. As I said above it is written to use the the CURL binary, but on the server I am on I am on a restricted shell and do not have access to the CURL binary. I need to either rewrite this function to use the CURL function or I need to find another class or different code that already does that.

This is the snoopy function.

function _httpsrequest($url,$URI,$http_method,$content_type="",$body="")
	{
		if($this->passcookies && $this->_redirectaddr)
			$this->setcookies();

	$headers = array();		

	$URI_PARTS = parse_url($URI);
	if(empty($url))
		$url = "/";
	// GET ... header not needed for curl
	//$headers[] = $http_method." ".$url." ".$this->_httpversion;		
	if(!empty($this->agent))
		$headers[] = "User-Agent: ".$this->agent;
	if(!empty($this->host))
		if(!empty($this->port))
			$headers[] = "Host: ".$this->host.":".$this->port;
		else
			$headers[] = "Host: ".$this->host;
	if(!empty($this->accept))
		$headers[] = "Accept: ".$this->accept;
	if(!empty($this->referer))
		$headers[] = "Referer: ".$this->referer;
	if(!empty($this->cookies))
	{			
		if(!is_array($this->cookies))
			$this->cookies = (array)$this->cookies;

		reset($this->cookies);
		if ( count($this->cookies) > 0 ) {
			$cookie_str = 'Cookie: ';
			foreach ( $this->cookies as $cookieKey => $cookieVal ) {
			$cookie_str .= $cookieKey."=".urlencode($cookieVal)."; ";
			}
			$headers[] = substr($cookie_str,0,-2);
		}
	}
	if(!empty($this->rawheaders))
	{
		if(!is_array($this->rawheaders))
			$this->rawheaders = (array)$this->rawheaders;
		while(list($headerKey,$headerVal) = each($this->rawheaders))
			$headers[] = $headerKey.": ".$headerVal;
	}
	if(!empty($content_type)) {
		if ($content_type == "multipart/form-data")
			$headers[] = "Content-type: $content_type; boundary=".$this->_mime_boundary;
		else
			$headers[] = "Content-type: $content_type";
	}
	if(!empty($body))	
		$headers[] = "Content-length: ".strlen($body);
	if(!empty($this->user) || !empty($this->pass))	
		$headers[] = "Authorization: BASIC ".base64_encode($this->user.":".$this->pass);

	for($curr_header = 0; $curr_header < count($headers); $curr_header++) {
		$safer_header = strtr( $headers[$curr_header], "\"", " " );
		$cmdline_params .= " -H \"".$safer_header."\"";
	}

	if(!empty($body))
		$cmdline_params .= " -d \"$body\"";

	if($this->read_timeout > 0)
		$cmdline_params .= " -m ".$this->read_timeout;

	$headerfile = tempnam($temp_dir, "sno");

	$safer_URI = strtr( $URI, "\"", " " ); // strip quotes from the URI to avoid shell access
	exec($this->curl_path." -D \"$headerfile\"".$cmdline_params." \"".$safer_URI."\"",$results,$return);

	if($return)
	{
		$this->error = "Error: cURL could not retrieve the document, error $return.";
		return false;
	}


	$results = implode("\r\n",$results);

	$result_headers = file("$headerfile");

	$this->_redirectaddr = false;
	unset($this->headers);

	for($currentHeader = 0; $currentHeader < count($result_headers); $currentHeader++)
	{

		// if a header begins with Location: or URI:, set the redirect
		if(preg_match("/^(Location: |URI: )/i",$result_headers[$currentHeader]))
		{
			// get URL portion of the redirect
			preg_match("/^(Location: |URI:)\s+(.*)/",chop($result_headers[$currentHeader]),$matches);
			// look for :// in the Location header to see if hostname is included
			if(!preg_match("|\:\/\/|",$matches[2]))
			{
				// no host in the path, so prepend
				$this->_redirectaddr = $URI_PARTS["scheme"]."://".$this->host.":".$this->port;
				// eliminate double slash
				if(!preg_match("|^/|",$matches[2]))
						$this->_redirectaddr .= "/".$matches[2];
				else
						$this->_redirectaddr .= $matches[2];
			}
			else
				$this->_redirectaddr = $matches[2];
		}

		if(preg_match("|^HTTP/|",$result_headers[$currentHeader]))
			$this->response_code = $result_headers[$currentHeader];

		$this->headers[] = $result_headers[$currentHeader];
	}

	// check if there is a a redirect meta tag

	if(preg_match("'<meta[\s]*http-equiv[^>]*?content[\s]*=[\s]*[\"\']?\d+;[\s]*URL[\s]*=[\s]*([^\"\']*?)[\"\']?>'i",$results,$match))
	{
		$this->_redirectaddr = $this->_expandlinks($match[1],$URI);	
	}

	// have we hit our frame depth and is there frame src to fetch?
	if(($this->_framedepth < $this->maxframes) && preg_match_all("'<frame\s+.*src[\s]*=[\'\"]?([^\'\"\>]+)'i",$results,$match))
	{
		$this->results[] = $results;
		for($x=0; $x<count($match[1]); $x++)
			$this->_frameurls[] = $this->_expandlinks($match[1][$x],$URI_PARTS["scheme"]."://".$this->host);
	}
	// have we already fetched framed content?
	elseif(is_array($this->results))
		$this->results[] = $results;
	// no framed content
	else
		$this->results = $results;

	unlink("$headerfile");

	return true;
}

I have some code that will log into the https server and will display the page that I need to follow links on. It uses the CURL function.

$URL="https://username:password@www.domain.com/members/powersearch/control/interresults?fromyear=1950&toyear=2008&region=4&make=&modeltext=&auction=&model=&numresultsperpage=2147483647&mileage=&interior=&engine=&top=&transmission=&radio=&certification=&consignor=&presalechannel=1&dealerexchangechannel=5&cyberlotchannel=2&cyberauctionchannel=4&encorechannel=3&numresultsperpage=21474836470&x=55&y=7";
if (isset($_GET["site"])) { $URL = $_GET["site"]; }
$ch = curl_init();
echo "URL = $URL <br>\n";
curl_setopt($ch, CURLOPT_VERBOSE, 1);
//curl_setopt ($ch, CURLOPT_HTTPPROXYTUNNEL, TRUE);
curl_setopt ($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
curl_setopt ($ch, CURLOPT_PROXY,"http://proxy.shr.secureserver.net:3128");
curl_setopt ($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt ($ch, CURLOPT_URL, $URL);
curl_setopt ($ch, CURLOPT_TIMEOUT, 120);
$result = curl_exec ($ch);
echo "<hr><br>\n";
echo 'Errors: ' . curl_errno($ch) . ' ' . curl_error($ch) . '<br><br>';
echo "<hr><br>\n";
curl_close ($ch);
print "result - $result";
echo "<hr><br>\n";

Greg