Hi!
I am using cURL to get info from remote server. I am using proxies. I have a lot of proxies in my database and I am telling the script to choose only one of them for each downloaded page (The script is downloading few pages). I have also set MAX TIME for the curl functions which means, that if the script starts downloading via one proxy and it reaches the time limit, it deletes that proxy form the proxy array and select randomly new one from the array (from which the slow or not responding previous proxy is removed) to try again. I do not know what and why the hell is going on, but when I start the scripts each one of them is executing between minutes and HOURS and deisplays me that it had tried between 500 and 16000 tries of proxies before it success to finish its work! (Note, that the total number of proxies I have in the arrays is 38 and at least 60% of them are up with fast speed always)!
Can you help me to resolve that problem?
Here is my code:
function curl_redir_exec($url, $post, $agent, $debug="")
{
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
static $curl_loops = 0;
static $curl_max_loops = 20;
if ($curl_loops++ >= $curl_max_loops)
{
$curl_loops = 0;
return FALSE;
}
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
curl_setopt($ch, CURLOPT_COOKIESESSION, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 14);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 3);
curl_setopt($ch, CURLOPT_COOKIEFILE, "tmp/cookies.txt");
curl_setopt($ch, CURLOPT_COOKIEJAR, "tmp/cookies.txt");
$data = curl_exec($ch);
//curl_close($ch);
$debbbb = $data;
list($header, $data) = explode("\n\n", $data, 2);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($http_code == 301 || $http_code == 302) {
$matches = array();
preg_match('/Location:(.*?)\n/', $header, $matches);
$url = @parse_url(trim(array_pop($matches)));
if (!$url)
{
//couldn't process the url to redirect to
$curl_loops = 0;
return $data;
}
$last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL));
$new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . ($url['query']?'?'.$url['query']:'');
curl_close($ch);
$ph = curl_init();
curl_setopt($ph, CURLOPT_URL, $new_url);
curl_setopt($ph, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ph, CURLOPT_USERAGENT, $agent);
curl_setopt($ph, CURLOPT_COOKIESESSION, true);
curl_setopt($ph, CURLOPT_CONNECTTIMEOUT, 3);
curl_setopt($ph, CURLOPT_TIMEOUT, 14);
curl_setopt($ph, CURLOPT_COOKIEFILE, "tmp/cookies.txt");
curl_setopt($ph, CURLOPT_COOKIEJAR, "tmp/cookies.txt");
$return = curl_exec($ph);
curl_close($ph);
return $return;
//return curl_error($ph);
//return $header;
} else {
$curl_loops=0;
return $debbbb;
}
}
$agents[] ='Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; WOW64; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; Media Center PC 5.0';
$agents[] = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0';
$agents[] = 'Opera/9.63 (Windows NT 6.0; U; ru) Presto/2.1.1';
$agents[] = 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5';
$agents[] = 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.18) Gecko/20081203 Firefox/2.0.0.18';
$agents[] = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16';
$agents[] = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1';
$proxyurl = file("http://livescore-bg.net/url.txt"); // array with proxy's submit target files
$proxydata = file("http://livescore-bg.net/data.txt"); // array with corresponding keys for each proxy, containing the names of the text fields with the address to be submitted to the target page from $proxyurl
$page = "http://example.com/thepageIneed.html";
while(true) {
//$numberofaatempts++;
$randomproxy = array_rand($proxyurl);
$proxyurljedini = $proxyurl[$randomproxy];
$proxydatajedini = $proxydata[$randomproxy];
$proxydatajedini = array($proxydatajedini => $page);
$g = curl_redir_exec($proxyurljedini, $proxydatajedini, array_rand($agents));
if((strpos($g, "checktextshowingthecontentisOK") !== false) or ($g == "checktextshowingthecontentisOK")) {
break;
} else {
foreach($proxydata as $key => $value) {
if($key == $randomproxy) {
unset($proxydata[$key]);
}
}
$proxydata = array_values($proxydata);
foreach($proxyurl as $key => $value) {
if($key == $randomproxy) {
unset($proxyurl[$key]);
}
}
$proxyurl = array_values($proxyurl);
}
}