um hold on... are you telling me you use pcntl_fork to do http requests? If so, you should definitely look into [man]curl_multi_exec[/man]. It allows you to perform multiple requests in asynchronously, including the ability to add on requests while continuing to process the existing requests. Here's a probably overly verbose example from actual code, I don't feel like chopping it down for you 😛
<?php
/**
* @param $sUrl
* @param array|null $sPostData
* @param array|null $aHeaders
* @return resource
*/
function getCurl($sUrl, $sPostData = null, $aHeaders = null)
{
$oHandle = curl_init($sUrl);
curl_setopt($oHandle, CURLOPT_RETURNTRANSFER, true);
curl_setopt($oHandle, CURLOPT_HEADER, true);
curl_setopt($oHandle, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($oHandle, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($oHandle, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($oHandle, CURLOPT_TIMEOUT, 60);
curl_setopt($oHandle, CURLOPT_CONNECTTIMEOUT, 15);
if (!empty($sPostData)) {
$aHeaders[] = 'Content-Type: application/json';
curl_setopt($oHandle, CURLOPT_POST, true);
curl_setopt($oHandle, CURLOPT_POSTFIELDS, $sPostData);
}
if (!empty($aHeaders)) {
curl_setopt($oHandle, CURLOPT_HTTPHEADER, $aHeaders);
}
return $oHandle;
}
/**
* @param resource $hCurlMulti Resource returned by curl_multi_init
* @param $aCurls
* @param $aTestCases
*/
function runCurlMulti($hCurlMulti, &$aCurls, &$aTestCases)
{
list($mrc, $active) = curlMultiExec($hCurlMulti);
while ($active && $mrc == CURLM_OK) {
$iReady = curl_multi_select($hCurlMulti);
if ($iReady != 0) {
do {
$mrc = curl_multi_exec($hCurlMulti, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
if ($iReady > 0) {
// A request has finished
$aInfo = curl_multi_info_read($hCurlMulti);
while ($aInfo !== false) {
$sTestId = array_search($aInfo['handle'], $aCurls);
$hCurl = $aCurls[$sTestId];
list($sTestId, $sType, $sHandleId) = explode('-', $sTestId, 3);
if ($aInfo['result'] === CURLE_OK) {
$aTestCases[$sTestId]['response'][$sType][$sHandleId]['stats'] = curl_getinfo($hCurl);
$iHeaderSize = curl_getinfo($hCurl, CURLINFO_HEADER_SIZE);
$sResponse = curl_multi_getcontent($hCurl);
$aTestCases[$sTestId]['response'][$sType][$sHandleId]['headers'] = trim(substr($sResponse, 0, $iHeaderSize));
$aResources = parseResponseForResources(
trim(substr($sResponse, $iHeaderSize)),
sprintf(
"%s://%s",
parse_url($aTestCases[$sTestId]['url'], PHP_URL_SCHEME),
parse_url($aTestCases[$sTestId]['url'], PHP_URL_HOST)
)
);
foreach ($aResources as $sResource) {
$aCurls["$sTestId-$sType-$sHandleId-$sResource"] = getCurl($sResource);
curl_multi_add_handle($hCurlMulti, $aCurls["$sTestId-$sType-$sHandleId-$sResource"]);
}
if (!empty($aResources)) {
list($mrc, $active) = curlMultiExec($hCurlMulti);
}
} else {
// Request failed, handle it
$aTestCases[$sTestId]['response'][$sType][$sHandleId]['error'] = array(
curl_errno($hCurl),
curl_error($hCurl)
);
}
// remove this handle since we've recorded its stats
curl_multi_remove_handle($hCurlMulti, $hCurl);
// read more info, in case multiple handles finished
$aInfo = curl_multi_info_read($hCurlMulti);
}
}
}
}
/**
* @param resource $hCurlMulti Resource returned by curl_multi_init
* @returns array
*/
function curlMultiExec($hCurlMulti)
{
do {
$mrc = curl_multi_exec($hCurlMulti, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
return array($mrc, $active);
}
$hCurlMulti = curl_multi_init();
$aTestCases = json_decode(file_get_contents($argv[1])) or die('Invalid test case file specified');
foreach ($aTestCases as $k => $aTestCase) {
$aCurls["$k-page-get"] = getCurl($aTestCase['url']);
curl_multi_add_handle($hCurlMulti, $aCurls["$k-page-get"]);
}
runCurlMulti($hCurlMulti, $aCurls, $aTestCases);
curl_multi_close($hCurlMulti);
And if you need it to go directly to a file, you can use:
$hFP = fopen('/path/to/some/file', 'w');
curl_setopt($hCurl, CURLOPT_FILE, $hFP);
