Knah-Tsaeb/Soshot
1
0
Fork 0
Code Issues 5 Pull requests Releases Wiki Activity

[fix] bad queue detection

Browse source 
[fix] manual gen error
This commit is contained in:
Knah Tsaeb 2015-07-13 12:34:33 +02:00
parent 57d88558cc
commit f1b94962e3
4 changed files with 62 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/rasterize.js
View file

@ -12,7 +12,7 @@ if (phantom.args.length < 2) {
page.open(address, function (status) { page.open(address, function (status) {
if (status !== 'success') { if (status !== 'success') {
//console.log('Unable to load the address!'); console.log('Unable to load the address!');
page.close(); page.close();
phantom.exit(); phantom.exit();
} else { } else {

21
bin/thumb_server.sh
View file

@ -9,6 +9,7 @@ thumbSize=$3 # size of thumb widthxheight ex : 190x90
sizeNameDir=$4 # name of size dir sizeNameDir=$4 # name of size dir
renderType=$5 # make only thumbshot / thumbshot + realSize / thumbshot + complete / thumbshot + complete + realSize renderType=$5 # make only thumbshot / thumbshot + realSize / thumbshot + complete / thumbshot + complete + realSize
waitForResult=$6 # if true we try to make soon as possible or add to queue waitForResult=$6 # if true we try to make soon as possible or add to queue
noErrorLog=$7
########################################## ##########################################
##### DON'T EDIT THIS PARAM USE ##### ##### DON'T EDIT THIS PARAM USE #####
@ -27,7 +28,7 @@ secondLevel=${hashUrl:2:2}
startPath=$sizeNameDir/$firstLevel/$secondLevel/ startPath=$sizeNameDir/$firstLevel/$secondLevel/
getArch=$(getconf LONG_BIT) getArch=$(getconf LONG_BIT)
#echo $getArch #echo $getArch
if [ "$site" -eq "manual" ] if [ "$site" = "manual" ]
then then
site=$(cat "../cache/tmp/manual.txt") site=$(cat "../cache/tmp/manual.txt")
echo '' > "../cache/tmp/manual.txt" echo '' > "../cache/tmp/manual.txt"
@ -42,6 +43,7 @@ fi
if [ $currentProcess -gt $maxQueue ] if [ $currentProcess -gt $maxQueue ]
then then
rm "../cache/tmp/$hashUrl.lock"
exit exit
fi fi
@ -66,7 +68,6 @@ if [ ! -f "../cache/tmp/$hashUrl.png" ]
then then
echo 0 echo 0
errorCode=`echo $(curl -k -L --write-out %{http_code} --silent -S --connect-timeout $timeOut \--no-keepalive --output /dev/null $site)` errorCode=`echo $(curl -k -L --write-out %{http_code} --silent -S --connect-timeout $timeOut \--no-keepalive --output /dev/null $site)`
echo $site
if [ $site == 'manual' ] if [ $site == 'manual' ]
then then
errorCode='manual' errorCode='manual'
@ -76,13 +77,18 @@ then
000) cp "000.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "000.png" "../cache/img/$startPath""$hashUrl.png" && cp "000.png" "../cache/img/$startPath""$hashUrl""_complete.png";; 000) cp "000.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "000.png" "../cache/img/$startPath""$hashUrl.png" && cp "000.png" "../cache/img/$startPath""$hashUrl""_complete.png";;
200) cp "error.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "error.png" "../cache/img/$startPath""$hashUrl.png" && cp "error.png" "../cache/img/$startPath""$hashUrl""_complete.png";; 200) cp "error.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "error.png" "../cache/img/$startPath""$hashUrl.png" && cp "error.png" "../cache/img/$startPath""$hashUrl""_complete.png";;
404) cp "404.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "404.png" "../cache/img/$startPath""$hashUrl.png" && cp "404.png" "../cache/img/$startPath""$hashUrl""_complete.png";; 404) cp "404.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "404.png" "../cache/img/$startPath""$hashUrl.png" && cp "404.png" "../cache/img/$startPath""$hashUrl""_complete.png";;
manual) rm "../cache/tmp/$hashUrl.lock" && exit;;
*) cp "error.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "error.png" "../cache/img/$startPath""$hashUrl.png" && cp "error.png" "../cache/img/$startPath""$hashUrl""_complete.png";; *) cp "error.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "error.png" "../cache/img/$startPath""$hashUrl.png" && cp "error.png" "../cache/img/$startPath""$hashUrl""_complete.png";;
esac esac
rm "../cache/tmp/$hashUrl.lock" rm "../cache/tmp/$hashUrl.lock"
if $log if $log == true
then then
end_time=`date +%s` end_time=`date +%s`
logDate=`date +'[%a %d %b %Y] [%H:%M:%S]'` logDate=`date +'[%a %d %b %Y] [%H:%M:%S]'`
if $noErrorLog
then
exit
fi
if [ $errorCode == 200 ] if [ $errorCode == 200 ]
then then
echo "none --- "$site" --- "$hashUrl" --- "$thumbSize" --- "$sizeNameDir" --- "$onlyThumb" --- true --- "$errorCode > "../cache/logs/retry/"$hashUrl".log" echo "none --- "$site" --- "$hashUrl" --- "$thumbSize" --- "$sizeNameDir" --- "$onlyThumb" --- true --- "$errorCode > "../cache/logs/retry/"$hashUrl".log"
@ -99,7 +105,7 @@ case $renderType in
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
if [[ -f "../cache/img/$startPath""$hashUrl""_thumb.png" ]] if [[ -f "../cache/img/$startPath""$hashUrl""_thumb.png" ]]
@ -116,7 +122,7 @@ case $renderType in
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl""_complete.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl""_complete.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
@ -134,7 +140,7 @@ case $renderType in
convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
@ -153,7 +159,7 @@ case $renderType in
convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png" eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
@ -164,7 +170,6 @@ case $renderType in
else else
echo 0 echo 0
fi fi
rm "../cache/tmp/$hashUrl.png"
rm "../cache/tmp/$hashUrl.lock" rm "../cache/tmp/$hashUrl.lock"
;; ;;
esac esac

56
inc/admin.php
View file

@ -85,8 +85,11 @@ function manualGen($hash, $log) {
$res['success'] = 1; $res['success'] = 1;
$res['filePath'] = pathForFile($line[4], $hash).'.png'; $res['filePath'] = pathForFile($line[4], $hash).'.png';
$res['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents($res['filePath'])); $res['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents($res['filePath']));
return $res; } else {
$res['error'] = 1;
$res['msg'] = 'Can\'t generate thumbshot manualy, try curl method.';
} }
return $res;
} }
function delAndRegen($hash, $size) { function delAndRegen($hash, $size) {
@ -137,11 +140,22 @@ function tryWithCurl($hash, $log) {
curl_setopt($ch, CURLOPT_USERAGENT, 'SoShot - perSOnal thumbSHOTer'); curl_setopt($ch, CURLOPT_USERAGENT, 'SoShot - perSOnal thumbSHOTer');
$resultat = curl_exec($ch); $resultat = curl_exec($ch);
curl_close($ch); curl_close($ch);
$searchStr = array(
'href="//',
'src="//',
'content="//'
);
$replaceStr = array(
'href="http://',
'src="http://',
'content="http://'
);
$resultat = str_replace($searchStr, $replaceStr, $resultat);
file_put_contents('cache/tmp/'.$line[2].'.html', $resultat); file_put_contents('cache/tmp/'.$line[2].'.html', $resultat);
chdir('bin/'); chdir('bin/');
exec('bash thumb_server.sh ../cache/tmp/'.$hash.'.html '.escapeshellarg($line[2]).' '.escapeshellarg($line[3]).' '.escapeshellarg($line[4]).' trc 1', $result); exec('bash thumb_server.sh ../cache/tmp/'.$hash.'.html '.escapeshellarg($line[2]).' '.escapeshellarg($line[3]).' '.escapeshellarg($line[4]).' trc 1 noErrorLog', $result);
chdir('../'); chdir('../');
//unlink('../cache/tmp/'.$hash.'.html'); unlink('cache/tmp/'.$hash.'.html');
if ((int)$result[0] == 1) { if ((int)$result[0] == 1) {
$res['success'] = 1; $res['success'] = 1;
$res['filePath'] = pathForFile($line[4], $hash).'.png'; $res['filePath'] = pathForFile($line[4], $hash).'.png';
@ -229,7 +243,7 @@ if (isset($ui['log']) && $ui['log'] === 'success') {
$nbThumb = 0; $nbThumb = 0;
$nbThumb = count($logs['genTime']); $nbThumb = count($logs['genTime']);
echo 'Total request : ', $nbThumb, '</br>'; echo 'Total request : ', $nbThumb, '</br>';
echo 'Moyenne per request : ', round(array_sum($logs['genTime']) / $nbThumb, 2), 's (min (', min($logs['genTime']), 's) max (', max($logs['genTime']), 's))</br>'; echo 'Moyenne per request : ', round(array_sum($logs['genTime']) / $nbThumb, 2), 's (min (', min($logs['genTime']), 's) max (', max($logs['genTime']), 's))</br>';
} else { } else {
echo 'No result'; echo 'No result';
} }
@ -239,30 +253,31 @@ if (isset($ui['log']) && ($ui['log'] === 'suspect' || $ui['log'] === 'retry' ||
if (isset($ui['hash']) && !empty($ui['hash'])) { if (isset($ui['hash']) && !empty($ui['hash'])) {
//$ui['hash'] = validHash($ui['hash']); //$ui['hash'] = validHash($ui['hash']);
if (isset($ui['root']) && (int)$ui['root'] === 1) { if (isset($ui['root']) && (int)$ui['root'] === 1) {
$success = tryRootDomain($ui['hash'], $ui['log']); $response = tryRootDomain($ui['hash'], $ui['log']);
} }
if (isset($ui['cache']) && (int)$ui['cache'] === 1) { if (isset($ui['cache']) && (int)$ui['cache'] === 1) {
$success['filePath'] = pathForFile($ui['s'], $ui['hash']).'.png'; $response['success'] = 1;
$success['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents($success['filePath'])); $response['filePath'] = pathForFile($ui['s'], $ui['hash']).'.png';
if (!file_exists($success['filePath'])) { $response['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents($response['filePath']));
$success['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents('bin/error.png')); if (!file_exists($response['filePath'])) {
$response['base64'] = 'data:image/png;base64,'.base64_encode(file_get_contents('bin/error.png'));
} }
} }
if (isset($ui['keep']) && (int)$ui['keep'] === 1) { if (isset($ui['keep']) && (int)$ui['keep'] === 1) {
keepThisThumb($ui['hash'], $ui['log']); keepThisThumb($ui['hash'], $ui['log']);
} }
if (isset($ui['mGen']) && (int)$ui['mGen'] === 1) { if (isset($ui['mGen']) && (int)$ui['mGen'] === 1) {
$success = manualGen($ui['hash'], $ui['log']); $response = manualGen($ui['hash'], $ui['log']);
} }
if (isset($ui['tryCurl']) && (int)$ui['tryCurl'] === 1) { if (isset($ui['tryCurl']) && (int)$ui['tryCurl'] === 1) {
$success = tryWithCurl($ui['hash'], $ui['log']); $response = tryWithCurl($ui['hash'], $ui['log']);
} }
} }
$logs = parseFailLog($ui['log']); $logs = parseFailLog($ui['log']);
if (!empty($logs)) { if (!empty($logs)) {
echo '<ul>'; echo '<ul>';
foreach ($logs as $value) { foreach ($logs as $value) {
echo '<li>', htmlentities(strip_tags($value['3'])), htmlentities(strip_tags($value['0'])), '</br><a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&mGen=1#result">manuel launch</a> <a href="', htmlentities(strip_tags($value['0'])), '">view site</a> <a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&cache=1&s=', $value['2'], '#result">view image in cache</a></li>'; echo '<li>', htmlentities(strip_tags($value['3'])), htmlentities(strip_tags($value['0'])), '</br><a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&mGen=1#result">manuel launch</a> <a href="', htmlentities(strip_tags($value['0'])), '">view site</a> <a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&cache=1&s=', $value['2'], '#result">view image in cache</a></li>';
} }
echo '</ul>'; echo '</ul>';
} else { } else {
@ -270,10 +285,21 @@ if (isset($ui['log']) && ($ui['log'] === 'suspect' || $ui['log'] === 'retry' ||
} }
} }
echo '</div>'; echo '</div>';
if (!empty($success)) { if (!empty($response) && $response['success'] === 1) {
echo '<div id="result">'; echo '<div id="result">';
echo '<img src="', $success['base64'], '" style="width:100%;"/>'; echo '<img src="', $response['base64'], '" style="width:100%;"/>';
echo '<p><input type="text" value="http://', $_SERVER['SERVER_NAME'], '/', $success['filePath'], '" onclick="this.select()"/></p>'; echo '<p><input type="text" value="http://', $_SERVER['SERVER_NAME'], '/', $response['filePath'], '" onclick="this.select()"/></p>';
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&mGen=1">Manuel launch</a> / ';
if (extension_loaded("curl")) {
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&tryCurl=1">Try with curl</a> / ';
}
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $ui['hash'], '&root=1">Try with root domain</a> / ';
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $ui['hash'], '&keep=1">Keep this thumbshot</a>';
echo '</div>';
}
if (!empty($response) && $response['error'] === 1) {
echo '<div id="result">';
echo '<p>', $response['msg'], '</p>';
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&mGen=1">Manuel launch</a> / '; echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&mGen=1">Manuel launch</a> / ';
if (extension_loaded("curl")) { if (extension_loaded("curl")) {
echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&tryCurl=1">Try with curl</a> / '; echo '<a href="?p=admin&log=', $ui['log'], '&hash=', $value['1'], '&tryCurl=1">Try with curl</a> / ';

14
index.php
View file

@ -138,7 +138,7 @@ function testExistImg($file) {
* @return * @return
*/ */
function launchScript($url, $hashUrl, $width, $size, $waitForResult = false) { function launchScript($url, $hashUrl, $width, $size, $waitForResult = false) {
$oriUrl = $hashUrl; $oriHash = $hashUrl;
$hashUrl = escapeshellarg($hashUrl); $hashUrl = escapeshellarg($hashUrl);
$url = escapeshellarg($url); $url = escapeshellarg($url);
$width = escapeshellarg($width); $width = escapeshellarg($width);
@ -152,8 +152,8 @@ function launchScript($url, $hashUrl, $width, $size, $waitForResult = false) {
$renderType = 't'; $renderType = 't';
} }
if ($GLOBALS['config']['disableExec'] === false) { if ($GLOBALS['config']['disableExec'] === false) {
if (!file_exists('cache/tmp/'.$hashUrl.'.lock')) { if (!file_exists('cache/tmp/'.$oriHash.'.lock')) {
touch('cache/tmp/'.$oriUrl.'.lock'); touch('cache/tmp/'.$oriHash.'.lock');
chdir('bin/'); chdir('bin/');
if ($waitForResult === false) { if ($waitForResult === false) {
exec('bash thumb_server.sh '.$url.' '.$hashUrl.' '.$width.' '.$size.' '.$renderType.' > /dev/null &', $result); exec('bash thumb_server.sh '.$url.' '.$hashUrl.' '.$width.' '.$size.' '.$renderType.' > /dev/null &', $result);
@ -577,7 +577,7 @@ if (!empty($ui['request']) && $ui['p'] !== 'install' && $ui['p'] !== 'login' &&
file_put_contents('cache/logs/suspect/'.$hashUrl.'.log', $_SERVER['REMOTE_ADDR'].' --- '.$ui['url'].' --- '.$hashUrl.' --- '.$width.' --- '.$ui['s'].' --- '.$renderType.' --- true'."\n"); file_put_contents('cache/logs/suspect/'.$hashUrl.'.log', $_SERVER['REMOTE_ADDR'].' --- '.$ui['url'].' --- '.$hashUrl.' --- '.$width.' --- '.$ui['s'].' --- '.$renderType.' --- true'."\n");
} }
if ($ui['request'] === 'api') { if ($ui['request'] === 'api') {
printThumbShot('bin/error'); printThumbShot('bin/error', $ui['t']);
} else { } else {
$success = array( $success = array(
'normal' => 'bin/error.png', 'normal' => 'bin/error.png',
@ -597,7 +597,7 @@ if (!empty($ui['request']) && $ui['p'] !== 'install' && $ui['p'] !== 'login' &&
if ($makeImg === true) { if ($makeImg === true) {
$GLOBALS['config']['disableExec'] = true; $GLOBALS['config']['disableExec'] = true;
if ($ui['request'] === 'api') { if ($ui['request'] === 'api') {
printThumbShot($file); printThumbShot($file, $ui['t']);
} else { } else {
$success = array( $success = array(
'normal' => $file.'.png', 'normal' => $file.'.png',
@ -616,7 +616,7 @@ if (!empty($ui['request']) && $ui['p'] !== 'install' && $ui['p'] !== 'login' &&
} }
if ($ui['request'] === 'api') { if ($ui['request'] === 'api') {
$file = 'bin/loadingGen'; $file = 'bin/loadingGen';
printThumbShot($file); printThumbShot($file, $ui['t']);
} else { } else {
if ($GLOBALS['config']['disableExec'] === true) { if ($GLOBALS['config']['disableExec'] === true) {
$file = 'bin/loadingGen'; $file = 'bin/loadingGen';
@ -641,7 +641,7 @@ if (!empty($ui['request']) && $ui['p'] !== 'install' && $ui['p'] !== 'login' &&
</head> </head>
<body> <body>
<?php <?php
require getPage($ui['p']); require getPage($ui['p']);
?> ?>
</body> </body>
</html> </html>