Knah-Tsaeb/Soshot
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases Wiki Activity

Fix overcharge

Browse source
This commit is contained in:
Knah Tsaeb 2016-12-12 15:27:29 +01:00
parent 46cae0018b
commit c2e85e24ec
7 changed files with 104 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
bin/mon.sh
View file

@ -2,14 +2,15 @@
while [ 1 ] while [ 1 ]
do do
clear clear
echo "#################################################" echo "#####################################################"
echo "#################################################" echo "#####################################################"
echo "#### Total request thumbshot : " $(ps -e | grep -v grep | grep thumb_server.sh | wc -l) && # count thumb_server.sh process echo "#### Request thumbshot (thumb_server.sh) : " $(ps ax | grep -v grep | grep thumb_server.sh | wc -l) && # count thumb_server.sh process
echo "#### Number of thumbshot are made now : " $(ps -e | grep -v grep | grep phantomjs | wc -l) && # count phantomjs process echo "#### Nb of thumbshot are made now (phantomjs) : " $(ps ax | grep -v grep | grep phantomjs | wc -l) && # count phantomjs process
echo "#### Number of test error now : " $(ps -e | grep -v grep | grep curl | wc -l) # count thumb_server.sh => curl process echo "#### Nb of test error now (curl) : " $(ps ax | grep -v grep | grep curl | wc -l) # count thumb_server.sh => curl process
echo "#################################################" echo "#### Nb of optimize operation (optipng) : " $(ps ax | grep -v grep | grep optipng | wc -l) # count thumb_server.sh => curl process
echo "######### PRESS CTRL+C FOR STOP MONITOR #########" echo "#####################################################"
echo "#################################################" echo "########### PRESS CTRL+C FOR STOP MONITOR ###########"
echo "#####################################################"
sleep 3 # 3 second between two refresh sleep 3 # 3 second between two refresh
done done

35
bin/rasterize.js
View file

@ -1,26 +1,39 @@
var page = require('webpage').create(), function renderPage(url, output) {
address, output;
var page = require('webpage').create();
var redirectURL = null;
if (phantom.args.length < 2) { if (phantom.args.length < 2) {
//console.log('Usage: rasterize.js URL filename'); //console.log('Usage: rasterize.js URL filename');
phantom.exit(); phantom.exit();
} else { } else {
address = phantom.args[0]; page.settings.userAgent = 'Mozilla/5.0 (X11; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/50.0';
output = phantom.args[1];
page.settings.userAgent = 'Mozilla/5.0 (X11; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0';
page.viewportSize = { width: 1280, height:1024}; page.viewportSize = { width: 1280, height:1024};
page.open(address, function (status) { page.onResourceReceived = function(resource) {
if (status !== 'success') { if (url == resource.url && resource.redirectURL) {
console.log('Unable to load the address!'); redirectURL = resource.redirectURL;
page.close(); }
phantom.exit(); };
} else {
page.open(url, function (status) {
if (redirectURL) {
renderPage(redirectURL);
} else if (status == 'success') {
window.setTimeout(function () { window.setTimeout(function () {
page.render(output); page.render(output);
page.close(); page.close();
phantom.exit(); phantom.exit();
}, 1000); }, 1000);
} else {
console.log('Unable to load the address!');
page.close();
phantom.exit();
} }
}); });
} }
}
url = phantom.args[0];
output = phantom.args[1];
renderPage(url, output);

6
bin/reset.sh Normal file
View file

@ -0,0 +1,6 @@
#/bin/bash
cd ../cache/tmp/
pwd
rm *.lock
rm *.png
rm *.jpg

4
bin/stop.sh Normal file
View file

@ -0,0 +1,4 @@
#!/bin/bash
kill -TERM $(ps ax | grep -v grep | grep thumb_server.sh | awk '{print $1}')
kill -TERM $(ps ax | grep -v grep | grep phantomjs | awk '{print $1}')
kill -TERM $(ps ax | grep -v grep | grep optipng | awk '{print $1}')

59
bin/thumb_server.sh
View file

@ -17,17 +17,24 @@ noErrorLog=$7
##### for overwrite options ##### ##### for overwrite options #####
########################################## ##########################################
maxThread=1 # max parralle work. For me the best is nb core - 1 maxThread=1 # max parralle work. For me the best is nb core - 1
timeOut=190 # default time out, after this time the site are declared in error timeOut=120 # default time out, after this time the site are declared in error
log=false # log all generation success and error log=false # log all generation success and error
randomSleep=`echo $((RANDOM%20))` randomSleep=`echo $((RANDOM%40))`
optimizeTool='' # external tool form optimize png optimizeTool='' # external tool form optimize png
maxQueue=50 # max process in queue maxQueue=50 # max process in queue
currentProcess=$(ps -e | grep -v grep | grep thumb_ | wc -l) currentProcess=$(ps x | grep -v grep | grep thumb_server.sh | wc -l)
firstLevel=${hashUrl:0:2} firstLevel=${hashUrl:0:2}
secondLevel=${hashUrl:2:2} secondLevel=${hashUrl:2:2}
startPath=$sizeNameDir/$firstLevel/$secondLevel/ startPath=$sizeNameDir/$firstLevel/$secondLevel/
getArch=$(getconf LONG_BIT) getArch=$(getconf LONG_BIT)
#echo $getArch #echo $getArch
if [ $currentProcess -gt $maxQueue ]
then
rm "../cache/tmp/$hashUrl.lock"
exit
fi
if [ "$site" = "manual" ] if [ "$site" = "manual" ]
then then
site=$(cat "../cache/tmp/manual.txt") site=$(cat "../cache/tmp/manual.txt")
@ -41,27 +48,33 @@ then
source "../cache/config/serverOptions.php" source "../cache/config/serverOptions.php"
fi fi
if [ $currentProcess -gt $maxQueue ]
then
rm "../cache/tmp/$hashUrl.lock"
exit
fi
if [ ! $waitForResult ] if [ ! $waitForResult ]
then then
sleep `echo $((RANDOM%20))` sleep $randomSleep
while [[ `ps -e | grep -v grep | grep phantomjs | wc -l` -ge $maxThread ]] while [[ `ps -e | grep -v grep | grep phantomjs | wc -l` -ge $maxThread ]]
do do
sleep `echo $((RANDOM%20))` sleep $randomSleep
done done
fi fi
start_time=`date +%s` start_time=`date +%s`
if [ $getArch -eq 64 ] if [ $getArch -eq 64 ]
then then
phanTom=$(./phantomjs_x64 --disk-cache=false --local-storage-path=../cache/tmp --ignore-ssl-errors=true --web-security=false rasterize.js "$site" "../cache/tmp/$hashUrl.png") timeout $timeOut nice -n 5 ./phantomjs_x64 --disk-cache=false --local-storage-path=../cache/tmp --ignore-ssl-errors=true --web-security=false rasterize.js "$site" "../cache/tmp/$hashUrl.png"
exitStatus=$?
else else
phanTom=$(./phantomjs_x86 --disk-cache=false --local-storage-path=../cache/tmp --ignore-ssl-errors=true --web-security=false rasterize.js "$site" "../cache/tmp/$hashUrl.png") timeout $timeOut nice -n 5 ./phantomjs_x86 --disk-cache=false --local-storage-path=../cache/tmp --ignore-ssl-errors=true --web-security=false rasterize.js "$site" "../cache/tmp/$hashUrl.png"
exitStatus=$?
fi
#echo $exitStatus
if [ $exitStatus -eq 124 ]
then
echo 0
cp "000.png" "../cache/img/$startPath""$hashUrl""_thumb.png" && cp "000.png" "../cache/img/$startPath""$hashUrl.png" && cp "000.png" "../cache/img/$startPath""$hashUrl""_complete.png"
rm "../cache/tmp/$hashUrl.lock"
exit
fi fi
if [[ ! -f "../cache/tmp/$hashUrl.png" ]] if [[ ! -f "../cache/tmp/$hashUrl.png" ]]
@ -102,10 +115,10 @@ fi
case $renderType in case $renderType in
# thumbnail only # thumbnail only
t) t)
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" &>/dev/null eval nice -n 5 $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
if [[ -f "../cache/img/$startPath""$hashUrl""_thumb.png" ]] if [[ -f "../cache/img/$startPath""$hashUrl""_thumb.png" ]]
@ -119,10 +132,10 @@ case $renderType in
;; ;;
# thumbnail + complete # thumbnail + complete
tc) tc)
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null nice -n 5 eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl""_complete.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl""_complete.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
@ -136,11 +149,11 @@ case $renderType in
;; ;;
# thumbnail + realsize # thumbnail + realsize
tr) tr)
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 "../cache/tmp/$hashUrl.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 "../cache/tmp/$hashUrl.png"
convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null eval nice -n 5 $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"
@ -155,11 +168,11 @@ case $renderType in
# thumbnail + realsize + complete # thumbnail + realsize + complete
trc) trc)
cp "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png" cp "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png"
convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 "../cache/tmp/$hashUrl.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -crop 1280x1024+0+0 "../cache/tmp/$hashUrl.png"
convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png" nice -n 5 convert "../cache/tmp/$hashUrl.png" -filter Lanczos -thumbnail "$thumbSize" "../cache/tmp/$hashUrl""_thumb.png"
if [ $optimizeTool ] if [ $optimizeTool ]
then then
eval $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png" &>/dev/null eval nice -n 5 $optimizeTool "../cache/tmp/$hashUrl""_thumb.png" "../cache/tmp/$hashUrl.png" "../cache/tmp/$hashUrl""_complete.png" &>/dev/null
fi fi
mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png" mv "../cache/tmp/$hashUrl.png" "../cache/img/$startPath""$hashUrl.png"
mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png" mv "../cache/tmp/$hashUrl""_thumb.png" "../cache/img/$startPath""$hashUrl""_thumb.png"

13
readme.md Normal file
View file

@ -0,0 +1,13 @@
# !!!! Be careful !!!!
## This script can overcharge your server.
## Use at your own risk
All info in the homepage http://forge.leslibres.org/projects/soshot
**Thanks**
* Icon by http://www.iconfinder.com/icondetails/7917/128/camera_file_image_icon
* Bash script idea by http://www.cambus.net/blog/page/3/
* Httpstatus in bash by https://gist.github.com/rsvp/1171304
* The engine phantomjs by http://phantomjs.org/

7
readme.txt
View file

@ -1,7 +0,0 @@
All info in the homepage http://forge.leslibres.org/projects/soshot
*** Thanks ***
Icon by http://www.iconfinder.com/icondetails/7917/128/camera_file_image_icon
Bash script idea by http://www.cambus.net/blog/page/3/
Httpstatus in bash by https://gist.github.com/rsvp/1171304
The engine phantomjs by http://phantomjs.org/