getSimpleHTMLDOMCached($item['uri']);
+ $articlePage = getSimpleHTMLDOMCached($item['uri']);
$content = $articlePage->find('#comic', 0);
if(is_null($content)) // load alternative
$content = $articlePage->find('#blog', 0);
diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php
index 3fe61de9..0de08baa 100644
--- a/bridges/ThePirateBayBridge.php
+++ b/bridges/ThePirateBayBridge.php
@@ -52,11 +52,11 @@ class ThePirateBayBridge extends BridgeAbstract{
$keywordsList = explode(";",$this->getInput('q'));
foreach($keywordsList as $keywords){
- $html = $this->getSimpleHTMLDOM(self::URI.'search/'.rawurlencode($keywords).'/0/3/0')
- or $this->returnServerError('Could not request TPB.');
+ $html = getSimpleHTMLDOM(self::URI.'search/'.rawurlencode($keywords).'/0/3/0')
+ or returnServerError('Could not request TPB.');
if ($html->find('table#searchResult', 0) == FALSE)
- $this->returnServerError('No result for query '.$keywords);
+ returnServerError('No result for query '.$keywords);
foreach($html->find('tr') as $element) {
diff --git a/bridges/TwitchApiBridge.php b/bridges/TwitchApiBridge.php
index 11da945d..106f4dd6 100644
--- a/bridges/TwitchApiBridge.php
+++ b/bridges/TwitchApiBridge.php
@@ -70,10 +70,10 @@ class TwitchApiBridge extends BridgeAbstract{
/* Finally we're ready to request data from the API. Each response provides information for the next request. */
for($i = 0; $i < $requests; $i++) {
- $response = $this->getSimpleHTMLDOM($request, false, $context);
+ $response = getSimpleHTMLDOM($request, false, $context);
if($response == false) {
- $this->returnServerError('Request failed! Check if the channel name is valid!');
+ returnServerError('Request failed! Check if the channel name is valid!');
}
$data = json_decode($response);
diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php
index e30f7f08..baf8508c 100644
--- a/bridges/TwitterBridge.php
+++ b/bridges/TwitterBridge.php
@@ -61,13 +61,13 @@ class TwitterBridge extends BridgeAbstract{
public function collectData(){
$html = '';
- $html = $this->getSimpleHTMLDOM($this->getURI());
+ $html = getSimpleHTMLDOM($this->getURI());
if(!$html){
switch($this->queriedContext){
case 'By keyword or hashtag':
- $this->returnServerError('No results for this query.');
+ returnServerError('No results for this query.');
case 'By username':
- $this->returnServerError('Requested username can\'t be found.');
+ returnServerError('Requested username can\'t be found.');
}
}
diff --git a/bridges/UnsplashBridge.php b/bridges/UnsplashBridge.php
index 85ce0c2a..8aacd4a9 100644
--- a/bridges/UnsplashBridge.php
+++ b/bridges/UnsplashBridge.php
@@ -33,8 +33,8 @@ class UnsplashBridge extends BridgeAbstract {
for ($page = 1; $page <= $lastpage; $page++) {
$link = self::URI.'/grid?page='.$page;
- $html = $this->getSimpleHTMLDOM($link)
- or $this->returnServerError('No results for this query.');
+ $html = getSimpleHTMLDOM($link)
+ or returnServerError('No results for this query.');
if ($page === 1) {
preg_match('/=(\d+)$/', $html->find('.pagination > a[!class]', -1)->href, $matches);
diff --git a/bridges/ViadeoCompanyBridge.php b/bridges/ViadeoCompanyBridge.php
index 75fd6aae..09bcf292 100644
--- a/bridges/ViadeoCompanyBridge.php
+++ b/bridges/ViadeoCompanyBridge.php
@@ -17,8 +17,8 @@ class ViadeoCompanyBridge extends BridgeAbstract{
$html = '';
$link = self::URI.'fr/company/'.$this->getInput('c');
- $html = $this->getSimpleHTMLDOM($link)
- or $this->returnServerError('Could not request Viadeo.');
+ $html = getSimpleHTMLDOM($link)
+ or returnServerError('Could not request Viadeo.');
foreach($html->find('//*[@id="company-newsfeed"]/ul/li') as $element) {
$title = $element->find('p', 0)->innertext;
diff --git a/bridges/VineBridge.php b/bridges/VineBridge.php
index 6c42dd1f..da0b7caa 100644
--- a/bridges/VineBridge.php
+++ b/bridges/VineBridge.php
@@ -17,8 +17,8 @@ class VineBridge extends BridgeAbstract {
$html = '';
$uri = self::URI.'/u/'.$this->getInput('u').'?mode=list';
- $html = $this->getSimpleHTMLDOM($uri)
- or $this->returnServerError('No results for this query.');
+ $html = getSimpleHTMLDOM($uri)
+ or returnServerError('No results for this query.');
foreach($html->find('.post') as $element) {
$a = $element->find('a', 0);
diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php
index 54895996..5dbae45c 100644
--- a/bridges/VkBridge.php
+++ b/bridges/VkBridge.php
@@ -18,8 +18,8 @@ class VkBridge extends BridgeAbstract {
return static::URI.urlencode($this->getInput('u'));
}
public function collectData(){
- $text_html = $this->getContents($this->getURI())
- or $this->returnServerError('No results for group or user name "'.$this->getInput('u').'".');
+ $text_html = getContents($this->getURI())
+ or returnServerError('No results for group or user name "'.$this->getInput('u').'".');
$text_html = iconv('windows-1251', 'utf-8', $text_html);
$html = str_get_html($text_html);
diff --git a/bridges/WallpaperStopBridge.php b/bridges/WallpaperStopBridge.php
index b10d0cc1..34b290f5 100644
--- a/bridges/WallpaperStopBridge.php
+++ b/bridges/WallpaperStopBridge.php
@@ -33,8 +33,8 @@ class WallpaperStopBridge extends BridgeAbstract {
for ($page = 1; $page <= $lastpage; $page++) {
$link = self::URI.'/'.$category.'-wallpaper/'.(!empty($subcategory)?$subcategory.'-wallpaper/':'').'desktop-wallpaper-'.$page.'.html';
- $html = $this->getSimpleHTMLDOM($link)
- or $this->returnServerError('No results for this query.');
+ $html = getSimpleHTMLDOM($link)
+ or returnServerError('No results for this query.');
if ($page === 1) {
preg_match('/-(\d+)\.html$/', $html->find('.pagination > .last', 0)->href, $matches);
diff --git a/bridges/WeLiveSecurityBridge.php b/bridges/WeLiveSecurityBridge.php
index 9aa18c74..b05f4732 100644
--- a/bridges/WeLiveSecurityBridge.php
+++ b/bridges/WeLiveSecurityBridge.php
@@ -18,7 +18,7 @@ class WeLiveSecurityBridge extends FeedExpander {
protected function parseItem($item){
$item = parent::parseItem($item);
- $article_html = $this->getSimpleHTMLDOMCached($item['uri']);
+ $article_html = getSimpleHTMLDOMCached($item['uri']);
if(!$article_html){
$item['content'] .= '
Could not request '.$this->getName().': '.$item['uri'].'
';
return $item;
diff --git a/bridges/WhydBridge.php b/bridges/WhydBridge.php
index ec377eba..fae39cfe 100644
--- a/bridges/WhydBridge.php
+++ b/bridges/WhydBridge.php
@@ -19,19 +19,19 @@ class WhydBridge extends BridgeAbstract{
$html = '';
if (strlen(preg_replace("/[^0-9a-f]/",'', $this->getInput('u'))) == 24){
// is input the userid ?
- $html = $this->getSimpleHTMLDOM(
+ $html = getSimpleHTMLDOM(
self::URI.'u/'.preg_replace("/[^0-9a-f]/",'', $this->getInput('u'))
- ) or $this->returnServerError('No results for this query.');
+ ) or returnServerError('No results for this query.');
} else { // input may be the username
- $html = $this->getSimpleHTMLDOM(
+ $html = getSimpleHTMLDOM(
self::URI.'search?q='.urlencode($this->getInput('u'))
- ) or $this->returnServerError('No results for this query.');
+ ) or returnServerError('No results for this query.');
for ($j = 0; $j < 5; $j++) {
if (strtolower($html->find('div.user', $j)->find('a',0)->plaintext) == strtolower($this->getInput('u'))) {
- $html = $this->getSimpleHTMLDOM(
+ $html = getSimpleHTMLDOM(
self::URI . $html->find('div.user', $j)->find('a', 0)->getAttribute('href')
- ) or $this->returnServerError('No results for this query');
+ ) or returnServerError('No results for this query');
break;
}
}
diff --git a/bridges/WikipediaBridge.php b/bridges/WikipediaBridge.php
index 5feb4291..c0f0fc02 100644
--- a/bridges/WikipediaBridge.php
+++ b/bridges/WikipediaBridge.php
@@ -90,10 +90,10 @@ class WikipediaBridge extends BridgeAbstract {
$fullArticle = $this->getInput('fullarticle');
// This will automatically send us to the correct main page in any language (try it!)
- $html = $this->getSimpleHTMLDOM($this->getURI() . '/wiki');
+ $html = getSimpleHTMLDOM($this->getURI() . '/wiki');
if(!$html)
- $this->returnServerError('Could not load site: ' . $this->getURI() . '!');
+ returnServerError('Could not load site: ' . $this->getURI() . '!');
/*
* Now read content depending on the language (make sure to create one function per language!)
@@ -103,7 +103,7 @@ class WikipediaBridge extends BridgeAbstract {
$function = 'GetContents' . strtoupper($this->getInput('language'));
if(!method_exists($this, $function))
- $this->returnServerError('A function to get the contents for your language is missing (\'' . $function . '\')!');
+ returnServerError('A function to get the contents for your language is missing (\'' . $function . '\')!');
/*
* The method takes care of creating all items.
@@ -175,15 +175,15 @@ class WikipediaBridge extends BridgeAbstract {
* Loads the full article from a given URI
*/
private function LoadFullArticle($uri){
- $content_html = $this->getSimpleHTMLDOMCached($uri);
+ $content_html = getSimpleHTMLDOMCached($uri);
if(!$content_html)
- $this->returnServerError('Could not load site: ' . $uri . '!');
+ returnServerError('Could not load site: ' . $uri . '!');
$content = $content_html->find('#mw-content-text', 0);
if(!$content)
- $this->returnServerError('Could not find content in page: ' . $uri . '!');
+ returnServerError('Could not find content in page: ' . $uri . '!');
// Let's remove a couple of things from the article
$table = $content->find('#toc', 0); // Table of contents
diff --git a/bridges/WordPressBridge.php b/bridges/WordPressBridge.php
index d686a98f..bc45b1a1 100644
--- a/bridges/WordPressBridge.php
+++ b/bridges/WordPressBridge.php
@@ -22,7 +22,7 @@ class WordPressBridge extends FeedExpander {
protected function parseItem($newItem){
$item=parent::parseItem($newItem);
- $article_html = $this->getSimpleHTMLDOMCached($item['uri']);
+ $article_html = getSimpleHTMLDOMCached($item['uri']);
$article=null;
switch(true){
@@ -63,7 +63,7 @@ class WordPressBridge extends FeedExpander {
public function collectData(){
if($this->getInput('url') && substr($this->getInput('url'),0,strlen('http'))!=='http'){
// just in case someone find a way to access local files by playing with the url
- $this->returnClientError('The url parameter must either refer to http or https protocol.');
+ returnClientError('The url parameter must either refer to http or https protocol.');
}
$this->collectExpandableDatas($this->getURI().'/feed/atom/');
diff --git a/bridges/WorldOfTanksBridge.php b/bridges/WorldOfTanksBridge.php
index a3179be9..256a6ca9 100644
--- a/bridges/WorldOfTanksBridge.php
+++ b/bridges/WorldOfTanksBridge.php
@@ -42,9 +42,9 @@ class WorldOfTanksBridge extends BridgeAbstract {
}
public function collectData(){
- $html = $this->getSimpleHTMLDOM($this->getURI())
- or $this->returnServerError('Could not request '.$this->getURI());
- $this->debugMessage("loaded HTML from ".$this->getURI());
+ $html = getSimpleHTMLDOM($this->getURI())
+ or returnServerError('Could not request '.$this->getURI());
+ debugMessage("loaded HTML from ".$this->getURI());
// customize name
$this->title = $html->find('title', 0)->innertext;
foreach($html->find('.b-imgblock_ico') as $infoLink) {
@@ -56,8 +56,8 @@ class WorldOfTanksBridge extends BridgeAbstract {
$item = array();
$item['uri'] = self::URI.$infoLink->href;
// now load that uri from cache
- $this->debugMessage("loading page ".$item['uri']);
- $articlePage = $this->getSimpleHTMLDOMCached($item['uri']);
+ debugMessage("loading page ".$item['uri']);
+ $articlePage = getSimpleHTMLDOMCached($item['uri']);
$content = $articlePage->find('.l-content', 0);
HTMLSanitizer::defaultImageSrcTo($content, self::URI);
$item['title'] = $content->find('h1', 0)->innertext;
diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php
index 6c258f47..a8bc2e54 100644
--- a/bridges/YoutubeBridge.php
+++ b/bridges/YoutubeBridge.php
@@ -48,7 +48,7 @@ class YoutubeBridge extends BridgeAbstract {
);
private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time) {
- $html = $this->getSimpleHTMLDOM(self::URI."watch?v=$vid");
+ $html = getSimpleHTMLDOM(self::URI."watch?v=$vid");
$author = $html->innertext;
$author = substr($author, strpos($author, '"author=') + 8);
$author = substr($author, 0, strpos($author, '\u0026'));
@@ -118,17 +118,17 @@ class YoutubeBridge extends BridgeAbstract {
$url_listing = self::URI.'channel/'.urlencode($this->request).'/videos';
}
if (!empty($url_feed) && !empty($url_listing)) {
- if ($xml = $this->getSimpleHTMLDOM($url_feed)) {
+ if ($xml = getSimpleHTMLDOM($url_feed)) {
$this->ytBridgeParseXmlFeed($xml);
- } else if ($html = $this->getSimpleHTMLDOM($url_listing)) {
+ } else if ($html = getSimpleHTMLDOM($url_listing)) {
$this->ytBridgeParseHtmlListing($html, 'li.channels-content-item', 'h3');
- } else $this->returnServerError("Could not request YouTube. Tried:\n - $url_feed\n - $url_listing");
+ } else returnServerError("Could not request YouTube. Tried:\n - $url_feed\n - $url_listing");
}
else if ($this->getInput('p')) { /* playlist mode */
$this->request = $this->getInput('p');
$url_listing = self::URI.'playlist?list='.urlencode($this->request);
- $html = $this->getSimpleHTMLDOM($url_listing) or $this->returnServerError("Could not request YouTube. Tried:\n - $url_listing");
+ $html = getSimpleHTMLDOM($url_listing) or returnServerError("Could not request YouTube. Tried:\n - $url_listing");
$this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a');
$this->request = 'Playlist: '.str_replace(' - YouTube', '', $html->find('title', 0)->plaintext);
}
@@ -136,13 +136,13 @@ class YoutubeBridge extends BridgeAbstract {
else if ($this->getInput('s')) { /* search mode */
$this->request = $this->getInput('s'); $page = 1; if ($this->getInput('pa')) $page = (int)preg_replace("/[^0-9]/",'', $this->getInput('pa'));
$url_listing = self::URI.'results?search_query='.urlencode($this->request).'&page='.$page.'&filters=video&search_sort=video_date_uploaded';
- $html = $this->getSimpleHTMLDOM($url_listing) or $this->returnServerError("Could not request YouTube. Tried:\n - $url_listing");
+ $html = getSimpleHTMLDOM($url_listing) or returnServerError("Could not request YouTube. Tried:\n - $url_listing");
$this->ytBridgeParseHtmlListing($html, 'div.yt-lockup', 'h3');
$this->request = 'Search: '.str_replace(' - YouTube', '', $html->find('title', 0)->plaintext);
}
else { /* no valid mode */
- $this->returnClientError("You must either specify either:\n - YouTube username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)");
+ returnClientError("You must either specify either:\n - YouTube username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)");
}
}
diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php
index af0eb871..aad006f1 100644
--- a/bridges/ZDNetBridge.php
+++ b/bridges/ZDNetBridge.php
@@ -214,7 +214,7 @@ class ZDNetBridge extends BridgeAbstract {
$baseUri = str_replace('www.', 'downloads.', $baseUri);
}
$url = $baseUri.trim($feed, '/').'/rss.xml';
- $html = $this->getSimpleHTMLDOM($url) or $this->returnServerError('Could not request ZDNet: '.$url);
+ $html = getSimpleHTMLDOM($url) or returnServerError('Could not request ZDNet: '.$url);
$limit = 0;
foreach ($html->find('item') as $element) {
@@ -224,7 +224,7 @@ class ZDNetBridge extends BridgeAbstract {
$article_title = StripCDATA($element->find('title', 0)->plaintext);
$article_subtitle = StripCDATA($element->find('description', 0)->plaintext);
$article_timestamp = strtotime(StripCDATA($element->find('pubDate', 0)->plaintext));
- $article = $this->getSimpleHTMLDOM($article_url) or $this->returnServerError('Could not request ZDNet: '.$article_url);
+ $article = getSimpleHTMLDOM($article_url) or returnServerError('Could not request ZDNet: '.$article_url);
if (!empty($article_author))
$author = $article_author;
diff --git a/index.php b/index.php
index bb9555cd..6249f5bb 100644
--- a/index.php
+++ b/index.php
@@ -124,8 +124,9 @@ try {
$bridge->setCache($cache);
$noproxy = filter_input(INPUT_GET, '_noproxy', FILTER_VALIDATE_BOOLEAN);
- if(defined('PROXY_URL') && PROXY_BYBRIDGE && $noproxy)
- $bridge->useProxy = false;
+ if(defined('PROXY_URL') && PROXY_BYBRIDGE && $noproxy){
+ define('NOPROXY',true);
+ }
$params = $_GET;
unset($params['action']);
diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php
index f746f1f5..67b3adaa 100644
--- a/lib/BridgeAbstract.php
+++ b/lib/BridgeAbstract.php
@@ -8,25 +8,11 @@ abstract class BridgeAbstract implements BridgeInterface {
const MAINTAINER = 'No maintainer';
const PARAMETERS = array();
- public $useProxy = true;
-
protected $cache;
protected $items = array();
protected $inputs = array();
protected $queriedContext = '';
- protected function returnError($message, $code){
- throw new \HttpException($message, $code);
- }
-
- protected function returnClientError($message){
- $this->returnError($message, 400);
- }
-
- protected function returnServerError($message){
- $this->returnError($message, 500);
- }
-
/**
* Return items stored in the bridge
* @return mixed
@@ -260,7 +246,7 @@ abstract class BridgeAbstract implements BridgeInterface {
if(empty(static::PARAMETERS)){
if(!empty($inputs)){
- $this->returnClientError('Invalid parameters value(s)');
+ returnClientError('Invalid parameters value(s)');
}
$this->collectData();
@@ -271,15 +257,15 @@ abstract class BridgeAbstract implements BridgeInterface {
}
if(!$this->validateData($inputs)){
- $this->returnClientError('Invalid parameters value(s)');
+ returnClientError('Invalid parameters value(s)');
}
// Guess the paramter context from input data
$this->queriedContext = $this->getQueriedContext($inputs);
if(is_null($this->queriedContext)){
- $this->returnClientError('Required parameter(s) missing');
+ returnClientError('Required parameter(s) missing');
} elseif($this->queriedContext === false){
- $this->returnClientError('Mixed context parameters');
+ returnClientError('Mixed context parameters');
}
$this->setInputs($inputs, $this->queriedContext);
@@ -313,157 +299,4 @@ abstract class BridgeAbstract implements BridgeInterface {
public function setCache(\CacheAbstract $cache){
$this->cache = $cache;
}
-
- public function debugMessage($text){
- if(!file_exists('DEBUG')) {
- return;
- }
-
- $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 3);
- $calling = $backtrace[2];
- $message = $calling['file'] . ':'
- . $calling['line'] . ' class '
- . get_class($this) . '->'
- . $calling['function'] . ' - '
- . $text;
-
- error_log($message);
- }
-
- protected function getContents($url
- , $use_include_path = false
- , $context = null
- , $offset = 0
- , $maxlen = null){
- $contextOptions = array(
- 'http' => array(
- 'user_agent' => ini_get('user_agent')
- )
- );
-
- if(defined('PROXY_URL') && $this->useProxy){
- $contextOptions['http']['proxy'] = PROXY_URL;
- $contextOptions['http']['request_fulluri'] = true;
-
- if(is_null($context)){
- $context = stream_context_create($contextOptions);
- } else {
- $prevContext = $context;
- if(!stream_context_set_option($context, $contextOptions)){
- $context = $prevContext;
- }
- }
- }
-
- if(is_null($maxlen)){
- $content = @file_get_contents($url, $use_include_path, $context, $offset);
- } else {
- $content = @file_get_contents($url, $use_include_path, $context, $offset, $maxlen);
- }
-
- if($content === false)
- $this->debugMessage('Cant\'t download ' . $url);
-
- // handle compressed data
- foreach($http_response_header as $header){
- if(stristr($header, 'content-encoding')){
- switch(true){
- case stristr($header, 'gzip'):
- $content = gzinflate(substr($content, 10, -8));
- break;
- case stristr($header, 'compress'):
- //TODO
- case stristr($header, 'deflate'):
- //TODO
- case stristr($header, 'brotli'):
- //TODO
- $this->returnServerError($header . '=> Not implemented yet');
- break;
- case stristr($header, 'identity'):
- break;
- default:
- $this->returnServerError($header . '=> Unknown compression');
- }
- }
- }
-
- return $content;
- }
-
- protected function getSimpleHTMLDOM($url
- , $use_include_path = false
- , $context = null
- , $offset = 0
- , $maxLen = null
- , $lowercase = true
- , $forceTagsClosed = true
- , $target_charset = DEFAULT_TARGET_CHARSET
- , $stripRN = true
- , $defaultBRText = DEFAULT_BR_TEXT
- , $defaultSpanText = DEFAULT_SPAN_TEXT){
- $content = $this->getContents($url, $use_include_path, $context, $offset, $maxLen);
- return str_get_html($content
- , $lowercase
- , $forceTagsClosed
- , $target_charset
- , $stripRN
- , $defaultBRText
- , $defaultSpanText);
- }
-
- /**
- * Maintain locally cached versions of pages to avoid multiple downloads.
- * @param url url to cache
- * @param duration duration of the cache file in seconds (default: 24h/86400s)
- * @return content of the file as string
- */
- public function getSimpleHTMLDOMCached($url
- , $duration = 86400
- , $use_include_path = false
- , $context = null
- , $offset = 0
- , $maxLen = null
- , $lowercase = true
- , $forceTagsClosed = true
- , $target_charset = DEFAULT_TARGET_CHARSET
- , $stripRN = true
- , $defaultBRText = DEFAULT_BR_TEXT
- , $defaultSpanText = DEFAULT_SPAN_TEXT){
- $this->debugMessage('Caching url ' . $url . ', duration ' . $duration);
-
- $filepath = __DIR__ . '/../cache/pages/' . sha1($url) . '.cache';
- $this->debugMessage('Cache file ' . $filepath);
-
- if(file_exists($filepath) && filectime($filepath) < time() - $duration){
- unlink ($filepath);
- $this->debugMessage('Cached file deleted: ' . $filepath);
- }
-
- if(file_exists($filepath)){
- $this->debugMessage('Loading cached file ' . $filepath);
- touch($filepath);
- $content = file_get_contents($filepath);
- } else {
- $this->debugMessage('Caching ' . $url . ' to ' . $filepath);
- $dir = substr($filepath, 0, strrpos($filepath, '/'));
-
- if(!is_dir($dir)){
- $this->debugMessage('Creating directory ' . $dir);
- mkdir($dir, 0777, true);
- }
-
- $content = $this->getContents($url, $use_include_path, $context, $offset, $maxLen);
- if($content !== false){
- file_put_contents($filepath, $content);
- }
- }
-
- return str_get_html($content
- , $lowercase
- , $forceTagsClosed
- , $target_charset
- , $stripRN
- , $defaultBRText
- , $defaultSpanText);
- }
}
diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php
index 01909a5a..b0d975ae 100644
--- a/lib/FeedExpander.php
+++ b/lib/FeedExpander.php
@@ -9,46 +9,46 @@ abstract class FeedExpander extends BridgeAbstract {
public function collectExpandableDatas($url, $maxItems = -1){
if(empty($url)){
- $this->returnServerError('There is no $url for this RSS expander');
+ returnServerError('There is no $url for this RSS expander');
}
- $this->debugMessage('Loading from ' . $url);
+ debugMessage('Loading from ' . $url);
/* Notice we do not use cache here on purpose:
* we want a fresh view of the RSS stream each time
*/
- $content = $this->getContents($url)
- or $this->returnServerError('Could not request ' . $url);
+ $content = getContents($url)
+ or returnServerError('Could not request ' . $url);
$rssContent = simplexml_load_string($content);
- $this->debugMessage('Detecting feed format/version');
+ debugMessage('Detecting feed format/version');
switch(true){
case isset($rssContent->item[0]):
- $this->debugMessage('Detected RSS 1.0 format');
+ debugMessage('Detected RSS 1.0 format');
$this->feedType = "RSS_1_0";
break;
case isset($rssContent->channel[0]):
- $this->debugMessage('Detected RSS 0.9x or 2.0 format');
+ debugMessage('Detected RSS 0.9x or 2.0 format');
$this->feedType = "RSS_2_0";
break;
case isset($rssContent->entry[0]):
- $this->debugMessage('Detected ATOM format');
+ debugMessage('Detected ATOM format');
$this->feedType = "ATOM_1_0";
break;
default:
- $this->debugMessage('Unknown feed format/version');
- $this->returnServerError('The feed format is unknown!');
+ debugMessage('Unknown feed format/version');
+ returnServerError('The feed format is unknown!');
break;
}
- $this->debugMessage('Calling function "collect_' . $this->feedType . '_data"');
+ debugMessage('Calling function "collect_' . $this->feedType . '_data"');
$this->{'collect_' . $this->feedType . '_data'}($rssContent, $maxItems);
}
protected function collect_RSS_1_0_data($rssContent, $maxItems){
$this->load_RSS_2_0_feed_data($rssContent->channel[0]);
foreach($rssContent->item as $item){
- $this->debugMessage('parsing item ' . var_export($item, true));
+ debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
}
@@ -56,13 +56,13 @@ abstract class FeedExpander extends BridgeAbstract {
protected function collect_RSS_2_0_data($rssContent, $maxItems){
$rssContent = $rssContent->channel[0];
- $this->debugMessage('RSS content is ===========\n'
+ debugMessage('RSS content is ===========\n'
. var_export($rssContent, true)
. '===========');
$this->load_RSS_2_0_feed_data($rssContent);
foreach($rssContent->item as $item){
- $this->debugMessage('parsing item ' . var_export($item, true));
+ debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
}
@@ -71,7 +71,7 @@ abstract class FeedExpander extends BridgeAbstract {
protected function collect_ATOM_1_0_data($content, $maxItems){
$this->load_ATOM_feed_data($content);
foreach($content->entry as $item){
- $this->debugMessage('parsing item ' . var_export($item, true));
+ debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
}
@@ -190,7 +190,7 @@ abstract class FeedExpander extends BridgeAbstract {
case 'ATOM_1_0':
return $this->parseATOMItem($item);
break;
- default: $this->returnClientError('Unknown version ' . $this->getInput('version') . '!');
+ default: returnClientError('Unknown version ' . $this->getInput('version') . '!');
}
}
diff --git a/lib/RssBridge.php b/lib/RssBridge.php
index cf271144..a0b5a424 100644
--- a/lib/RssBridge.php
+++ b/lib/RssBridge.php
@@ -16,6 +16,8 @@ require __DIR__ . '/FeedExpander.php';
require __DIR__ . '/Cache.php';
require __DIR__ . '/CacheAbstract.php';
require __DIR__ . '/HTMLUtils.php';
+require __DIR__ . '/error.php';
+require __DIR__ . '/contents.php';
$vendorLibSimpleHtmlDom = __DIR__ . PATH_VENDOR . '/simplehtmldom/simple_html_dom.php';
if( !file_exists($vendorLibSimpleHtmlDom) ){
diff --git a/lib/contents.php b/lib/contents.php
new file mode 100644
index 00000000..747db5bf
--- /dev/null
+++ b/lib/contents.php
@@ -0,0 +1,142 @@
+ array(
+ 'user_agent' => ini_get('user_agent')
+ )
+ );
+
+ if(defined('PROXY_URL') && !defined('NOPROXY')){
+ $contextOptions['http']['proxy'] = PROXY_URL;
+ $contextOptions['http']['request_fulluri'] = true;
+
+ if(is_null($context)){
+ $context = stream_context_create($contextOptions);
+ } else {
+ $prevContext = $context;
+ if(!stream_context_set_option($context, $contextOptions)){
+ $context = $prevContext;
+ }
+ }
+ }
+
+ if(is_null($maxlen)){
+ $content = @file_get_contents($url, $use_include_path, $context, $offset);
+ } else {
+ $content = @file_get_contents($url, $use_include_path, $context, $offset, $maxlen);
+ }
+
+ if($content === false)
+ debugMessage('Cant\'t download ' . $url);
+
+ // handle compressed data
+ foreach($http_response_header as $header){
+ if(stristr($header, 'content-encoding')){
+ switch(true){
+ case stristr($header, 'gzip'):
+ $content = gzinflate(substr($content, 10, -8));
+ break;
+ case stristr($header, 'compress'):
+ //TODO
+ case stristr($header, 'deflate'):
+ //TODO
+ case stristr($header, 'brotli'):
+ //TODO
+ returnServerError($header . '=> Not implemented yet');
+ break;
+ case stristr($header, 'identity'):
+ break;
+ default:
+ returnServerError($header . '=> Unknown compression');
+ }
+ }
+ }
+
+ return $content;
+}
+
+function getSimpleHTMLDOM($url
+ , $use_include_path = false
+ , $context = null
+ , $offset = 0
+ , $maxLen = null
+ , $lowercase = true
+ , $forceTagsClosed = true
+ , $target_charset = DEFAULT_TARGET_CHARSET
+ , $stripRN = true
+ , $defaultBRText = DEFAULT_BR_TEXT
+ , $defaultSpanText = DEFAULT_SPAN_TEXT
+){
+ $content = getContents($url, $use_include_path, $context, $offset, $maxLen);
+ return str_get_html($content
+ , $lowercase
+ , $forceTagsClosed
+ , $target_charset
+ , $stripRN
+ , $defaultBRText
+ , $defaultSpanText);
+}
+
+/**
+ * Maintain locally cached versions of pages to avoid multiple downloads.
+ * @param url url to cache
+ * @param duration duration of the cache file in seconds (default: 24h/86400s)
+ * @return content of the file as string
+ */
+function getSimpleHTMLDOMCached($url
+ , $duration = 86400
+ , $use_include_path = false
+ , $context = null
+ , $offset = 0
+ , $maxLen = null
+ , $lowercase = true
+ , $forceTagsClosed = true
+ , $target_charset = DEFAULT_TARGET_CHARSET
+ , $stripRN = true
+ , $defaultBRText = DEFAULT_BR_TEXT
+ , $defaultSpanText = DEFAULT_SPAN_TEXT
+){
+ debugMessage('Caching url ' . $url . ', duration ' . $duration);
+
+ $filepath = __DIR__ . '/../cache/pages/' . sha1($url) . '.cache';
+ debugMessage('Cache file ' . $filepath);
+
+ if(file_exists($filepath) && filectime($filepath) < time() - $duration){
+ unlink ($filepath);
+ debugMessage('Cached file deleted: ' . $filepath);
+ }
+
+ if(file_exists($filepath)){
+ debugMessage('Loading cached file ' . $filepath);
+ touch($filepath);
+ $content = file_get_contents($filepath);
+ } else {
+ debugMessage('Caching ' . $url . ' to ' . $filepath);
+ $dir = substr($filepath, 0, strrpos($filepath, '/'));
+
+ if(!is_dir($dir)){
+ debugMessage('Creating directory ' . $dir);
+ mkdir($dir, 0777, true);
+ }
+
+ $content = getContents($url, $use_include_path, $context, $offset, $maxLen);
+ if($content !== false){
+ file_put_contents($filepath, $content);
+ }
+ }
+
+ return str_get_html($content
+ , $lowercase
+ , $forceTagsClosed
+ , $target_charset
+ , $stripRN
+ , $defaultBRText
+ , $defaultSpanText);
+}
+
+?>
diff --git a/lib/error.php b/lib/error.php
new file mode 100644
index 00000000..4a814b19
--- /dev/null
+++ b/lib/error.php
@@ -0,0 +1,30 @@
+'
+ . $calling['function'] . ' - '
+ . $text;
+
+ error_log($message);
+}
+
+?>