From de8cee6a1cc1c79a356d80b17d4318a86aeb9290 Mon Sep 17 00:00:00 2001 From: ORelio Date: Sun, 9 Sep 2018 21:20:13 +0200 Subject: [PATCH] Catching up | [Main] Debug mode, parse utils, MIME | [Bridges] Add/Improve 20 bridges (#802) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Debug mode improvements - Improve debug warning message - Restore error reporting in debug mode - Fix 'notice' messages for unset fields * Add parsing utility functions html.php - extractFromDelimiters - stripWithDelimiters - stripRecursiveHTMLSection - markdownToHtml (partial) bridges - remove now-duplicate functions - call functions from html.php instead * [Anidex] New bridge Anime torrent tracker * [Anime-Ultime] Restore thumbnail * [CNET] Recreate bridge Full rewrite as the previous one was broken * [Dilbert] Minor URI fix Use new self::URI property * [EstCeQuonMetEnProd] Fix content extraction Bridge was broken * [Facebook] Fix "SpSonsSoriSsés" label ... which was taking space in item title * [Futura-Sciences] Use HTTPS, More cleanup Use HTTPS as FS now offer HTTPS Clean additional useless HTML elements * [GBATemp] Multiple fixes - Fix categories: missing "break" statements - Restore thumbnail as enclosure - Fix date extraction - Fix user blog post extraction - Use getSimpleHTMLDOMCached * [JapanExpo] Fix bridge, HTTPS, thumbnails - Fix getSimpleHTMLDOMCached call - Upgrade to HTTPS as JE now offers HTTPS - Restore thumbnails as enclosures * [LeMondeInformatique] Fix bridge, HTTPS - Upgrade to HTTPS as LMI now offers HTTPS - Restore thumbnails using small images - Fix content extraction - Fix text encoding issue * [Nextgov] Fix content extraction - Restore thumbnail and use small image - Field extraction fixes * [NextInpact] Add categories and filtering by type - Offer all RSS feeds - Allow filtering by article type - Implement extraction for brief articles - Remove article limit, many brief articles are publied all at once * [NyaaTorrents] New bridge Anime torrent tracker * [Releases3DS] Cache content, restore thumbnail - Use getSimpleHTMLDOMCached - Restore thumbnail as enclosure * [TheHackerNews] Fix bridge - Fix content extraction including article body - Restore thumbnail as enclosure * [WeLiveSecurity] HTTPS, Fix content extraction - Upgrade to HTTPS as WLS now offers HTTPS - Fix content extraction including article body * [WordPress] Reduce timeout, more content selectors - Reduce timeout to use default one (1h) - Add new content selector (articleBody) - Find thumbnail and set as enclosure - Fix '); + $article_html = stripWithDelimiters($article_html, ''); + return $article_html; + } + + public function collectData() { + + // Retrieve and check user input + $topic = str_replace('-', '/', $this->getInput('topic')); + if (!empty($topic) && (substr_count($topic, '/') > 1 || !ctype_alpha(str_replace('/', '', $topic)))) + returnClientError('Invalid topic: ' . $topic); + + // Retrieve webpage + $pageUrl = self::URI . (empty($topic) ? 'news/' : $topic.'/'); + $html = getSimpleHTMLDOM($pageUrl) + or returnServerError('Could not request CNET: '.$pageUrl); + + // Process articles + foreach($html->find('div.assetBody, div.riverPost') as $element) { + + if(count($this->items) >= 10) { + break; + } + + $article_title = trim($element->find('h2, h3', 0)->plaintext); + $article_uri = self::URI . substr($element->find('a', 0)->href, 1); + $article_thumbnail = $element->parent()->find('img[src]', 0)->src; + $article_timestamp = strtotime($element->find('time.assetTime, div.timeAgo', 0)->plaintext); + $article_author = trim($element->find('a[rel=author], a.name', 0)->plaintext); + $article_content = '

' . trim($element->find('p.dek', 0)->plaintext) . '

'; + + if (is_null($article_thumbnail)) + $article_thumbnail = extractFromDelimiters($element->innertext, 'find('div.originalImage', 0); + if (empty($article_thumbnail)) + $article_thumbnail = $article_html->find('span.imageContainer', 0); + if (is_object($article_thumbnail)) + $article_thumbnail = $article_thumbnail->find('img', 0)->src; + + $article_content .= trim( + $this->cleanArticle( + extractFromDelimiters( + $article_html, 'items[] = $item; + } + } + } +} diff --git a/bridges/DilbertBridge.php b/bridges/DilbertBridge.php index 959a91a2..a84e5e87 100644 --- a/bridges/DilbertBridge.php +++ b/bridges/DilbertBridge.php @@ -9,8 +9,8 @@ class DilbertBridge extends BridgeAbstract { public function collectData(){ - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request Dilbert: ' . $this->getURI()); + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request Dilbert: ' . self::URI); foreach($html->find('section.comic-item') as $element) { diff --git a/bridges/EstCeQuonMetEnProdBridge.php b/bridges/EstCeQuonMetEnProdBridge.php index db9d1d5f..4439d694 100644 --- a/bridges/EstCeQuonMetEnProdBridge.php +++ b/bridges/EstCeQuonMetEnProdBridge.php @@ -7,19 +7,9 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Should we put a website in production today? (French)'; - public function collectData(){ - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request EstCeQuonMetEnProd: ' . $this->getURI()); + public function collectData() { + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request EstCeQuonMetEnProd: ' . self::URI); $item = array(); $item['uri'] = $this->getURI() . '#' . date('Y-m-d'); @@ -28,8 +18,8 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { $item['timestamp'] = strtotime('today midnight'); $item['content'] = str_replace( 'src="/', - 'src="' . $this->getURI(), - trim(extractFromDelimiters($html->outertext, '', '

')) + 'src="' . self::URI, + trim(extractFromDelimiters($html->outertext, '', '
]+ href="#" role="link" [^>}]+>.+<\/a>/iU', + '', + $content); + //Remove html nodes, keep only img, links, basic formatting $content = strip_tags($content, '

'); @@ -550,6 +545,9 @@ EOD; $item['title'] = $title; $item['author'] = $author; $item['timestamp'] = $date; + if(strpos($item['content'], 'items[] = $item; } } diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php index b9479c3d..772f4438 100644 --- a/bridges/FuturaSciencesBridge.php +++ b/bridges/FuturaSciencesBridge.php @@ -3,7 +3,7 @@ class FuturaSciencesBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'Futura-Sciences Bridge'; - const URI = 'http://www.futura-sciences.com/'; + const URI = 'https://www.futura-sciences.com/'; const DESCRIPTION = 'Returns the newest articles.'; const PARAMETERS = array( array( @@ -90,42 +90,11 @@ class FuturaSciencesBridge extends FeedExpander { or returnServerError('Could not request Futura-Sciences: ' . $item['uri']); $item['content'] = $this->extractArticleContent($article); $author = $this->extractAuthor($article); - $item['author'] = empty($author) ? $item['author'] : $author; + if (!empty($author)) + $item['author'] = $author; return $item; } - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } return $string; - } - - private function stripRecursiveHTMLSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = ''; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while ($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - private function extractArticleContent($article){ $contents = $article->find('section.article-text-classic', 0)->innertext; $headline = trim($article->find('p.description', 0)->plaintext); @@ -137,6 +106,7 @@ class FuturaSciencesBridge extends FeedExpander { '

stripRecursiveHTMLSection($contents, 'div', $div_start); + $contents = stripRecursiveHTMLSection($contents, 'div', $div_start); } - $contents = $this->stripWithDelimiters($contents, '
'); - $contents = $this->stripWithDelimiters($contents, '

'); - $contents = $this->stripWithDelimiters($contents, 'fs:definition="', '"'); - $contents = $this->stripWithDelimiters($contents, 'fs:xt:clicktype="', '"'); - $contents = $this->stripWithDelimiters($contents, 'fs:xt:clickname="', '"'); - $contents = $this->stripWithDelimiters($contents, ''); return $headline . trim($contents); } diff --git a/bridges/GBAtempBridge.php b/bridges/GBAtempBridge.php index f80a25ce..1266b0de 100644 --- a/bridges/GBAtempBridge.php +++ b/bridges/GBAtempBridge.php @@ -20,50 +20,58 @@ class GBAtempBridge extends BridgeAbstract { ) )); - private function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - private function buildItem($uri, $title, $author, $timestamp, $content){ + private function buildItem($uri, $title, $author, $timestamp, $thumbnail, $content){ $item = array(); $item['uri'] = $uri; $item['title'] = $title; $item['author'] = $author; $item['timestamp'] = $timestamp; $item['content'] = $content; + if (!empty($thumbnail)) { + $item['enclosures'] = array($thumbnail); + } return $item; } private function cleanupPostContent($content, $site_url){ $content = str_replace(':arrow:', '➤', $content); $content = str_replace('href="attachments/', 'href="'.$site_url.'attachments/', $content); - $content = $this->stripWithDelimiters($content, ''); + $content = stripWithDelimiters($content, ''); return $content; } + private function findItemDate($item){ + $time = 0; + $dateField = $item->find('abbr.DateTime', 0); + if (is_object($dateField)) { + $time = intval( + extractFromDelimiters( + $dateField->outertext, + 'data-time="', + '"' + ) + ); + } else { + $dateField = $item->find('span.DateTime', 0); + $time = DateTime::createFromFormat( + 'M j, Y \a\t g:i A', + extractFromDelimiters( + $dateField->outertext, + 'title="', + '"' + ) + )->getTimestamp(); + } + return $time; + } + private function fetchPostContent($uri, $site_url){ - $html = getSimpleHTMLDOM($uri); + $html = getSimpleHTMLDOMCached($uri); if(!$html) { - return 'Could not request GBAtemp ' . $uri; + return 'Could not request GBAtemp: ' . $uri; } - $content = $html->find('div.messageContent', 0)->innertext; + $content = $html->find('div.messageContent, blockquote.baseHtml', 0)->innertext; return $this->cleanupPostContent($content, $site_url); } @@ -76,70 +84,56 @@ class GBAtempBridge extends BridgeAbstract { case 'N': foreach($html->find('li[class=news_item full]') as $newsItem) { $url = self::URI . $newsItem->find('a', 0)->href; - $time = intval( - $this->extractFromDelimiters( - $newsItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $img = $this->getURI() . $newsItem->find('img', 0)->src . '#.image'; + $time = $this->findItemDate($newsItem); $author = $newsItem->find('a.username', 0)->plaintext; $title = $newsItem->find('a', 1)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content); + unset($newsItem); // Some items are heavy, freeing the item proactively helps saving memory } + break; case 'R': foreach($html->find('li.portal_review') as $reviewItem) { $url = self::URI . $reviewItem->find('a', 0)->href; + $img = $this->getURI() . extractFromDelimiters($reviewItem->find('a', 0)->style, 'image:url(', ')'); $title = $reviewItem->find('span.review_title', 0)->plaintext; $content = getSimpleHTMLDOM($url) or returnServerError('Could not request GBAtemp: ' . $uri); $author = $content->find('a.username', 0)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $content->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($content); $intro = '

' . ($content->find('div#review_intro', 0)->plaintext) . '

'; $review = $content->find('div#review_main', 0)->innertext; $subheader = '

' . $content->find('div.review_subheader', 0)->plaintext . '

'; $procons = $content->find('table.review_procons', 0)->outertext; $scores = $content->find('table.reviewscores', 0)->outertext; $content = $this->cleanupPostContent($intro . $review . $subheader . $procons . $scores, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content); + unset($reviewItem); // Free up memory } + break; case 'T': foreach($html->find('li.portal-tutorial') as $tutorialItem) { $url = self::URI . $tutorialItem->find('a', 0)->href; $title = $tutorialItem->find('a', 0)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $tutorialItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($tutorialItem); $author = $tutorialItem->find('a.username', 0)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, null, $content); + unset($tutorialItem); // Free up memory } + break; case 'F': foreach($html->find('li.rc_item') as $postItem) { $url = self::URI . $postItem->find('a', 1)->href; $title = $postItem->find('a', 1)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $postItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($postItem); $author = $postItem->find('a.username', 0)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, null, $content); + unset($postItem); // Free up memory } + break; } } diff --git a/bridges/JapanExpoBridge.php b/bridges/JapanExpoBridge.php index c80bb24c..f2dd972a 100644 --- a/bridges/JapanExpoBridge.php +++ b/bridges/JapanExpoBridge.php @@ -3,7 +3,7 @@ class JapanExpoBridge extends BridgeAbstract { const MAINTAINER = 'Ginko'; const NAME = 'Japan Expo Actualités'; - const URI = 'http://www.japan-expo-paris.com/fr/actualites'; + const URI = 'https://www.japan-expo-paris.com/fr/actualites'; const CACHE_TIMEOUT = 14400; // 4h const DESCRIPTION = 'Returns most recent entries from Japan Expo actualités.'; const PARAMETERS = array( array( @@ -51,7 +51,7 @@ class JapanExpoBridge extends BridgeAbstract { foreach($html->find('a._tile2') as $element) { $url = $element->href; - $thumbnail = 'http://s.japan-expo.com/katana/images/JES049/paris.png'; + $thumbnail = 'https://s.japan-expo.com/katana/images/JES049/paris.png'; preg_match('/url\(([^)]+)\)/', $element->find('img.rspvimgset', 0)->style, $img_search_result); if(count($img_search_result) >= 2) @@ -62,7 +62,8 @@ class JapanExpoBridge extends BridgeAbstract { break; } - $article_html = getSimpleHTMLDOMCached('Could not request JapanExpo: ' . $url); + $article_html = getSimpleHTMLDOMCached($url) + or returnServerError('Could not request JapanExpo: ' . $url); $header = $article_html->find('header.pageHeadBox', 0); $timestamp = strtotime($header->find('time', 0)->datetime); $title_html = $header->find('div.section', 0)->next_sibling(); @@ -92,6 +93,7 @@ class JapanExpoBridge extends BridgeAbstract { $item['uri'] = $url; $item['title'] = $title; $item['timestamp'] = $timestamp; + $item['enclosures'] = array($thumbnail); $item['content'] = $content; $this->items[] = $item; $count++; diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index 706752ff..09bcf6a3 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -3,8 +3,7 @@ class LeMondeInformatiqueBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'Le Monde Informatique'; - const URI = 'http://www.lemondeinformatique.fr/'; - const CACHE_TIMEOUT = 1800; // 30min + const URI = 'https://www.lemondeinformatique.fr/'; const DESCRIPTION = 'Returns the newest articles.'; public function collectData(){ @@ -15,30 +14,26 @@ class LeMondeInformatiqueBridge extends FeedExpander { $item = parent::parseItem($newsItem); $article_html = getSimpleHTMLDOMCached($item['uri']) or returnServerError('Could not request LeMondeInformatique: ' . $item['uri']); - $item['content'] = $this->cleanArticle($article_html->find('div#article', 0)->innertext); - $item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext; + + //Deduce thumbnail URL from article image URL + $item['enclosures'] = array( + str_replace( + '/grande/', + '/petite/', + $article_html->find('.article-image', 0)->find('img', 0)->src + ) + ); + + //No response header sets the encoding, explicit conversion is needed or subsequent xml_encode() will fail + $item['content'] = utf8_encode($this->cleanArticle($article_html->find('div.col-primary', 0)->innertext)); + $item['author'] = utf8_encode($article_html->find('div.author-infos', 0)->find('b', 0)->plaintext); + return $item; } - private function stripCDATA($string){ - $string = str_replace('', '', $string); - return $string; - } - - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - private function cleanArticle($article_html){ - $article_html = $this->stripWithDelimiters($article_html, ''); - $article_html = $this->stripWithDelimiters($article_html, '

'); + $article_html = stripWithDelimiters($article_html, ''); + $article_html = explode('

'; } $article_content .= str_replace( 'src="/', 'src="' . self::URI, $article_html->find('div.newsGenerique_con', 0)->innertext ); - $article_content = $this->stripWithDelimiters($article_content, ''); - $article_content = $this->stripWithDelimiters($article_content, ''); - $article_content = $this->stripWithDelimiters($article_content, ''); + $article_content = stripWithDelimiters($article_content, ''); + $article_content = stripWithDelimiters($article_content, ''); + $article_content = stripWithDelimiters($article_content, ''); $item['content'] = $article_content; diff --git a/bridges/NextInpactBridge.php b/bridges/NextInpactBridge.php index 5de5c8b1..c6bf2f53 100644 --- a/bridges/NextInpactBridge.php +++ b/bridges/NextInpactBridge.php @@ -6,29 +6,105 @@ class NextInpactBridge extends FeedExpander { const URI = 'https://www.nextinpact.com/'; const DESCRIPTION = 'Returns the newest articles.'; + const PARAMETERS = array( array( + 'feed' => array( + 'name' => 'Feed', + 'type' => 'list', + 'values' => array( + 'Tous nos articles' => 'news', + 'Nos contenus en accès libre' => 'acces-libre', + 'Blog' => 'blog', + 'Bons plans' => 'bonsplans' + ) + ), + 'filter_premium' => array( + 'name' => 'Premium', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'Hide Premium' => '1', + 'Only Premium' => '2' + ) + ), + 'filter_brief' => array( + 'name' => 'Brief', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'Hide Brief' => '1', + 'Only Brief' => '2' + ) + ) + )); + public function collectData(){ - $this->collectExpandableDatas(self::URI . 'rss/news.xml', 10); + $feed = $this->getInput('feed'); + if (empty($feed)) + $feed = 'news'; + $this->collectExpandableDatas(self::URI . 'rss/' . $feed . '.xml'); } protected function parseItem($newsItem){ $item = parent::parseItem($newsItem); - $item['content'] = $this->extractContent($item['uri']); + $item['content'] = $this->extractContent($item, $item['uri']); + if (is_null($item['content'])) + return null; //Filtered article return $item; } - private function extractContent($url){ - $html2 = getSimpleHTMLDOMCached($url); - $text = '

' - . $html2->find('span.sub_title', 0)->innertext - . '

-

' - . $html2->find('div[itemprop=articleBody]', 0)->innertext - . '
'; + private function extractContent($item, $url){ + $html = getSimpleHTMLDOMCached($url); + if (!is_object($html)) + return 'Failed to request NextInpact: ' . $url; + + foreach(array( + 'filter_premium' => 'h2.title_reserve_article', + 'filter_brief' => 'div.brief-inner-content' + ) as $param_name => $selector) { + $param_val = intval($this->getInput($param_name)); + if ($param_val != 0) { + $element_present = is_object($html->find($selector, 0)); + $element_wanted = ($param_val == 2); + if ($element_present != $element_wanted) { + return null; //Filter article + } + } + } + + if (is_object($html->find('div[itemprop=articleBody], div.brief-inner-content', 0))) { + + $subtitle = trim($html->find('span.sub_title, div.brief-head', 0)); + if(is_object($subtitle) && $subtitle->plaintext !== $item['title']) { + $subtitle = '

' . $subtitle->plaintext . '

'; + } else { + $subtitle = ''; + } + + $postimg = $html->find( + 'div.container_main_image_article, div.image-brief-container, div.image-brief-side-container', 0 + ); + if(is_object($postimg)) { + $postimg = '

-

'; + } else { + $postimg = ''; + } + + $text = $subtitle + . $postimg + . $html->find('div[itemprop=articleBody], div.brief-inner-content', 0)->outertext; + + } else { + $text = $item['content'] + . '

Failed retrieve full article content

'; + } + + $premium_article = $html->find('h2.title_reserve_article', 0); + if (is_object($premium_article)) { + $text .= '

' . $premium_article->innertext . '

'; + } - $premium_article = $html2->find('h2.title_reserve_article', 0); - if (is_object($premium_article)) - $text = $text . '

' . $premium_article->innertext . '

'; return $text; } } diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php index 370b0bf6..74bfc54a 100644 --- a/bridges/NextgovBridge.php +++ b/bridges/NextgovBridge.php @@ -32,43 +32,39 @@ class NextgovBridge extends FeedExpander { protected function parseItem($newsItem){ $item = parent::parseItem($newsItem); - $item['content'] = ''; + $article_thumbnail = 'https://cdn.nextgov.com/nextgov/images/logo.png'; + $item['content'] = '

' . $item['content'] . '

'; $namespaces = $newsItem->getNamespaces(true); if(isset($namespaces['media'])) { $media = $newsItem->children($namespaces['media']); if(isset($media->content)) { $attributes = $media->content->attributes(); - $item['content'] = ''; + $item['content'] = '

' . $item['content']; + $article_thumbnail = str_replace( + 'large.jpg', + 'small.jpg', + strval($attributes['url']) + ); } } + $item['enclosures'] = array($article_thumbnail); $item['content'] .= $this->extractContent($item['uri']); return $item; } - private function stripWithDelimiters($string, $start, $end){ - while (strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - private function extractContent($url){ - $article = getSimpleHTMLDOMCached($url) - or returnServerError('Could not request Nextgov: ' . $url); + $article = getSimpleHTMLDOMCached($url); - $contents = $article->find('div.wysiwyg', 0)->innertext; - $contents = $this->stripWithDelimiters($contents, '
', '
'); - $contents = $this->stripWithDelimiters($contents, ''); //ad outer div - return $this->stripWithDelimiters($contents, ''); - $contents = ($article_thumbnail == '' ? '' : '

') - . '

' - . $article_subtitle - . '

' - . trim($contents); + if (!is_object($article)) + return 'Could not request Nextgov: ' . $url; + + $contents = $article->find('div.wysiwyg', 0); + $contents->find('svg.content-tombstone', 0)->outertext = ''; + $contents = $contents->innertext; + $contents = stripWithDelimiters($contents, '
', '
'); + $contents = stripWithDelimiters($contents, ''); //ad outer div + return trim(stripWithDelimiters($contents, '')); } } diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php new file mode 100644 index 00000000..ba1cb9d2 --- /dev/null +++ b/bridges/NyaaTorrentsBridge.php @@ -0,0 +1,127 @@ + array( + 'name' => 'Filter', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'No remakes' => '1', + 'Trusted only' => '2' + ) + ), + 'c' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All categories' => '0_0', + 'Anime' => '1_0', + 'Anime - AMV' => '1_1', + 'Anime - English' => '1_2', + 'Anime - Non-English' => '1_3', + 'Anime - Raw' => '1_4', + 'Audio' => '2_0', + 'Audio - Lossless' => '2_1', + 'Audio - Lossy' => '2_2', + 'Literature' => '3_0', + 'Literature - English' => '3_1', + 'Literature - Non-English' => '3_2', + 'Literature - Raw' => '3_3', + 'Live Action' => '4_0', + 'Live Action - English' => '4_1', + 'Live Action - Idol/PV' => '4_2', + 'Live Action - Non-English' => '4_3', + 'Live Action - Raw' => '4_4', + 'Pictures' => '5_0', + 'Pictures - Graphics' => '5_1', + 'Pictures - Photos' => '5_2', + 'Software' => '6_0', + 'Software - Apps' => '6_1', + 'Software - Games' => '6_2', + ) + ), + 'q' => array( + 'name' => 'Keyword', + 'description' => 'Keyword(s)', + 'type' => 'text' + ) + ) + ); + + public function collectData() { + + // Build Search URL from user-provided parameters + $search_url = self::URI . '?s=id&o=desc&' + . http_build_query(array( + 'f' => $this->getInput('f'), + 'c' => $this->getInput('c'), + 'q' => $this->getInput('q') + )); + + // Retrieve torrent listing from search results, which does not contain torrent description + $html = getSimpleHTMLDOM($search_url) + or returnServerError('Could not request Nyaa: ' . $search_url); + $links = $html->find('a'); + $results = array(); + foreach ($links as $link) + if (strpos($link->href, '/view/') === 0 && !in_array($link->href, $results)) + $results[] = $link->href; + if (empty($results) && empty($this->getInput('q'))) + returnServerError('No results from Nyaa: ' . $url, 500); + + //Process each item individually + foreach ($results as $element) { + + //Limit total amount of requests + if(count($this->items) >= 20) { + break; + } + + $torrent_id = str_replace('/view/', '', $element); + + //Ignore entries without valid torrent ID + if ($torrent_id != 0 && ctype_digit($torrent_id)) { + + //Retrieve data for this torrent ID + $item_uri = self::URI . 'view/' . $torrent_id; + + //Retrieve full description from torrent page + if ($item_html = getSimpleHTMLDOMCached($item_uri)) { + + //Retrieve data from page contents + $item_title = str_replace(' :: Nyaa', '', $item_html->find('title', 0)->plaintext); + $item_desc = str_get_html(markdownToHtml($item_html->find('#torrent-description', 0)->innertext)); + $item_author = extractFromDelimiters($item_html->outertext, 'href="/user/', '"'); + $item_date = intval(extractFromDelimiters($item_html->outertext, 'data-timestamp="', '"')); + + //Retrieve image for thumbnail or generic logo fallback + $item_image = $this->getURI() . 'static/img/avatar/default.png'; + foreach ($item_desc->find('img') as $img) { + if (strpos($img->src, 'prez') === false) { + $item_image = $img->src; + break; + } + } + + //Build and add final item + $item = array(); + $item['uri'] = $item_uri; + $item['title'] = $item_title; + $item['author'] = $item_author; + $item['timestamp'] = $item_date; + $item['enclosures'] = array($item_image); + $item['content'] = $item_desc; + $this->items[] = $item; + } + } + $element = null; + } + $results = null; + } +} diff --git a/bridges/Releases3DSBridge.php b/bridges/Releases3DSBridge.php index a7e1778e..6c159d12 100644 --- a/bridges/Releases3DSBridge.php +++ b/bridges/Releases3DSBridge.php @@ -9,16 +9,6 @@ class Releases3DSBridge extends BridgeAbstract { public function collectData(){ - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - function typeToString($type){ switch($type) { case 1: return '3DS Game'; @@ -76,8 +66,8 @@ class Releases3DSBridge extends BridgeAbstract { $ignDate = time(); $ignCoverArt = ''; - $ignSearchUrl = 'http://www.ign.com/search?q=' . urlencode($name); - if($ignResult = getSimpleHTMLDOM($ignSearchUrl)) { + $ignSearchUrl = 'https://www.ign.com/search?q=' . urlencode($name); + if($ignResult = getSimpleHTMLDOMCached($ignSearchUrl)) { $ignCoverArt = $ignResult->find('div.search-item-media', 0)->find('img', 0)->src; $ignDesc = $ignResult->find('div.search-item-description', 0)->plaintext; $ignLink = $ignResult->find('div.search-item-sub-title', 0)->find('a', 1)->href; @@ -127,6 +117,7 @@ class Releases3DSBridge extends BridgeAbstract { $item['title'] = $name; $item['author'] = $publisher; $item['timestamp'] = $ignDate; + $item['enclosures'] = array($ignCoverArt); $item['uri'] = empty($ignLink) ? $searchLinkDuckDuckGo : $ignLink; $item['content'] = $ignDescription . $releaseDescription . $releaseSearchLinks; $this->items[] = $item; diff --git a/bridges/TheHackerNewsBridge.php b/bridges/TheHackerNewsBridge.php index 41066588..d0d2e973 100644 --- a/bridges/TheHackerNewsBridge.php +++ b/bridges/TheHackerNewsBridge.php @@ -8,67 +8,66 @@ class TheHackerNewsBridge extends BridgeAbstract { public function collectData(){ - function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - function stripRecursiveHtmlSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = ''; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr( - $string, - $section_start, - $section_end - $section_start + $close_tag_length - ); - - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request TheHackerNews: ' . $this->getURI()); $limit = 0; - foreach($html->find('article') as $element) { + foreach($html->find('div.body-post') as $element) { if($limit < 5) { - $article_url = $element->find('a.entry-title', 0)->href; - $article_author = trim($element->find('span.vcard', 0)->plaintext); - $article_title = $element->find('a.entry-title', 0)->plaintext; - $article_timestamp = strtotime($element->find('span.updated', 0)->plaintext); - $article = getSimpleHTMLDOM($article_url) - or returnServerError('Could not request TheHackerNews: ' . $article_url); + $article_url = $element->find('a.story-link', 0)->href; + $article_author = trim($element->find('i.fa-user', 0)->parent()->plaintext); + $article_title = $element->find('h2.home-title', 0)->plaintext; - $contents = $article->find('div.articlebodyonly', 0)->innertext; - $contents = stripRecursiveHtmlSection($contents, 'div', '
'); + //Date without time + $article_timestamp = strtotime( + extractFromDelimiters( + $element->find('i.fa-calendar', 0)->parent()->outertext, + '', + '' + ) + ); + + //Article thumbnail in lazy-loading image + if (is_object($element->find('img[data-echo]', 0))) { + $article_thumbnail = array( + extractFromDelimiters( + $element->find('img[data-echo]', 0)->outertext, + "data-echo='", + "'" + ) + ); + } else { + $article_thumbnail = array(); + } + + if ($article = getSimpleHTMLDOMCached($article_url)) { + + //Article body + $contents = $article->find('div.articlebody', 0)->innertext; + $contents = stripRecursiveHtmlSection($contents, 'div', '
'); + $contents = stripWithDelimiters($contents, ''); + + //Date with time + if (is_object($article->find('meta[itemprop=dateModified]', 0))) { + $article_timestamp = strtotime( + extractFromDelimiters( + $article->find('meta[itemprop=dateModified]', 0)->outertext, + "content='", + "'" + ) + ); + } + } else { + $contents = 'Could not request TheHackerNews: ' . $article_url; + } $item = array(); $item['uri'] = $article_url; $item['title'] = $article_title; $item['author'] = $article_author; + $item['enclosures'] = $article_thumbnail; $item['timestamp'] = $article_timestamp; $item['content'] = trim($contents); $this->items[] = $item; diff --git a/bridges/WeLiveSecurityBridge.php b/bridges/WeLiveSecurityBridge.php index 466a4b28..59a094a7 100644 --- a/bridges/WeLiveSecurityBridge.php +++ b/bridges/WeLiveSecurityBridge.php @@ -3,37 +3,24 @@ class WeLiveSecurityBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'We Live Security'; - const URI = 'http://www.welivesecurity.com/'; + const URI = 'https://www.welivesecurity.com/'; const DESCRIPTION = 'Returns the newest articles.'; - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - protected function parseItem($item){ $item = parent::parseItem($item); $article_html = getSimpleHTMLDOMCached($item['uri']); if(!$article_html) { - $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; + $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; return $item; } - $article_content = $article_html->find('div.wlistingsingletext', 0)->innertext; - $article_content = $this->stripWithDelimiters($article_content, ''); - $article_content = '

' - . $item['content'] - . '

' - . trim($article_content); - - $item['content'] = $article_content; + $article_content = $article_html->find('div.formatted', 0)->innertext; + $article_content = stripWithDelimiters($article_content, ''); + $article_content = stripRecursiveHTMLSection($article_content, 'div', '
/', '', $content); return $content; @@ -27,6 +26,10 @@ class WordPressBridge extends FeedExpander { $article = null; switch(true) { + case !is_null($article_html->find('[itemprop=articleBody]', 0)): + // highest priority content div + $article = $article_html->find('[itemprop=articleBody]', 0); + break; case !is_null($article_html->find('article', 0)): // most common content div $article = $article_html->find('article', 0); @@ -39,15 +42,37 @@ class WordPressBridge extends FeedExpander { // another common content div $article = $article_html->find('.post-content', 0); break; - case !is_null($article_html->find('.post', 0)): // for old WordPress themes without HTML5 $article = $article_html->find('.post', 0); break; } + foreach ($article->find('h1.entry-title') as $title) + if ($title->plaintext == $item['title']) + $title->outertext = ''; + + $article_image = $article_html->find('img.wp-post-image', 0); + if(!empty($item['content']) && (!is_object($article_image) || empty($article_image->src))) { + $article_image = str_get_html($item['content'])->find('img.wp-post-image', 0); + } + if(is_object($article_image) && !empty($article_image->src)) { + if(empty($article_image->getAttribute('data-lazy-src'))) { + $article_image = $article_image->src; + } else { + $article_image = $article_image->getAttribute('data-lazy-src'); + } + $mime_type = getMimeType($article_image); + if (strpos($mime_type, 'image') === false) + $article_image .= '#.image'; // force image + if (empty($item['enclosures'])) + $item['enclosures'] = array($article_image); + else + $item['enclosures'] = array_merge($item['enclosures'], $article_image); + } + if(!is_null($article)) { - $item['content'] = $this->clearContent($article->innertext); + $item['content'] = $this->cleanContent($article->innertext); } return $item; diff --git a/bridges/YGGTorrentBridge.php b/bridges/YGGTorrentBridge.php index 74d0aa05..f057d87c 100644 --- a/bridges/YGGTorrentBridge.php +++ b/bridges/YGGTorrentBridge.php @@ -110,8 +110,8 @@ class YGGTorrentBridge extends BridgeAbstract { foreach($results->find('tr') as $row) { $count++; - if($count == 1) continue; - if($count == 12) break; + if($count == 1) continue; // Skip table header + if($count == 22) break; // Stop processing after 21 items (20 + 1 table header) $item = array(); $item['timestamp'] = $row->find('.hidden', 1)->plaintext; $item['title'] = $row->find('a', 1)->plaintext; @@ -135,7 +135,7 @@ class YGGTorrentBridge extends BridgeAbstract { $url_full[5] = urlencode($url_full[5]); $url_full[6] = urlencode($url_full[6]); $url = implode('/', $url_full); - $page = getSimpleHTMLDOM($url) or returnServerError('Unable to query Yggtorrent page !'); + $page = getSimpleHTMLDOMCached($url) or returnServerError('Unable to query Yggtorrent page !'); $author = $page->find('.informations', 0)->find('a', 4)->plaintext; $content = $page->find('.default', 1); return array('author' => $author, 'content' => $content); diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php index 86e4b494..75df3b15 100644 --- a/bridges/ZDNetBridge.php +++ b/bridges/ZDNetBridge.php @@ -1,9 +1,9 @@ ', '', $string); - return trim($string); - } - - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - function stripRecursiveHtmlSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = ''; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr( - $string, - $section_start, - $section_end - $section_start + $close_tag_length - ); - - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while ($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - - $baseUri = self::URI; + $baseUri = static::URI; $feed = $this->getInput('feed'); if(strpos($feed, 'downloads!') !== false) { $feed = str_replace('downloads!', '', $feed); $baseUri = str_replace('www.', 'downloads.', $baseUri); } $url = $baseUri . trim($feed, '/') . '/rss.xml'; - $html = getSimpleHTMLDOM($url) - or returnServerError('Could not request ZDNet: ' . $url); - $limit = 0; + $this->collectExpandableDatas($url); + } - foreach($html->find('item') as $element) { - if($limit < 10) { - $article_url = preg_replace( - '/([^#]+)#ftag=.*/', - '$1', - stripCdata(extractFromDelimiters($element->innertext, '', '')) - ); + protected function parseItem($item){ + $item = parent::parseItem($item); - $article_author = stripCdata(extractFromDelimiters($element->innertext, 'role="author">', '<')); - $article_title = stripCdata($element->find('title', 0)->plaintext); - $article_subtitle = stripCdata($element->find('description', 0)->plaintext); - $article_timestamp = strtotime(stripCdata($element->find('pubDate', 0)->plaintext)); - $article = getSimpleHTMLDOM($article_url) - or returnServerError('Could not request ZDNet: ' . $article_url); + $article = getSimpleHTMLDOMCached($item['uri']); + if(!$article) + returnServerError('Could not request ZDNet: ' . $url); - if(!empty($article_author)) { - $author = $article_author; - } else { - $author = $article->find('meta[name=author]', 0); - if(is_object($author)) { - $author = $author->content; - } else { - $author = 'ZDNet'; - } - } - - $thumbnail = $article->find('meta[itemprop=image]', 0); - if(is_object($thumbnail)) { - $thumbnail = $thumbnail->content; - } else { - $thumbnail = ''; - } - - $contents = $article->find('article', 0)->innertext; - foreach(array( - '
'); - $contents = stripWithDelimiters($contents, ''); - $contents = trim(stripWithDelimiters($contents, '

'; //Include thumbnail - } - $contents = $content_img - . '

' - . $article_subtitle - . '

' - . $contents; - - $item = array(); - $item['author'] = $author; - $item['uri'] = $article_url; - $item['title'] = $article_title; - $item['timestamp'] = $article_timestamp; - $item['content'] = $contents; - $this->items[] = $item; - $limit++; - } + $contents = $article->find('article', 0)->innertext; + foreach(array( + '
'); + $contents = stripWithDelimiters($contents, ''); + $contents = stripWithDelimiters($contents, '')); + $item['content'] = $contents; + + return $item; } } diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 7d69535e..5abb1c3a 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -39,7 +39,7 @@ class AtomFormat extends FormatAbstract{ foreach($item['enclosures'] as $enclosure) { $entryEnclosures .= '' + . '" type="' . getMimeType($enclosure) . '" />' . PHP_EOL; } } diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 6d079288..72a829e7 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -37,7 +37,7 @@ class MrssFormat extends FormatAbstract { if(isset($item['enclosures'])) { $entryEnclosures .= ''; + . '" type="' . getMimeType($item['enclosures'][0]) . '" />'; if(count($item['enclosures']) > 1) { $entryEnclosures .= PHP_EOL; @@ -45,7 +45,7 @@ class MrssFormat extends FormatAbstract { Some media files might not be shown to you. Consider using the ATOM format instead!'; foreach($item['enclosures'] as $enclosure) { $entryEnclosures .= '' + . $enclosure . '" type="' . getMimeType($enclosure) . '" />' . PHP_EOL; } } diff --git a/index.php b/index.php index 442ebc9b..a5a1786a 100644 --- a/index.php +++ b/index.php @@ -11,12 +11,18 @@ if(file_exists('DEBUG')) { $debug_whitelist = trim(file_get_contents('DEBUG')); $debug_enabled = empty($debug_whitelist) - || in_array($_SERVER['REMOTE_ADDR'], explode("\n", $debug_whitelist)); + || in_array($_SERVER['REMOTE_ADDR'], + explode("\n", str_replace("\r", '', $debug_whitelist) + ) + ); if($debug_enabled) { ini_set('display_errors', '1'); error_reporting(E_ALL); define('DEBUG', true); + if (empty($debug_whitelist)) { + define('DEBUG_INSECURE', true); + } } } @@ -36,14 +42,17 @@ Configuration::loadConfiguration(); Authentication::showPromptIfNeeded(); date_default_timezone_set('UTC'); -error_reporting(0); /* Move the CLI arguments to the $_GET array, in order to be able to use rss-bridge from the command line */ -parse_str(implode('&', array_slice($argv, 1)), $cliArgs); -$params = array_merge($_GET, $cliArgs); +if (isset($argv)) { + parse_str(implode('&', array_slice($argv, 1)), $cliArgs); + $params = array_merge($_GET, $cliArgs); +} else { + $params = $_GET; +} // FIXME : beta test UA spoofing, please report any blacklisting by PHP-fopen-unfriendly websites diff --git a/lib/BridgeList.php b/lib/BridgeList.php index e3f55cc2..c1756133 100644 --- a/lib/BridgeList.php +++ b/lib/BridgeList.php @@ -55,17 +55,27 @@ EOD; } private static function getHeader() { - $status = ''; + $warning = ''; if(defined('DEBUG') && DEBUG === true) { - $status .= 'debug mode active'; + if(defined('DEBUG_INSECURE') && DEBUG_INSECURE === true) { + $warning .= <<Warning : Debug mode is active from any location, +make sure only you can access RSS-Bridge.
+EOD; + } else { + $warning .= <<Warning : Debug mode is active from your IP address, +your requests will bypass the cache. +EOD; + } } return <<

RSS-Bridge

-

·Reconnecting the Web·

-

{$status}

+

Reconnecting the Web

+ {$warning} EOD; } diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index d3417a7f..fb300da0 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -115,12 +115,27 @@ abstract class FeedExpander extends BridgeAbstract { } protected function parseATOMItem($feedItem){ - $item = array(); + // Some ATOM entries also contain RSS 2.0 fields + $item = $this->parseRSS_2_0_Item($feedItem); + if(isset($feedItem->id)) $item['uri'] = (string)$feedItem->id; if(isset($feedItem->title)) $item['title'] = (string)$feedItem->title; if(isset($feedItem->updated)) $item['timestamp'] = strtotime((string)$feedItem->updated); if(isset($feedItem->author)) $item['author'] = (string)$feedItem->author->name; if(isset($feedItem->content)) $item['content'] = (string)$feedItem->content; + + //When "link" field is present, URL is more reliable than "id" field + if (count($feedItem->link) === 1) { + $this->uri = (string)$feedItem->link[0]['href']; + } else { + foreach($feedItem->link as $link) { + if(strtolower($link['rel']) === 'alternate') { + $item['uri'] = (string)$link['href']; + break; + } + } + } + return $item; } @@ -130,6 +145,7 @@ abstract class FeedExpander extends BridgeAbstract { if(isset($feedItem->title)) $item['title'] = (string)$feedItem->title; // rss 0.91 doesn't support timestamps // rss 0.91 doesn't support authors + // rss 0.91 doesn't support enclosures if(isset($feedItem->description)) $item['content'] = (string)$feedItem->description; return $item; } @@ -154,11 +170,17 @@ abstract class FeedExpander extends BridgeAbstract { $namespaces = $feedItem->getNamespaces(true); if(isset($namespaces['dc'])) $dc = $feedItem->children($namespaces['dc']); + if(isset($namespaces['media'])) $media = $feedItem->children($namespaces['media']); if(isset($feedItem->guid)) { foreach($feedItem->guid->attributes() as $attribute => $value) { if($attribute === 'isPermaLink' - && ($value === 'true' || filter_var($feedItem->guid, FILTER_VALIDATE_URL))) { + && ($value === 'true' || ( + filter_var($feedItem->guid, FILTER_VALIDATE_URL) + && !filter_var($item['uri'], FILTER_VALIDATE_URL) + ) + ) + ) { $item['uri'] = (string)$feedItem->guid; break; } @@ -170,11 +192,21 @@ abstract class FeedExpander extends BridgeAbstract { } elseif(isset($dc->date)) { $item['timestamp'] = strtotime((string)$dc->date); } + if(isset($feedItem->author)) { $item['author'] = (string)$feedItem->author; + } elseif (isset($feedItem->creator)) { + $item['author'] = (string)$feedItem->creator; } elseif(isset($dc->creator)) { $item['author'] = (string)$dc->creator; + } elseif(isset($media->credit)) { + $item['author'] = (string)$media->credit; } + + if(isset($feedItem->enclosure) && !empty($feedItem->enclosure['url'])) { + $item['enclosures'] = array((string)$feedItem->enclosure['url']); + } + return $item; } @@ -199,14 +231,14 @@ abstract class FeedExpander extends BridgeAbstract { } public function getURI(){ - return $this->uri ?: parent::getURI(); + return !empty($this->uri) ? $this->uri : parent::getURI(); } public function getName(){ - return $this->name ?: parent::getName(); + return !empty($this->name) ? $this->name : parent::getName(); } public function getIcon(){ - return $this->icon ?: parent::getIcon(); + return !empty($this->icon) ? $this->icon : parent::getIcon(); } } diff --git a/lib/contents.php b/lib/contents.php index 0576983f..00202c44 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -21,7 +21,7 @@ function getContents($url, $header = array(), $opts = array()){ curl_setopt($ch, CURLOPT_PROXY, PROXY_URL); } - // We always want the resonse header as part of the data! + // We always want the response header as part of the data! curl_setopt($ch, CURLOPT_HEADER, true); $data = curl_exec($ch); @@ -152,3 +152,53 @@ function parseResponseHeader($header) { return $headers; } + +/** + * Determine MIME type from URL/Path file extension + * Remark: Built-in functions mime_content_type or fileinfo requires fetching remote content + * Remark: A bridge can hint for a MIME type by appending #.ext to a URL, e.g. #.image + * Based on https://stackoverflow.com/a/1147952 + */ +function getMimeType($url) { + static $mime = null; + + if (is_null($mime)) { + $mime = array( + 'jpg' => 'image/jpeg', + 'gif' => 'image/gif', + 'png' => 'image/png', + 'image' => 'image/*' + ); + if (is_file('/etc/mime.types')) { + $file = fopen('/etc/mime.types', 'r'); + while(($line = fgets($file)) !== false) { + $line = trim(preg_replace('/#.*/', '', $line)); + if(!$line) + continue; + $parts = preg_split('/\s+/', $line); + if(count($parts) == 1) + continue; + $type = array_shift($parts); + foreach($parts as $part) + $mime[$part] = $type; + } + fclose($file); + } + } + + if (strpos($url, '?') !== false) { + $url_temp = substr($url, 0, strpos($url, '?')); + if (strpos($url, '#') !== false) { + $anchor = substr($url, strpos($url, '#')); + $url_temp .= $anchor; + } + $url = $url_temp; + } + + $ext = strtolower(pathinfo($url, PATHINFO_EXTENSION)); + if (!empty($mime[$ext])) { + return $mime[$ext]; + } + + return 'application/octet-stream'; +} diff --git a/lib/error.php b/lib/error.php index ae18f6ff..34ad9dd3 100644 --- a/lib/error.php +++ b/lib/error.php @@ -20,7 +20,7 @@ function debugMessage($text){ $calling = $backtrace[2]; $message = $calling['file'] . ':' . $calling['line'] . ' class ' - . $calling['class'] . '->' + . (isset($calling['class']) ? $calling['class'] : '') . '->' . $calling['function'] . ' - ' . $text; diff --git a/lib/html.php b/lib/html.php index 5098f6cb..52338bb0 100644 --- a/lib/html.php +++ b/lib/html.php @@ -40,7 +40,19 @@ function backgroundToImg($htmlContent) { } +/** + * Convert relative links in HTML into absolute links + * @param $content HTML content to fix. Supports HTML objects or string objects + * @param $server full URL to the page containing relative links + * @return content with fixed URLs, as HTML object or string depending on input type + */ function defaultLinkTo($content, $server){ + $string_convert = false; + if (is_string($content)) { + $string_convert = true; + $content = str_get_html($content); + } + foreach($content->find('img') as $image) { $image->src = urljoin($server, $image->src); } @@ -49,5 +61,116 @@ function defaultLinkTo($content, $server){ $anchor->href = urljoin($server, $anchor->href); } + if ($string_convert) { + $content = $content->outertext; + } + return $content; } + +/** + * Extract the first part of a string matching the specified start and end delimiters + * @param $string input string, e.g. '
Post author: John Doe
' + * @param $start start delimiter, e.g. 'author: ' + * @param $end end delimiter, e.g. '<' + * @return extracted string, e.g. 'John Doe', or false if the delimiters were not found. + */ +function extractFromDelimiters($string, $start, $end) { + if (strpos($string, $start) !== false) { + $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); + $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); + return $section_retrieved; + } return false; +} + +/** + * Remove one or more part(s) of a string using a start and end delmiters + * @param $string input string, e.g. 'foobar' + * @param $start start delimiter, e.g. '' + * @return cleaned string, e.g. 'foobar' + */ +function stripWithDelimiters($string, $start, $end) { + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + return $string; +} + +/** + * Remove HTML sections containing one or more sections using the same HTML tag + * @param $string input string, e.g. 'foo
ads
ads
bar' + * @param $tag_name name of the HTML tag, e.g. 'div' + * @param $tag_start start of the HTML tag to remove, e.g. '
' + * @return cleaned string, e.g. 'foobar' + */ +function stripRecursiveHTMLSection($string, $tag_name, $tag_start){ + $open_tag = '<' . $tag_name; + $close_tag = ''; + $close_tag_length = strlen($close_tag); + if(strpos($tag_start, $open_tag) === 0) { + while(strpos($string, $tag_start) !== false) { + $max_recursion = 100; + $section_to_remove = null; + $section_start = strpos($string, $tag_start); + $search_offset = $section_start; + do { + $max_recursion--; + $section_end = strpos($string, $close_tag, $search_offset); + $search_offset = $section_end + $close_tag_length; + $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); + $open_tag_count = substr_count($section_to_remove, $open_tag); + $close_tag_count = substr_count($section_to_remove, $close_tag); + } while ($open_tag_count > $close_tag_count && $max_recursion > 0); + $string = str_replace($section_to_remove, '', $string); + } + } + return $string; +} + +/** + * Convert Markdown tags into HTML tags. Only a subset of the Markdown syntax is implemented. + * @param $string input string in Markdown format + * @return output string in HTML format + */ +function markdownToHtml($string) { + + //For more details about how these regex work: + // https://github.com/RSS-Bridge/rss-bridge/pull/802#discussion_r216138702 + // Images: https://regex101.com/r/JW9Evr/1 + // Links: https://regex101.com/r/eRGVe7/1 + // Bold: https://regex101.com/r/2p40Y0/1 + // Italic: https://regex101.com/r/xJkET9/1 + // Separator: https://regex101.com/r/ZBEqFP/1 + // Plain URL: https://regex101.com/r/2JHYwb/1 + // Site name: https://regex101.com/r/qIuKYE/1 + + $string = preg_replace('/\!\[([^\]]+)\]\(([^\) ]+)(?: [^\)]+)?\)/', '$1', $string); + $string = preg_replace('/\[([^\]]+)\]\(([^\)]+)\)/', '$1', $string); + $string = preg_replace('/\*\*(.*)\*\*/U', '$1', $string); + $string = preg_replace('/\*(.*)\*/U', '$1', $string); + $string = preg_replace('/__(.*)__/U', '$1', $string); + $string = preg_replace('/_(.*)_/U', '$1', $string); + $string = preg_replace('/[-]{6,99}/', '
', $string); + $string = str_replace(' ', '
', $string); + $string = preg_replace('/([^"])(https?:\/\/[^ "<]+)([^"])/', '$1$2$3', $string.' '); + $string = preg_replace('/([^"\/])(www\.[^ "<]+)([^"])/', '$1$2$3', $string.' '); + + //As the regex are not perfect, we need to fix and that are introduced in URLs + // Fixup regex : https://regex101.com/r/NTRPf6/1 + // Fixup regex : https://regex101.com/r/aNklRp/1 + + $count = 1; + while($count > 0) { + $string = preg_replace('/ (src|href)="([^"]+)([^"]+)"/U', ' $1="$2_$3"', $string, -1, $count); + } + + $count = 1; + while($count > 0) { + $string = preg_replace('/ (src|href)="([^"]+)<\/i>([^"]+)"/U', ' $1="$2_$3"', $string, -1, $count); + } + + return '
' . trim($string) . '
'; +} diff --git a/static/style.css b/static/style.css index 5354ffb9..585b20a0 100644 --- a/static/style.css +++ b/static/style.css @@ -40,11 +40,17 @@ a:hover { margin-left: 1em; font-size: 200%; } - header > p.status { - font-weight: bold; - margin: 1em; - color: red; + header > section.warning { + width: 40%; + background-color: #ffc600; + color: #5f5f5f; } + header > section.critical-warning { + width: 40%; + background-color: #cf3e3e; + font-weight: bold; + color: white; + } /* Input boxes */ input[type="text"] {