From 47dd6c3b03abc3d02a8465152da0356f507609e4 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 3 Feb 2016 20:36:16 +0100 Subject: [PATCH 1/6] [LeMondeInformatique] fix title stripping Title wasn't properly stripped due to changes on their website. --- bridges/LeMondeInformatiqueBridge.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index a0c16b2b..e2ba4636 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -1,15 +1,15 @@ maintainer = "ORelio"; - $this->name = "Le Monde Informatique"; - $this->uri = "http://www.lemondeinformatique.fr/"; - $this->description = "Returns the newest articles."; - $this->update = "2015-09-08"; + $this->maintainer = "ORelio"; + $this->name = "Le Monde Informatique"; + $this->uri = "http://www.lemondeinformatique.fr/"; + $this->description = "Returns the newest articles."; + $this->update = "2016-01-28"; - } + } public function collectData(array $param) { @@ -29,7 +29,7 @@ class LeMondeInformatiqueBridge extends BridgeAbstract { function CleanArticle($article_html) { $article_html = StripWithDelimiters($article_html, ''); - $article_html = StripWithDelimiters($article_html, '

', '

'); + $article_html = StripWithDelimiters($article_html, '

'); return $article_html; } @@ -73,6 +73,5 @@ class LeMondeInformatiqueBridge extends BridgeAbstract { public function getCacheDuration() { return 1800; // 30 minutes - // return 0; } } From e1b7ffdc81146b84669df997d293688f2d1731dc Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 3 Feb 2016 20:37:09 +0100 Subject: [PATCH 2/6] [T411] Change encoding to UTF-8 - Stop encoding to UTF-8, their website is now natively in UTF-8 - More error handling --- bridges/T411Bridge.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bridges/T411Bridge.php b/bridges/T411Bridge.php index e1e7a951..da2e9d05 100644 --- a/bridges/T411Bridge.php +++ b/bridges/T411Bridge.php @@ -37,7 +37,9 @@ class T411Bridge extends BridgeAbstract { //Retrieve torrent listing as truncated rss, which does not contain torrent description $url = 'http://www.t411.in/torrents/search/?'.$param['search'].'&order=added&type=desc'; $html = file_get_html($url) or $this->returnError('Could not request t411: '.$url, 500); - $results = $html->find('table.results')[0] or $this->returnError('No results from t411: '.$url, 500); + $results = $html->find('table.results', 0); + if (is_null($results)) + $this->returnError('No results from t411: '.$url, 500); $limit = 0; //Process each item individually @@ -77,7 +79,7 @@ class T411Bridge extends BridgeAbstract { $item->author = $item_author; $item->timestamp = $item_date; $item->thumbnailUri = $item_image; - $item->content = utf8_encode($item_desc->innertext); + $item->content = $item_desc->innertext; $this->items[] = $item; $limit++; } From a9c40e98c470414b3c82d34929fed0deaebd0f0c Mon Sep 17 00:00:00 2001 From: ORelio Date: Sat, 13 Feb 2016 13:52:07 +0100 Subject: [PATCH 3/6] [CNET] Improve script removal --- bridges/CNETBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php index 7b4e1b31..ec7decf0 100644 --- a/bridges/CNETBridge.php +++ b/bridges/CNETBridge.php @@ -9,7 +9,7 @@ class CNETBridge extends BridgeAbstract { $this->name = 'CNET News'; $this->uri = 'http://www.cnet.com/'; $this->description = 'Returns the newest articles.
You may specify a topic found in some section URLs, else all topics are selected.'; - $this->update = '2016-01-23'; + $this->update = '2016-02-06'; $this->parameters[] = '[ @@ -41,7 +41,7 @@ class CNETBridge extends BridgeAbstract { function CleanArticle($article_html) { $article_html = '

'.substr($article_html, strpos($article_html, '

') + 3); $article_html = StripWithDelimiters($article_html, '', ''); - $article_html = StripWithDelimiters($article_html, ''); + $article_html = StripWithDelimiters($article_html, ''); $article_html = StripWithDelimiters($article_html, '

', ''); return $article_html; From bba6a0b5c2555b70809fb64011a5f460851d6dc2 Mon Sep 17 00:00:00 2001 From: ORelio Date: Sat, 13 Feb 2016 13:52:28 +0100 Subject: [PATCH 4/6] [9emeArt] Add NeuviemeArtBridge --- bridges/NeuviemeArtBridge.php | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 bridges/NeuviemeArtBridge.php diff --git a/bridges/NeuviemeArtBridge.php b/bridges/NeuviemeArtBridge.php new file mode 100644 index 00000000..d558b2d3 --- /dev/null +++ b/bridges/NeuviemeArtBridge.php @@ -0,0 +1,70 @@ +maintainer = "ORelio"; + $this->name = "9ème Art"; + $this->uri = "http://www.9emeart.fr/"; + $this->description = "Returns the newest articles."; + $this->update = "2016-02-05"; + } + + public function collectData(array $param) { + + function StripWithDelimiters($string, $start, $end) { + while (strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } return $string; + } + + $feedUrl = 'http://www.9emeart.fr/9emeart.rss'; + $html = file_get_html($feedUrl) or $this->returnError('Could not request 9eme Art: '.$feedUrl, 500); + $limit = 0; + + foreach ($html->find('item') as $element) { + if ($limit < 5) { + + //Retrieve article Uri and get that page + $article_uri = $element->find('guid', 0)->plaintext; + $article_html = file_get_html($article_uri) or $this->returnError('Could not request 9eme Art: '.$article_uri, 500); + + //Build article contents from corresponding elements + $article_title = trim($element->find('title', 0)->plaintext); + $article_image = $element->find('enclosure', 0)->url; + foreach ($article_html->find('img.img_full') as $img) + if ($img->alt == $article_title) + $article_image = 'http://www.9emeart.fr'.$img->src; + $article_content = '

' + .str_replace('src="/', 'src="http://www.9emeart.fr/', $article_html->find('div.newsGenerique_con', 0)->innertext); + $article_content = StripWithDelimiters($article_content, ''); + $article_content = StripWithDelimiters($article_content, ''); + $article_content = StripWithDelimiters($article_content, ''); + + //Build and add final item + $item = new \Item(); + $item->uri = $article_uri; + $item->title = $article_title; + $item->thumbnailUri = $element->find('enclosure', 0)->url; + $item->author = $article_html->find('a[class=upp transition_fast upp]', 0)->plaintext; + $item->timestamp = strtotime($element->find('pubDate', 0)->plaintext); + $item->content = $article_content; + $this->items[] = $item; + $limit++; + } + } + } + + public function getName() { + return '9ème Art Bridge'; + } + + public function getURI() { + return 'http://www.9emeart.fr/'; + } + + public function getCacheDuration() { + return 3600; //1 hour + } +} \ No newline at end of file From 91fe6c4accd308427f4f7a8572d88cc64abcf7d1 Mon Sep 17 00:00:00 2001 From: ORelio Date: Sat, 13 Feb 2016 14:01:21 +0100 Subject: [PATCH 5/6] [t411] Increase item limit On searches for popular keywords, a feed reader not reading the feed often enough can miss items (ie more than 5 items were added since last fetch) so this change increases the limit from 5 items to 10 items, and retrieves pages faster to compensate (500ms seems enough for not spamming t411). --- bridges/T411Bridge.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridges/T411Bridge.php b/bridges/T411Bridge.php index da2e9d05..54904cce 100644 --- a/bridges/T411Bridge.php +++ b/bridges/T411Bridge.php @@ -7,7 +7,7 @@ class T411Bridge extends BridgeAbstract { $this->name = "T411"; $this->uri = "https://t411.in/"; $this->description = "Returns the 5 newest torrents with specified search terms
Use url part after '?' mark when using their search engine"; - $this->update = "2015-12-17"; + $this->update = "2016-02-06"; $this->parameters[] = '[ @@ -46,10 +46,10 @@ class T411Bridge extends BridgeAbstract { foreach($results->find('tr') as $element) { //Limit total amount of requests - if ($limit < 5) { + if ($limit < 10) { //Requests are rate-limited - sleep(1); //So we need to wait + usleep(500000); //So we need to wait (500ms) //Retrieve data from RSS entry $item_uri = 'http://'.ExtractFromDelimiters($element->outertext, 'find('img') as $img) { - if (strpos($img->src, 'dreamprez') === false) { + if (strpos($img->src, 'prez') === false) { $item_image = $img->src; break; } From 4ccd1b84e50eea96e7bf07326389d69660b1593b Mon Sep 17 00:00:00 2001 From: ORelio Date: Sat, 13 Feb 2016 14:03:28 +0100 Subject: [PATCH 6/6] [AnimeUltime] Simple tab/space fix in source file Don't have anything against tabs but let's keep concistent at least on a per-file scope - this simlply replaces tabs with spaces as this file is using spaces for code indentation. --- bridges/AnimeUltimeBridge.php | 67 ++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/bridges/AnimeUltimeBridge.php b/bridges/AnimeUltimeBridge.php index 68e199e7..2f832d70 100644 --- a/bridges/AnimeUltimeBridge.php +++ b/bridges/AnimeUltimeBridge.php @@ -3,44 +3,45 @@ class AnimeUltimeBridge extends BridgeAbstract { private $filter = 'Releases'; - public function loadMetadatas() { + public function loadMetadatas() { - $this->maintainer = "ORelio"; - $this->name = "Anime-Ultime"; - $this->uri = "http://www.anime-ultime.net/"; - $this->description = "Returns the 10 newest releases posted on Anime-Ultime"; - $this->update = "2015-10-30"; + $this->maintainer = 'ORelio'; + $this->name = 'Anime-Ultime'; + $this->uri = 'http://www.anime-ultime.net/'; + $this->description = 'Returns the 10 newest releases posted on Anime-Ultime'; + $this->update = '2015-10-30'; - $this->parameters[] = - '[ - { - "name" : "Type", - "type" : "list", - "identifier" : "type", - "values" : [ - { - "name" : "everything", - "value" : "" + $this->parameters[] = + '[ + { + "name" : "Type", + "type" : "list", + "identifier" : "type", + "values" : + [ + { + "name" : "Everything", + "value" : "" - }, - { - "name" : "Anime", - "value" : "A" + }, + { + "name" : "Anime", + "value" : "A" - }, - { - "name" : "Drama", - "value" : "D" - }, - { - "name" : "Tokusatsu", - "value" : "T" + }, + { + "name" : "Drama", + "value" : "D" + }, + { + "name" : "Tokusatsu", + "value" : "T" - } - ] - } - ]'; - } + } + ] + } + ]'; + } public function collectData(array $param) {