LesLibres.org/Rss-Bridge
2
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'Caching' of https://github.com/logmanoriginal/rss-bridge

Browse source
This commit is contained in:
logmanoriginal 2016-08-28 20:10:42 +02:00
commit 7e2129fa2a
13 changed files with 39 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bridges/CpasbienBridge.php
View file

@ -33,7 +33,7 @@ class CpasbienBridge extends HttpCachingBridgeAbstract{
if ($episode->getAttribute('class')=='ligne0' || $episode->getAttribute('class')=='ligne1')
{
$htmlepisode=str_get_html($this->get_cached($episode->find('a', 0)->getAttribute('href')));
$htmlepisode=$this->get_cached($episode->find('a', 0)->getAttribute('href'));
$item = array();
$item['author'] = $episode->find('a', 0)->text();

2
bridges/FreenewsBridge.php
View file

@ -25,7 +25,7 @@ class FreenewsBridge extends RssExpander {
}
// now load that uri from cache
$this->debugMessage("now loading page ".$item['uri']);
$articlePage = str_get_html($this->get_cached($item['uri']));
$articlePage = $this->get_cached($item['uri']);
$content = $articlePage->find('.post-container', 0);
$item['content'] = $content->innertext;

2
bridges/GawkerBridge.php
View file

@ -45,7 +45,7 @@ class GawkerBridge extends RssExpander{
try {
// now load that uri from cache
$this->debugMessage("loading page ".$item['uri']);
$articlePage = str_get_html($this->get_cached($item['uri']));
$articlePage = $this->get_cached($item['uri']);
if(is_object($articlePage)) {
$content = $articlePage->find('.post-content', 0);
HTMLSanitizer::defaultImageSrcTo($content, $this->getURI());

7
bridges/JapanExpoBridge.php
View file

@ -1,5 +1,5 @@
<?php
class JapanExpoBridge extends BridgeAbstract{
class JapanExpoBridge extends HttpCachingBridgeAbstract {
public function loadMetadatas() {
$this->maintainer = 'Ginko';
@ -64,7 +64,10 @@ class JapanExpoBridge extends BridgeAbstract{
if ($fullcontent) {
if ($count < 5) {
$article_html = $this->getSimpleHTMLDOM($url) or $this->returnServerError('Could not request JapanExpo: '.$url);
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$article_html = $this->get_cached($url) or $this->returnServerError('Could not request JapanExpo: '.$url);
$header = $article_html->find('header.pageHeadBox', 0);
$timestamp = strtotime($header->find('time', 0)->datetime);
$title_html = $header->find('div.section', 0)->next_sibling();

7
bridges/KununuBridge.php
View file

@ -1,5 +1,5 @@
<?php
class KununuBridge extends BridgeAbstract{
class KununuBridge extends HttpCachingBridgeAbstract {
public function loadMetadatas(){
$this->maintainer = "logmanoriginal";
$this->name = "Kununu Bridge"; /* This will be replaced later! */
@ -248,7 +248,10 @@ class KununuBridge extends BridgeAbstract{
*/
private function extract_full_description($uri){
// Load full article
$html = $this->getSimpleHTMLDOM($uri);
if($this->get_cached_time($uri) <= strtotime('-24 hours'))
$this->remove_from_cache($uri);
$html = $this->get_cached($uri);
if($html === false)
$this->returnServerError('Could not load full description!');

2
bridges/Les400CulsBridge.php
View file

@ -29,7 +29,7 @@ class Les400CulsBridge extends RssExpander{
}
// now load that uri from cache
$this->debugMessage("now loading page ".$item['uri']);
// $articlePage = str_get_html($this->get_cached($item['uri']));
// $articlePage = $this->get_cached($item['uri']);
// $content = $articlePage->find('.post-container', 0);
$item['content'] = (string) $newsItem->description;

7
bridges/LichessBridge.php
View file

@ -1,6 +1,6 @@
<?php
class LichessBridge extends BridgeAbstract
class LichessBridge extends HttpCachingBridgeAbstract
{
public function loadMetadatas()
{
@ -36,7 +36,10 @@ class LichessBridge extends BridgeAbstract
private function retrieve_lichess_post($blog_post_uri)
{
$blog_post_html = $this->getSimpleHTMLDOM($blog_post_uri);
if($this->get_cached_time($blog_post_uri) <= strtotime('-24 hours'))
$this->remove_from_cache($blog_post_uriuri);
$blog_post_html = $this->get_cached($blog_post_uri);
$blog_post_div = $blog_post_html->find('#lichess_blog', 0);
$post_chapo = $blog_post_div->find('.shortlede', 0)->innertext;

7
bridges/NumeramaBridge.php
View file

@ -1,5 +1,5 @@
<?php
class NumeramaBridge extends BridgeAbstract{
class NumeramaBridge extends HttpCachingBridgeAbstract {
public function loadMetadatas() {
@ -31,7 +31,10 @@ class NumeramaBridge extends BridgeAbstract{
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$article_url = NumeramaStripCDATA($element->find('guid', 0)->plaintext);
$article_html = $this->getSimpleHTMLDOM($article_url) or $this->returnServerError('Could not request Numerama: '.$article_url);
if($this->get_cached_time($article_url) <= strtotime('-24 hours'))
$this->remove_from_cache($article_url);
$article_html = $this->get_cached($article_url) or $this->returnServerError('Could not request Numerama: '.$article_url);
$contents = $article_html->find('section[class=related-article]', 0)->innertext = ''; // remove related articles block
$contents = '<img alt="" style="max-width:300px;" src="'.$article_html->find('meta[property=og:image]', 0)->getAttribute('content').'">'; // add post picture
$contents = $contents.$article_html->find('article[class=post-content]', 0)->innertext; // extract the post

2
bridges/TheOatMealBridge.php
View file

@ -43,7 +43,7 @@ class TheOatmealBridge extends RssExpander{
$item['uri']=(string) $newsItem->attributes($namespaces['rdf'])->about;
// now load that uri from cache
$this->debugMessage("now loading page ".$item['uri']);
$articlePage = str_get_html($this->get_cached($item['uri']));
$articlePage = $this->get_cached($item['uri']);
$content = $articlePage->find('#comic', 0);
if($content==null) {

7
bridges/WikipediaBridge.php
View file

@ -3,7 +3,7 @@
define('WIKIPEDIA_SUBJECT_TFA', 0); // Today's featured article
define('WIKIPEDIA_SUBJECT_DYK', 1); // Did you know...
class WikipediaBridge extends BridgeAbstract{
class WikipediaBridge extends HttpCachingBridgeAbstract {
public function loadMetadatas(){
$this->maintainer = 'logmanoriginal';
$this->name = 'Wikipedia bridge for many languages';
@ -188,7 +188,10 @@ class WikipediaBridge extends BridgeAbstract{
* Loads the full article from a given URI
*/
private function LoadFullArticle($uri){
$content_html = $this->getSimpleHTMLDOM($uri);
if($this->get_cached_time($uri) <= strtotime('-24 hours'))
$this->remove_from_cache($uri);
$content_html = $this->get_cached($uri);
if(!$content_html)
$this->returnServerError('Could not load site: ' . $uri . '!');

7
bridges/WordPressBridge.php
View file

@ -1,7 +1,7 @@
<?php
define('WORDPRESS_TYPE_ATOM', 1); // Content is of type ATOM
define('WORDPRESS_TYPE_RSS', 2); // Content is of type RSS
class WordPressBridge extends BridgeAbstract {
class WordPressBridge extends HttpCachingBridgeAbstract {
private $url;
public $sitename; // Name of the site
@ -95,7 +95,10 @@ class WordPressBridge extends BridgeAbstract {
$item['timestamp'] = strtotime($article->find('updated', 0)->innertext);
}
$article_html = $this->getSimpleHTMLDOM($item['uri']);
if($this->get_cached_time($item['uri']) <= strtotime('-24 hours'))
$this->remove_from_cache($item['uri']);
$article_html = $this->get_cached($item['uri']);
// Attempt to find most common content div
if(!isset($item['content'])){

2
bridges/WorldOfTanksBridge.php
View file

@ -58,7 +58,7 @@ class WorldOfTanksBridge extends HttpCachingBridgeAbstract{
$item['uri'] = $this->uri.$infoLink->href;
// now load that uri from cache
$this->debugMessage("loading page ".$item['uri']);
$articlePage = str_get_html($this->get_cached($item['uri']));
$articlePage = $this->get_cached($item['uri']);
$content = $articlePage->find('.l-content', 0);
HTMLSanitizer::defaultImageSrcTo($content, $this->uri);
$item['title'] = $content->find('h1', 0)->innertext;

6
lib/Bridge.php
View file

@ -419,7 +419,7 @@ abstract class HttpCachingBridgeAbstract extends BridgeAbstract {
}
}
return $content;
return str_get_html($content);
}
public function get_cached_time($url){
@ -465,8 +465,8 @@ abstract class HttpCachingBridgeAbstract extends BridgeAbstract {
// TODO build this from the variable given to Cache
$cacheDir = __DIR__ . '/../cache/pages/';
$filepath = $this->buildCacheFilePath($url, $cacheDir);
$this->debugMessage('removing from cache \'' . $filepath . '\' WELL, NOT REALLY');
// unlink($filepath);
$this->debugMessage('removing from cache \'' . $filepath . '\'');
unlink($filepath);
}
}