Compare commits

...

12 Commits

Author SHA1 Message Date
Knah Tsaeb 95731eb812 Merge remote-tracking branch 'origin/master' into kt_bridge 2019-05-10 16:05:36 +02:00
Lyra 2cd310c025 Bump version to 2019-05-08 2019-05-08 22:36:22 +02:00
sysadminstory b764204c3a [YoutubeBridge] Playlist bug fix (#1117)
This commit allow the bridge to parse an infinite number of items of a
Youtube playlist.

It should fix #647 !
2019-05-08 22:17:48 +02:00
Tobias Alexander Franke a9e2574016 [ArtStationBridge] Added new bridge (#1122)
* [ArtStationBridge] Added new bridge
2019-05-08 22:14:53 +02:00
pofilo e3f6e1c6db [DELETE] Deletion Google Plus bridge (#1124) 2019-05-08 22:11:50 +02:00
Lyra 8150a73922 [CourrierInternationalBridge] Use newer https-based URL 2019-05-08 22:09:49 +02:00
Lyra a2f3866383 [RoadAndTrackBridge] Major rewrite, due to the depreciation of their API 2019-05-08 21:57:59 +02:00
Obsidienne a3446ae77b [AO3Bridge] Add new bridge (#1123)
* [AO3Bridge] Add new bridge
2019-05-06 13:28:42 +02:00
Eugene Molotov 75359bc11b [core] Implemented MemcachedCache (#1000)
* [core] Implemented MemcachedCache
2019-05-03 11:56:07 +02:00
Roliga fe103974f5 [BadDragonBridge] Add new bridge (#1082)
* [BadDragonBridge] Add new bridge
2019-05-02 22:02:13 +02:00
fulmeek 33c16f8be5 [BakaUpdatesMangaReleasesBridge] Sanitize hash for more solid UIDs (#1113)
This should minimize occasional hiccups on regular updates.
2019-04-30 21:01:48 +02:00
fulmeek 21d3bf3b60 caches: Refactor the API (#1060)
- For consistency, functions should always return null on non-existing data.

- WordPressPluginUpdateBridge appears to have used its own cache instance in the past. Obviously not used anymore.

- Since $key can be anything, the cache implementation must ensure to assign the related data reliably; most commonly by serializing and hashing the key in an appropriate way.

- Even though the default path for storage is perfectly fine, some people may want to use a different location. This is an example how a cache implementation is responsible for its requirements.
2019-04-29 20:12:43 +02:00
20 changed files with 949 additions and 374 deletions

View File

@ -134,8 +134,11 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8
* [da2x](https://github.com/da2x)
* [Daiyousei](https://github.com/Daiyousei)
* [disk0x](https://github.com/disk0x)
* [DJCrashdummy](https://github.com/DJCrashdummy)
* [Djuuu](https://github.com/Djuuu)
* [DnAp](https://github.com/DnAp)
* [Draeli](https://github.com/Draeli)
* [Dreckiger-Dan](https://github.com/Dreckiger-Dan)
* [em92](https://github.com/em92)
* [eMerzh](https://github.com/eMerzh)
* [EtienneM](https://github.com/EtienneM)
@ -179,6 +182,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8
* [nel50n](https://github.com/nel50n)
* [niawag](https://github.com/niawag)
* [Nono-m0le](https://github.com/Nono-m0le)
* [ObsidianWitch](https://github.com/ObsidianWitch)
* [ORelio](https://github.com/ORelio)
* [PaulVayssiere](https://github.com/PaulVayssiere)
* [pellaeon](https://github.com/pellaeon)
@ -186,6 +190,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8
* [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf)
* [pitchoule](https://github.com/pitchoule)
* [pmaziere](https://github.com/pmaziere)
* [Pofilo](https://github.com/Pofilo)
* [prysme01](https://github.com/prysme01)
* [quentinus95](https://github.com/quentinus95)
* [regisenguehard](https://github.com/regisenguehard)
@ -200,6 +205,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8
* [sysadminstory](https://github.com/sysadminstory)
* [tameroski](https://github.com/tameroski)
* [teromene](https://github.com/teromene)
* [thefranke](https://github.com/thefranke)
* [TheRadialActive](https://github.com/TheRadialActive)
* [triatic](https://github.com/triatic)
* [WalterBarrett](https://github.com/WalterBarrett)

View File

@ -86,9 +86,9 @@ class DisplayAction extends ActionAbstract {
// Initialize cache
$cache = Cache::create(Configuration::getConfig('cache', 'type'));
$cache->setPath(PATH_CACHE);
$cache->setScope('');
$cache->purgeCache(86400); // 24 hours
$cache->setParameters($cache_params);
$cache->setKey($cache_params);
$items = array();
$infos = array();

121
bridges/AO3Bridge.php Normal file
View File

@ -0,0 +1,121 @@
<?php
class AO3Bridge extends BridgeAbstract {
const NAME = 'AO3';
const URI = 'https://archiveofourown.org/';
const CACHE_TIMEOUT = 1800;
const DESCRIPTION = 'Returns works or chapters from Archive of Our Own';
const MAINTAINER = 'Obsidienne';
const PARAMETERS = array(
'List' => array(
'url' => array(
'name' => 'url',
'required' => true,
// Example: F/F tag, complete works only
'exampleValue' => self::URI
. 'works?work_search[complete]=T&tag_id=F*s*F',
),
),
'Bookmarks' => array(
'user' => array(
'name' => 'user',
'required' => true,
// Example: Nyaaru's bookmarks
'exampleValue' => 'Nyaaru',
),
),
'Work' => array(
'id' => array(
'name' => 'id',
'required' => true,
// Example: latest chapters from A Better Past by LysSerris
'exampleValue' => '18181853',
),
)
);
// Feed for lists of works (e.g. recent works, search results, filtered tags,
// bookmarks, series, collections).
private function collectList($url) {
$html = getSimpleHTMLDOM($url)
or returnServerError('could not request AO3');
$html = defaultLinkTo($html, self::URI);
foreach($html->find('.index.group > li') as $element) {
$item = array();
$title = $element->find('div h4 a', 0);
if (!isset($title)) continue; // discard deleted works
$item['title'] = $title->plaintext;
$item['content'] = $element;
$item['uri'] = $title->href;
$strdate = $element->find('div p.datetime', 0)->plaintext;
$item['timestamp'] = strtotime($strdate);
$chapters = $element->find('dl dd.chapters', 0);
// bookmarked series and external works do not have a chapters count
$chapters = (isset($chapters) ? $chapters->plaintext : 0);
$item['uid'] = $item['uri'] . "/$strdate/$chapters";
$this->items[] = $item;
}
}
// Feed for recent chapters of a specific work.
private function collectWork($id) {
$url = self::URI . "/works/$id/navigate";
$html = getSimpleHTMLDOM($url)
or returnServerError('could not request AO3');
$html = defaultLinkTo($html, self::URI);
$this->title = $html->find('h2 a', 0)->plaintext;
foreach($html->find('ol.index.group > li') as $element) {
$item = array();
$item['title'] = $element->find('a', 0)->plaintext;
$item['content'] = $element;
$item['uri'] = $element->find('a', 0)->href;
$strdate = $element->find('span.datetime', 0)->plaintext;
$strdate = str_replace('(', '', $strdate);
$strdate = str_replace(')', '', $strdate);
$item['timestamp'] = strtotime($strdate);
$item['uid'] = $item['uri'] . "/$strdate";
$this->items[] = $item;
}
$this->items = array_reverse($this->items);
}
public function collectData() {
switch($this->queriedContext) {
case 'Bookmarks':
$user = $this->getInput('user');
$this->title = $user;
$url = self::URI
. '/users/' . $user
. '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
return $this->collectList($url);
case 'List': return $this->collectList(
$this->getInput('url')
);
case 'Work': return $this->collectWork(
$this->getInput('id')
);
}
}
public function getName() {
$name = parent::getName() . " $this->queriedContext";
if (isset($this->title)) $name .= " - $this->title";
return $name;
}
public function getIcon() {
return self::URI . '/favicon.ico';
}
}

View File

@ -0,0 +1,93 @@
<?php
class ArtStationBridge extends BridgeAbstract {
const NAME = 'ArtStation';
const URI = 'https://www.artstation.com';
const DESCRIPTION = 'Fetches the latest ten artworks from a search query on ArtStation.';
const MAINTAINER = 'thefranke';
const CACHE_TIMEOUT = 3600; // 1h
const PARAMETERS = array(
'Search Query' => array(
'q' => array(
'name' => 'Search term',
'required' => true
)
)
);
public function getIcon() {
return 'https://www.artstation.com/assets/favicon-58653022bc38c1905ac7aa1b10bffa6b.ico';
}
public function getName() {
return self::NAME . ': ' . $this->getInput('q');
}
private function fetchSearch($searchQuery) {
$data = '{"query":"' . $searchQuery . '","page":1,"per_page":50,"sorting":"date",';
$data .= '"pro_first":"1","filters":[],"additional_fields":[]}';
$header = array(
'Content-Type: application/json',
'Accept: application/json'
);
$opts = array(
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => $data,
CURLOPT_RETURNTRANSFER => true
);
$jsonSearchURL = self::URI . '/api/v2/search/projects.json';
$jsonSearchStr = getContents($jsonSearchURL, $header, $opts)
or returnServerError('Could not fetch JSON for search query.');
return json_decode($jsonSearchStr);
}
private function fetchProject($hashID) {
$jsonProjectURL = self::URI . '/projects/' . $hashID . '.json';
$jsonProjectStr = getContents($jsonProjectURL)
or returnServerError('Could not fetch JSON for project.');
return json_decode($jsonProjectStr);
}
public function collectData() {
$searchTerm = $this->getInput('q');
$jsonQuery = $this->fetchSearch($searchTerm);
foreach($jsonQuery->data as $media) {
// get detailed info about media item
$jsonProject = $this->fetchProject($media->hash_id);
// create item
$item = array();
$item['title'] = $media->title;
$item['uri'] = $media->url;
$item['timestamp'] = strtotime($jsonProject->published_at);
$item['author'] = $media->user->full_name;
$item['categories'] = implode(',', $jsonProject->tags);
$item['content'] = '<a href="'
. $media->url
. '"><img style="max-width: 100%" src="'
. $jsonProject->cover_url
. '"></a><p>'
. $jsonProject->description
. '</p>';
$numAssets = count($jsonProject->assets);
if ($numAssets > 1)
$item['content'] .= '<p><a href="'
. $media->url
. '">Project contains '
. ($numAssets - 1)
. ' more item(s).</a></p>';
$this->items[] = $item;
if (count($this->items) >= 10)
break;
}
}
}

435
bridges/BadDragonBridge.php Normal file
View File

@ -0,0 +1,435 @@
<?php
class BadDragonBridge extends BridgeAbstract {
const NAME = 'Bad Dragon Bridge';
const URI = 'https://bad-dragon.com/';
const CACHE_TIMEOUT = 300; // 5min
const DESCRIPTION = 'Returns sales or new clearance items';
const MAINTAINER = 'Roliga';
const PARAMETERS = array(
'Sales' => array(
),
'Clearance' => array(
'ready_made' => array(
'name' => 'Ready Made',
'type' => 'checkbox'
),
'flop' => array(
'name' => 'Flops',
'type' => 'checkbox'
),
'skus' => array(
'name' => 'Products',
'exampleValue' => 'chanceflared, crackers',
'title' => 'Comma separated list of product SKUs'
),
'onesize' => array(
'name' => 'One-Size',
'type' => 'checkbox'
),
'mini' => array(
'name' => 'Mini',
'type' => 'checkbox'
),
'small' => array(
'name' => 'Small',
'type' => 'checkbox'
),
'medium' => array(
'name' => 'Medium',
'type' => 'checkbox'
),
'large' => array(
'name' => 'Large',
'type' => 'checkbox'
),
'extralarge' => array(
'name' => 'Extra Large',
'type' => 'checkbox'
),
'category' => array(
'name' => 'Category',
'type' => 'list',
'values' => array(
'All' => 'all',
'Accessories' => 'accessories',
'Merchandise' => 'merchandise',
'Dildos' => 'insertable',
'Masturbators' => 'penetrable',
'Packers' => 'packer',
'Lil\' Squirts' => 'shooter',
'Lil\' Vibes' => 'vibrator',
'Wearables' => 'wearable'
),
'defaultValue' => 'all',
),
'soft' => array(
'name' => 'Soft Firmness',
'type' => 'checkbox'
),
'med_firm' => array(
'name' => 'Medium Firmness',
'type' => 'checkbox'
),
'firm' => array(
'name' => 'Firm',
'type' => 'checkbox'
),
'split' => array(
'name' => 'Split Firmness',
'type' => 'checkbox'
),
'maxprice' => array(
'name' => 'Max Price',
'type' => 'number',
'required' => true,
'defaultValue' => 300
),
'minprice' => array(
'name' => 'Min Price',
'type' => 'number',
'defaultValue' => 0
),
'cumtube' => array(
'name' => 'Cumtube',
'type' => 'checkbox'
),
'suctionCup' => array(
'name' => 'Suction Cup',
'type' => 'checkbox'
),
'noAccessories' => array(
'name' => 'No Accessories',
'type' => 'checkbox'
)
)
);
/*
* This sets index $strFrom (or $strTo if set) in $outArr to 'on' if
* $inArr[$param] contains $strFrom.
* It is used for translating BD's shop filter URLs into something we can use.
*
* For the query '?type[]=ready_made&type[]=flop' we would have an array like:
* Array (
* [type] => Array (
* [0] => ready_made
* [1] => flop
* )
* )
* which could be translated into:
* Array (
* [ready_made] => on
* [flop] => on
* )
* */
private function setParam($inArr, &$outArr, $param, $strFrom, $strTo = null) {
if(isset($inArr[$param]) && in_array($strFrom, $inArr[$param])) {
$outArr[($strTo ?: $strFrom)] = 'on';
}
}
public function detectParameters($url) {
$params = array();
// Sale
$regex = '/^(https?:\/\/)?bad-dragon\.com\/sales/';
if(preg_match($regex, $url, $matches) > 0) {
return $params;
}
// Clearance
$regex = '/^(https?:\/\/)?bad-dragon\.com\/shop\/clearance/';
if(preg_match($regex, $url, $matches) > 0) {
parse_str(parse_url($url, PHP_URL_QUERY), $urlParams);
$this->setParam($urlParams, $params, 'type', 'ready_made');
$this->setParam($urlParams, $params, 'type', 'flop');
if(isset($urlParams['skus'])) {
$skus = array();
foreach($urlParams['skus'] as $sku) {
is_string($sku) && $skus[] = $sku;
is_array($sku) && $skus[] = $sku[0];
}
$params['skus'] = implode(',', $skus);
}
$this->setParam($urlParams, $params, 'sizes', 'onesize');
$this->setParam($urlParams, $params, 'sizes', 'mini');
$this->setParam($urlParams, $params, 'sizes', 'small');
$this->setParam($urlParams, $params, 'sizes', 'medium');
$this->setParam($urlParams, $params, 'sizes', 'large');
$this->setParam($urlParams, $params, 'sizes', 'extralarge');
if(isset($urlParams['category'])) {
$params['category'] = strtolower($urlParams['category']);
} else{
$params['category'] = 'all';
}
$this->setParam($urlParams, $params, 'firmnessValues', 'soft');
$this->setParam($urlParams, $params, 'firmnessValues', 'medium', 'med_firm');
$this->setParam($urlParams, $params, 'firmnessValues', 'firm');
$this->setParam($urlParams, $params, 'firmnessValues', 'split');
if(isset($urlParams['price'])) {
isset($urlParams['price']['max'])
&& $params['maxprice'] = $urlParams['price']['max'];
isset($urlParams['price']['min'])
&& $params['minprice'] = $urlParams['price']['min'];
}
isset($urlParams['cumtube'])
&& $urlParams['cumtube'] === '1'
&& $params['cumtube'] = 'on';
isset($urlParams['suctionCup'])
&& $urlParams['suctionCup'] === '1'
&& $params['suctionCup'] = 'on';
isset($urlParams['noAccessories'])
&& $urlParams['noAccessories'] === '1'
&& $params['noAccessories'] = 'on';
return $params;
}
return null;
}
public function getName() {
switch($this->queriedContext) {
case 'Sales':
return 'Bad Dragon Sales';
case 'Clearance':
return 'Bad Dragon Clearance Search';
default:
return parent::getName();
}
}
public function getURI() {
switch($this->queriedContext) {
case 'Sales':
return self::URI . 'sales';
case 'Clearance':
return $this->inputToURL();
default:
return parent::getURI();
}
}
public function collectData() {
switch($this->queriedContext) {
case 'Sales':
$sales = json_decode(getContents(self::URI . 'api/sales'))
or returnServerError('Failed to query BD API');
foreach($sales as $sale) {
$item = array();
$item['title'] = $sale->title;
$item['timestamp'] = strtotime($sale->startDate);
$item['uri'] = $this->getURI() . '/' . $sale->slug;
$contentHTML = '<p><img src="' . $sale->image->url . '"></p>';
if(isset($sale->endDate)) {
$contentHTML .= '<p><b>This promotion ends on '
. gmdate('M j, Y \a\t g:i A T', strtotime($sale->endDate))
. '</b></p>';
} else {
$contentHTML .= '<p><b>This promotion never ends</b></p>';
}
$ul = false;
$content = json_decode($sale->content);
foreach($content->blocks as $block) {
switch($block->type) {
case 'header-one':
$contentHTML .= '<h1>' . $block->text . '</h1>';
break;
case 'header-two':
$contentHTML .= '<h2>' . $block->text . '</h2>';
break;
case 'header-three':
$contentHTML .= '<h3>' . $block->text . '</h3>';
break;
case 'unordered-list-item':
if(!$ul) {
$contentHTML .= '<ul>';
$ul = true;
}
$contentHTML .= '<li>' . $block->text . '</li>';
break;
default:
if($ul) {
$contentHTML .= '</ul>';
$ul = false;
}
$contentHTML .= '<p>' . $block->text . '</p>';
break;
}
}
$item['content'] = $contentHTML;
$this->items[] = $item;
}
break;
case 'Clearance':
$toyData = json_decode(getContents($this->inputToURL(true)))
or returnServerError('Failed to query BD API');
$productList = json_decode(getContents(self::URI
. 'api/inventory-toy/product-list'))
or returnServerError('Failed to query BD API');
foreach($toyData->toys as $toy) {
$item = array();
$item['uri'] = $this->getURI()
. '#'
. $toy->id;
$item['timestamp'] = strtotime($toy->created);
foreach($productList as $product) {
if($product->sku == $toy->sku) {
$item['title'] = $product->name;
break;
}
}
// images
$content = '<p>';
foreach($toy->images as $image) {
$content .= '<a href="'
. $image->fullFilename
. '"><img src="'
. $image->thumbFilename
. '" /></a>';
}
// price
$content .= '</p><p><b>Price:</b> $'
. $toy->price
// size
. '<br /><b>Size:</b> '
. $toy->size
// color
. '<br /><b>Color:</b> '
. $toy->color
// features
. '<br /><b>Features:</b> '
. ($toy->suction_cup ? 'Suction cup' : '')
. ($toy->suction_cup && $toy->cumtube ? ', ' : '')
. ($toy->cumtube ? 'Cumtube' : '')
. ($toy->suction_cup || $toy->cumtube ? '' : 'None');
// firmness
$firmnessTexts = array(
'2' => 'Extra soft',
'3' => 'Soft',
'5' => 'Medium',
'8' => 'Firm'
);
$firmnesses = explode('/', $toy->firmness);
if(count($firmnesses) === 2) {
$content .= '<br /><b>Firmness:</b> '
. $firmnessTexts[$firmnesses[0]]
. ', '
. $firmnessTexts[$firmnesses[1]];
} else{
$content .= '<br /><b>Firmness:</b> '
. $firmnessTexts[$firmnesses[0]];
}
// flop
if($toy->type === 'flop') {
$content .= '<br /><b>Flop reason:</b> '
. $toy->flop_reason;
}
$content .= '</p>';
$item['content'] = $content;
$enclosures = array();
foreach($toy->images as $image) {
$enclosures[] = $image->fullFilename;
}
$item['enclosures'] = $enclosures;
$categories = array();
$categories[] = $toy->sku;
$categories[] = $toy->type;
$categories[] = $toy->size;
if($toy->cumtube) {
$categories[] = 'cumtube';
}
if($toy->suction_cup) {
$categories[] = 'suction_cup';
}
$item['categories'] = $categories;
$this->items[] = $item;
}
break;
}
}
private function inputToURL($api = false) {
$url = self::URI;
$url .= ($api ? 'api/inventory-toys?' : 'shop/clearance?');
// Default parameters
$url .= 'limit=60';
$url .= '&page=1';
$url .= '&sort[field]=created';
$url .= '&sort[direction]=desc';
// Product types
$url .= ($this->getInput('ready_made') ? '&type[]=ready_made' : '');
$url .= ($this->getInput('flop') ? '&type[]=flop' : '');
// Product names
foreach(array_filter(explode(',', $this->getInput('skus'))) as $sku) {
$url .= '&skus[]=' . urlencode(trim($sku));
}
// Size
$url .= ($this->getInput('onesize') ? '&sizes[]=onesize' : '');
$url .= ($this->getInput('mini') ? '&sizes[]=mini' : '');
$url .= ($this->getInput('small') ? '&sizes[]=small' : '');
$url .= ($this->getInput('medium') ? '&sizes[]=medium' : '');
$url .= ($this->getInput('large') ? '&sizes[]=large' : '');
$url .= ($this->getInput('extralarge') ? '&sizes[]=extralarge' : '');
// Category
$url .= ($this->getInput('category') ? '&category='
. urlencode($this->getInput('category')) : '');
// Firmness
if($api) {
$url .= ($this->getInput('soft') ? '&firmnessValues[]=3' : '');
$url .= ($this->getInput('med_firm') ? '&firmnessValues[]=5' : '');
$url .= ($this->getInput('firm') ? '&firmnessValues[]=8' : '');
if($this->getInput('split')) {
$url .= '&firmnessValues[]=3/5';
$url .= '&firmnessValues[]=3/8';
$url .= '&firmnessValues[]=8/3';
$url .= '&firmnessValues[]=5/8';
$url .= '&firmnessValues[]=8/5';
}
} else{
$url .= ($this->getInput('soft') ? '&firmnessValues[]=soft' : '');
$url .= ($this->getInput('med_firm') ? '&firmnessValues[]=medium' : '');
$url .= ($this->getInput('firm') ? '&firmnessValues[]=firm' : '');
$url .= ($this->getInput('split') ? '&firmnessValues[]=split' : '');
}
// Price
$url .= ($this->getInput('maxprice') ? '&price[max]='
. $this->getInput('maxprice') : '&price[max]=300');
$url .= ($this->getInput('minprice') ? '&price[min]='
. $this->getInput('minprice') : '&price[min]=0');
// Features
$url .= ($this->getInput('cumtube') ? '&cumtube=1' : '');
$url .= ($this->getInput('suctionCup') ? '&suctionCup=1' : '');
$url .= ($this->getInput('noAccessories') ? '&noAccessories=1' : '');
return $url;
}
}

View File

@ -68,7 +68,7 @@ class BakaUpdatesMangaReleasesBridge extends BridgeAbstract {
$item['title'] = implode(' ', $title);
$item['uri'] = $this->getURI();
$item['uid'] = hash('sha1', $item['title']);
$item['uid'] = $this->getSanitizedHash($item['title']);
$this->items[] = $item;
}
@ -89,8 +89,12 @@ class BakaUpdatesMangaReleasesBridge extends BridgeAbstract {
return parent::getName();
}
private function getSanitizedHash($string) {
return hash('sha1', preg_replace('/[^a-zA-Z0-9\-\.]/', '', ucwords(strtolower($string))));
}
private function filterText($text) {
return rtrim($text, '*');
return rtrim($text, '* ');
}
private function filterHTML($text) {

View File

@ -3,7 +3,7 @@ class CourrierInternationalBridge extends BridgeAbstract {
const MAINTAINER = 'teromene';
const NAME = 'Courrier International Bridge';
const URI = 'http://CourrierInternational.com/';
const URI = 'https://www.courrierinternational.com/';
const CACHE_TIMEOUT = 300; // 5 min
const DESCRIPTION = 'Courrier International bridge';

View File

@ -121,8 +121,8 @@ class ElloBridge extends BridgeAbstract {
private function getAPIKey() {
$cache = Cache::create(Configuration::getConfig('cache', 'type'));
$cache->setPath(PATH_CACHE);
$cache->setParameters(['key']);
$cache->setScope(get_called_class());
$cache->setKey(['key']);
$key = $cache->loadData();
if($key == null) {

View File

@ -1,208 +0,0 @@
<?php
class GooglePlusPostBridge extends BridgeAbstract{
private $title;
private $url;
const MAINTAINER = 'Grummfy, logmanoriginal';
const NAME = 'Google Plus Post Bridge';
const URI = 'https://plus.google.com';
const CACHE_TIMEOUT = 600; //10min
const DESCRIPTION = 'Returns user public post (without API).';
const PARAMETERS = array( array(
'username' => array(
'name' => 'username or Id',
'required' => true
),
'include_media' => array(
'name' => 'Include media',
'type' => 'checkbox',
'title' => 'Enable to include media in the feed content'
)
));
public function getIcon() {
return 'https://ssl.gstatic.com/images/branding/product/ico/google_plus_alldp.ico';
}
public function collectData(){
$username = $this->getInput('username');
// Usernames start with a + if it's not an ID
if(!is_numeric($username) && substr($username, 0, 1) !== '+') {
$username = '+' . $username;
}
$html = getSimpleHTMLDOM(static::URI . '/' . urlencode($username) . '/posts')
or returnServerError('No results for this query.');
$html = defaultLinkTo($html, static::URI);
$this->title = $html->find('meta[property=og:title]', 0)->getAttribute('content');
$this->url = $html->find('meta[property=og:url]', 0)->getAttribute('content');
foreach($html->find('div[jsname=WsjYwc]') as $post) {
$item = array();
$item['author'] = $post->find('div div div div a', 0)->innertext;
$item['uri'] = $post->find('div div div a', 1)->href;
$timestamp = $post->find('a.qXj2He span', 0);
if($timestamp) {
$item['timestamp'] = strtotime('+' . preg_replace(
'/[^0-9A-Za-z]/',
'',
$timestamp->getAttribute('aria-label')));
}
$message = $post->find('div[jsname=EjRJtf]', 0);
// Empty messages are not supported right now
if(!$message) {
continue;
}
$item['content'] = '<div style="float: left; padding: 0 10px 10px 0;"><a href="'
. $this->url
. '"><img align="top" alt="'
. $item['author']
. '" src="'
. $post->find('div img', 0)->src
. '" /></a></div><div>'
. trim(strip_tags($message, '<a><p><div><img>'))
. '</div>';
// Make title at least 50 characters long, but don't add '...' if it is shorter!
if(strlen($message->plaintext) > 50) {
$end = strpos($message->plaintext, ' ', 50) ?: strlen($message->plaintext);
} else {
$end = strlen($message->plaintext);
}
if(strlen(substr($message->plaintext, 0, $end)) === strlen($message->plaintext)) {
$item['title'] = $message->plaintext;
} else {
$item['title'] = substr($message->plaintext, 0, $end) . '...';
}
$media = $post->find('[jsname="MTOxpb"]', 0);
if($media) {
$item['enclosures'] = array();
foreach($media->find('img') as $img) {
$item['enclosures'][] = $this->fixImage($img)->src;
}
if($this->getInput('include_media') === true && count($item['enclosures'] > 0)) {
$item['content'] .= '<div style="clear: both;"><a href="'
. $item['enclosures'][0]
. '"><img src="'
. $item['enclosures'][0]
. '" /></a></div>';
}
}
// Add custom parameters (only useful for JSON or Plaintext)
$item['fullname'] = $item['author'];
$item['avatar'] = $post->find('div img', 0)->src;
$item['id'] = $post->find('div div div', 0)->getAttribute('id');
$item['content_simple'] = $message->plaintext;
$this->items[] = $item;
}
}
public function getName(){
return $this->title ?: 'Google Plus Post Bridge';
}
public function getURI(){
return $this->url ?: parent::getURI();
}
private function fixImage($img) {
// There are certain images like .gif which link to a static picture and
// get replaced dynamically via JS in the browser. If we want the "real"
// image we need to account for that.
$urlparts = parse_url($img->src);
if(array_key_exists('host', $urlparts)) {
// For some reason some URIs don't contain the scheme, assume https
if(!array_key_exists('scheme', $urlparts)) {
$urlparts['scheme'] = 'https';
}
$pathelements = explode('/', $urlparts['path']);
switch($urlparts['host']) {
case 'lh3.googleusercontent.com':
if(pathinfo(end($pathelements), PATHINFO_EXTENSION)) {
// The second to last element of the path specifies the
// image format. The URL is still valid if we remove it.
unset($pathelements[count($pathelements) - 2]);
} elseif(strrpos(end($pathelements), '=') !== false) {
// Some images go throug a proxy. For those images they
// add size information after an equal sign.
// Example: '=w530-h298-n'. Again this can safely be
// removed to get the original image.
$pathelements[count($pathelements) - 1] = substr(
end($pathelements),
0,
strrpos(end($pathelements), '=')
);
}
break;
}
$urlparts['path'] = implode('/', $pathelements);
}
$img->src = $this->build_url($urlparts);
return $img;
}
/**
* From: https://gist.github.com/Ellrion/f51ba0d40ae1d62eeae44fd1adf7b704
* slightly adjusted to work with PHP < 7.0
* @param array $parts
* @return string
*/
private function build_url(array $parts)
{
$scheme = isset($parts['scheme']) ? ($parts['scheme'] . '://') : '';
$host = isset($parts['host']) ? $parts['host'] : '';
$port = isset($parts['port']) ? (':' . $parts['port']) : '';
$user = isset($parts['user']) ? $parts['user'] : '';
$pass = isset($parts['pass']) ? (':' . $parts['pass']) : '';
$pass = ($user || $pass) ? ($pass . '@') : '';
$path = isset($parts['path']) ? $parts['path'] : '';
$query = isset($parts['query']) ? ('?' . $parts['query']) : '';
$fragment = isset($parts['fragment']) ? ('#' . $parts['fragment']) : '';
return implode('', [$scheme, $user, $pass, $host, $port, $path, $query, $fragment]);
}
}

View File

@ -6,91 +6,60 @@ class RoadAndTrackBridge extends BridgeAbstract {
const CACHE_TIMEOUT = 86400; // 24h
const DESCRIPTION = 'Returns the latest news from Road & Track.';
const PARAMETERS = array(
array(
'new-cars' => array(
'name' => 'New Cars',
'type' => 'checkbox',
'exampleValue' => 'checked',
'title' => 'Activate to load New Cars articles'
),
'motorsports' => array(
'name' => 'Motorsports',
'type' => 'checkbox',
'exampleValue' => 'checked',
'title' => 'Activate to load Motorsports articles'
),
'car-culture' => array(
'name' => 'Car Culture',
'type' => 'checkbox',
'exampleValue' => 'checked',
'title' => 'Activate to load Car Culture articles'
),
'car-shows' => array(
'name' => 'Car shows',
'type' => 'checkbox',
'exampleValue' => 'checked',
'title' => 'Activate to load Car shows articles'
)
)
);
const API_TOKEN = '2e18e904-d9cd-4911-b30c-1817b1e0b04b';
const SIG_URL = 'https://cloud.mazdigital.com/feeds/production/comboapp/204/api/v3/';
const GSIG_URL = 'https://dashboard.mazsystems.com/services/cf_access?app_id=204&app_type=comboapp&api_token=';
public function collectData() {
$signVal = json_decode(getContents(self::GSIG_URL . self::API_TOKEN));
$signVal = $signVal->signature;
$page = getSimpleHTMLDOM(self::URI);
$newsElements = array();
if($this->getInput('new-cars')) {
$newsElements = array_merge($newsElements,
json_decode(getContents(self::SIG_URL . '7591/item_feed' . $signVal))
);
}
if($this->getInput('motorsports')) {
$newsElements = array_merge($newsElements,
json_decode(getContents(self::SIG_URL . '7590/item_feed' . $signVal))
);
}
if($this->getInput('car-culture')) {
$newsElements = array_merge($newsElements,
json_decode(getContents(self::SIG_URL . '7588/item_feed' . $signVal))
);
}
if($this->getInput('car-shows')) {
$newsElements = array_merge($newsElements,
json_decode(getContents(self::SIG_URL . '7589/item_feed' . $signVal))
);
}
usort($newsElements, function($a, $b) {
return $b->published - $a->published;
});
//Process the first element
$firstArticleLink = $page->find('.custom-promo-title', 0)->href;
$this->items[] = $this->fetchArticle($firstArticleLink);
$limit = 19;
foreach($newsElements as $element) {
$item = array();
$item['uri'] = $element->sourceUrl;
$item['timestamp'] = $element->published;
$item['enclosures'] = array($element->cover->url);
$item['title'] = $element->title;
$item['content'] = $this->getArticleContent($element);
$this->items[] = $item;
if($limit > 0) {
$limit--;
} else {
break;
}
foreach($page->find('.full-item-title') as $article) {
$this->items[] = $this->fetchArticle($article->href);
$limit -= 1;
if($limit == 0) break;
}
}
private function fixImages($content) {
$enclosures = [];
foreach($content->find('img') as $image) {
$image->src = explode('?', $image->getAttribute('data-src'))[0];
$enclosures[] = $image->src;
}
foreach($content->find('.embed-image-wrap, .content-lede-image-wrap') as $imgContainer) {
$imgContainer->style = '';
}
return $enclosures;
}
private function fetchArticle($articleLink) {
$articleLink = self::URI . $articleLink;
$article = getSimpleHTMLDOM($articleLink);
$item = array();
$item['title'] = $article->find('.content-hed', 0)->innertext;
$item['author'] = $article->find('.byline-name', 0)->innertext;
$item['timestamp'] = strtotime($article->find('.content-info-date', 0)->getAttribute('datetime'));
$content = $article->find('.content-container', 0);
if($content->find('.content-rail', 0) !== null)
$content->find('.content-rail', 0)->innertext = '';
$enclosures = $this->fixImages($content);
$item['enclosures'] = $enclosures;
$item['content'] = $content;
return $item;
}
private function getArticleContent($article) {
return getContents($article->contentUrl);

View File

@ -71,16 +71,4 @@ class WordPressPluginUpdateBridge extends BridgeAbstract {
return parent::getName();
}
private function getCachedDate($url){
Debug::log('getting pubdate from url ' . $url . '');
// Initialize cache
$cache = Cache::create(Configuration::getConfig('cache', 'type'));
$cache->setPath(PATH_CACHE . 'pages/');
$params = [$url];
$cache->setParameters($params);
// Get cachefile timestamp
$time = $cache->getTime();
return ($time !== false ? $time : time());
}
}

View File

@ -229,7 +229,7 @@ class YoutubeBridge extends BridgeAbstract {
$url_listing = self::URI . 'playlist?list=' . urlencode($this->request);
$html = $this->ytGetSimpleHTMLDOM($url_listing)
or returnServerError("Could not request YouTube. Tried:\n - $url_listing");
$item_count = $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a', false);
$item_count = $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a', true);
if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) {
$this->ytBridgeParseXmlFeed($xml);
} else {

View File

@ -3,20 +3,21 @@
* Cache with file system
*/
class FileCache implements CacheInterface {
protected $path;
protected $param;
protected $key;
public function loadData(){
if(file_exists($this->getCacheFile())) {
return unserialize(file_get_contents($this->getCacheFile()));
}
return null;
}
public function saveData($datas){
public function saveData($data){
// Notice: We use plain serialize() here to reduce memory footprint on
// large input data.
$writeStream = file_put_contents($this->getCacheFile(), serialize($datas));
$writeStream = file_put_contents($this->getCacheFile(), serialize($data));
if($writeStream === false) {
throw new \Exception('Cannot write the cache... Do you have the right permissions ?');
@ -29,13 +30,14 @@ class FileCache implements CacheInterface {
$cacheFile = $this->getCacheFile();
clearstatcache(false, $cacheFile);
if(file_exists($cacheFile)) {
return filemtime($cacheFile);
$time = filemtime($cacheFile);
return ($time !== false) ? $time : null;
}
return false;
return null;
}
public function purgeCache($duration){
public function purgeCache($seconds){
$cachePath = $this->getPath();
if(file_exists($cachePath)) {
$cacheIterator = new RecursiveIteratorIterator(
@ -47,7 +49,7 @@ class FileCache implements CacheInterface {
if(in_array($cacheFile->getBasename(), array('.', '..', '.gitkeep')))
continue;
elseif($cacheFile->isFile()) {
if(filemtime($cacheFile->getPathname()) < time() - $duration)
if(filemtime($cacheFile->getPathname()) < time() - $seconds)
unlink($cacheFile->getPathname());
}
}
@ -55,34 +57,34 @@ class FileCache implements CacheInterface {
}
/**
* Set cache path
* Set scope
* @return self
*/
public function setPath($path){
if(is_null($path) || !is_string($path)) {
throw new \Exception('The given path is invalid!');
public function setScope($scope){
if(is_null($scope) || !is_string($scope)) {
throw new \Exception('The given scope is invalid!');
}
$this->path = $path;
// Make sure path ends with '/' or '\'
$lastchar = substr($this->path, -1, 1);
if($lastchar !== '/' && $lastchar !== '\\')
$this->path .= '/';
if(!is_dir($this->path))
mkdir($this->path, 0755, true);
$this->path = PATH_CACHE . trim($scope, " \t\n\r\0\x0B\\\/") . '/';
return $this;
}
/**
* Set HTTP GET parameters
* Set key
* @return self
*/
public function setParameters(array $param){
$this->param = array_map('strtolower', $param);
public function setKey($key){
if (!empty($key) && is_array($key)) {
$key = array_map('strtolower', $key);
}
$key = json_encode($key);
if (!is_string($key)) {
throw new \Exception('The given key is invalid!');
}
$this->key = $key;
return $this;
}
@ -90,9 +92,15 @@ class FileCache implements CacheInterface {
* Return cache path (and create if not exist)
* @return string Cache path
*/
protected function getPath(){
private function getPath(){
if(is_null($this->path)) {
throw new \Exception('Call "setPath" first!');
throw new \Exception('Call "setScope" first!');
}
if(!is_dir($this->path)) {
if (mkdir($this->path, 0755, true) !== true) {
throw new \Exception('Unable to create ' . $this->path);
}
}
return $this->path;
@ -102,7 +110,7 @@ class FileCache implements CacheInterface {
* Get the file name use for cache store
* @return string Path to the file cache
*/
protected function getCacheFile(){
private function getCacheFile(){
return $this->getPath() . $this->getCacheName();
}
@ -110,13 +118,11 @@ class FileCache implements CacheInterface {
* Determines file name for store the cache
* return string
*/
protected function getCacheName(){
if(is_null($this->param)) {
throw new \Exception('Call "setParameters" first!');
private function getCacheName(){
if(is_null($this->key)) {
throw new \Exception('Call "setKey" first!');
}
// Change character when making incompatible changes to prevent loading
// errors due to incompatible file contents \|/
return hash('md5', http_build_query($this->param) . 'A') . '.cache';
return hash('md5', $this->key) . '.cache';
}
}

115
caches/MemcachedCache.php Normal file
View File

@ -0,0 +1,115 @@
<?php
class MemcachedCache implements CacheInterface {
private $scope;
private $key;
private $conn;
private $expiration = 0;
private $time = false;
private $data = null;
public function __construct() {
if (!extension_loaded('memcached')) {
returnServerError('"memcached" extension not loaded. Please check "php.ini"');
}
$host = Configuration::getConfig(get_called_class(), 'host');
$port = Configuration::getConfig(get_called_class(), 'port');
if (empty($host) && empty($port)) {
returnServerError('Configuration for ' . get_called_class() . ' missing. Please check your config.ini.php');
} else if (empty($host)) {
returnServerError('"host" param is not set for ' . get_called_class() . '. Please check your config.ini.php');
} else if (empty($port)) {
returnServerError('"port" param is not set for ' . get_called_class() . '. Please check your config.ini.php');
} else if (!ctype_digit($port)) {
returnServerError('"port" param is invalid for ' . get_called_class() . '. Please check your config.ini.php');
}
$port = intval($port);
if ($port < 1 || $port > 65535) {
returnServerError('"port" param is invalid for ' . get_called_class() . '. Please check your config.ini.php');
}
$conn = new Memcached();
$conn->addServer($host, $port) or returnServerError('Could not connect to memcached server');
$this->conn = $conn;
}
public function loadData(){
if ($this->data) return $this->data;
$result = $this->conn->get($this->getCacheKey());
if ($result === false) {
return false;
}
$this->time = $result['time'];
$this->data = $result['data'];
return $result['data'];
}
public function saveData($datas){
$time = time();
$object_to_save = array(
'data' => $datas,
'time' => $time,
);
$result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration);
if($result === false) {
returnServerError('Cannot write the cache to memcached server');
}
$this->time = $time;
return $this;
}
public function getTime(){
if ($this->time === false) {
$this->loadData();
}
return $this->time;
}
public function purgeCache($duration){
// Note: does not purges cache right now
// Just sets cache expiration and leave cache purging for memcached itself
$this->expiration = $duration;
}
/**
* Set scope
* @return self
*/
public function setScope($scope){
$this->scope = $scope;
return $this;
}
/**
* Set key
* @return self
*/
public function setKey($key){
if (!empty($key) && is_array($key)) {
$key = array_map('strtolower', $key);
}
$key = json_encode($key);
if (!is_string($key)) {
throw new \Exception('The given key is invalid!');
}
$this->key = $key;
return $this;
}
private function getCacheKey(){
if(is_null($this->key)) {
returnServerError('Call "setKey" first!');
}
return 'rss_bridge_cache_' . hash('md5', $this->scope . $this->key . 'A');
}
}

View File

@ -3,16 +3,25 @@
* Cache based on SQLite 3 <https://www.sqlite.org>
*/
class SQLiteCache implements CacheInterface {
protected $path;
protected $param;
protected $scope;
protected $key;
private $db = null;
public function __construct() {
if (!extension_loaded('sqlite3'))
if (!extension_loaded('sqlite3')) {
die('"sqlite3" extension not loaded. Please check "php.ini"');
}
$file = PATH_CACHE . 'cache.sqlite';
$file = Configuration::getConfig(get_called_class(), 'file');
if (empty($file)) {
die('Configuration for ' . get_called_class() . ' missing. Please check your config.ini.php');
}
if (dirname($file) == '.') {
$file = PATH_CACHE . $file;
} elseif (!is_dir(dirname($file))) {
die('Invalid configuration for ' . get_called_class() . '. Please check your config.ini.php');
}
if (!is_file($file)) {
$this->db = new SQLite3($file);
@ -39,10 +48,10 @@ class SQLiteCache implements CacheInterface {
return null;
}
public function saveData($datas){
public function saveData($data){
$Qupdate = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)');
$Qupdate->bindValue(':key', $this->getCacheKey());
$Qupdate->bindValue(':value', serialize($datas));
$Qupdate->bindValue(':value', serialize($data));
$Qupdate->bindValue(':updated', time());
$Qupdate->execute();
@ -60,40 +69,53 @@ class SQLiteCache implements CacheInterface {
}
}
return false;
return null;
}
public function purgeCache($duration){
public function purgeCache($seconds){
$Qdelete = $this->db->prepare('DELETE FROM storage WHERE updated < :expired');
$Qdelete->bindValue(':expired', time() - $duration);
$Qdelete->bindValue(':expired', time() - $seconds);
$Qdelete->execute();
}
/**
* Set cache path
* Set scope
* @return self
*/
public function setPath($path){
$this->path = $path;
public function setScope($scope){
if(is_null($scope) || !is_string($scope)) {
throw new \Exception('The given scope is invalid!');
}
$this->scope = $scope;
return $this;
}
/**
* Set HTTP GET parameters
* Set key
* @return self
*/
public function setParameters(array $param){
$this->param = array_map('strtolower', $param);
public function setKey($key){
if (!empty($key) && is_array($key)) {
$key = array_map('strtolower', $key);
}
$key = json_encode($key);
if (!is_string($key)) {
throw new \Exception('The given key is invalid!');
}
$this->key = $key;
return $this;
}
////////////////////////////////////////////////////////////////////////////
protected function getCacheKey(){
if(is_null($this->param)) {
throw new \Exception('Call "setParameters" first!');
private function getCacheKey(){
if(is_null($this->key)) {
throw new \Exception('Call "setKey" first!');
}
return hash('sha1', $this->path . http_build_query($this->param), true);
return hash('sha1', $this->scope . $this->key, true);
}
}

View File

@ -52,3 +52,12 @@ username = ""
; The password for authentication. Insert this password when prompted for login.
; Use a strong password to prevent others from guessing your login!
password = ""
; --- Cache specific configuration ---------------------------------------------
[SQLiteCache]
file = "cache.sqlite"
[MemcachedCache]
host = "localhost"
port = 11211

View File

@ -37,7 +37,6 @@ $whitelist_default = array(
'DuckDuckGoBridge',
'FacebookBridge',
'FlickrBridge',
'GooglePlusPostBridge',
'GoogleSearchBridge',
'IdenticaBridge',
'InstagramBridge',

View File

@ -13,38 +13,54 @@
/**
* The cache interface
*
* @todo Add missing function to the interface
* @todo Explain parameters and return values in more detail
* @todo Return self more often (to allow call chaining)
*/
interface CacheInterface {
/**
* Set scope of the current cache
*
* If $scope is an empty string, the cache is set to a global context.
*
* @param string $scope The scope the data is related to
*/
public function setScope($scope);
/**
* Set key to assign the current data
*
* Since $key can be anything, the cache implementation must ensure to
* assign the related data reliably; most commonly by serializing and
* hashing the key in an appropriate way.
*
* @param array $key The key the data is related to
*/
public function setKey($key);
/**
* Loads data from cache
*
* @return mixed The cache data
* @return mixed The cached data or null
*/
public function loadData();
/**
* Stores data to the cache
*
* @param mixed $datas The data to store
* @param mixed $data The data to store
* @return self The cache object
*/
public function saveData($datas);
public function saveData($data);
/**
* Returns the timestamp for the curent cache file
* Returns the timestamp for the curent cache data
*
* @return int Timestamp
* @return int Timestamp or null
*/
public function getTime();
/**
* Removes any data that is older than the specified duration from cache
* Removes any data that is older than the specified age from cache
*
* @param int $duration The cache duration in seconds
* @param int $seconds The cache age in seconds
*/
public function purgeCache($duration);
public function purgeCache($seconds);
}

View File

@ -28,7 +28,7 @@ final class Configuration {
*
* @todo Replace this property by a constant.
*/
public static $VERSION = 'dev.2019-03-17';
public static $VERSION = 'dev.2019-05-08';
/**
* Holds the configuration data.

View File

@ -46,11 +46,11 @@ function getContents($url, $header = array(), $opts = array()){
// Initialize cache
$cache = Cache::create(Configuration::getConfig('cache', 'type'));
$cache->setPath(PATH_CACHE . 'server/');
$cache->setScope('server');
$cache->purgeCache(86400); // 24 hours (forced)
$params = [$url];
$cache->setParameters($params);
$cache->setKey($params);
// Use file_get_contents if in CLI mode with no root certificates defined
if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) {
@ -271,11 +271,11 @@ function getSimpleHTMLDOMCached($url,
// Initialize cache
$cache = Cache::create(Configuration::getConfig('cache', 'type'));
$cache->setPath(PATH_CACHE . 'pages/');
$cache->setScope('pages');
$cache->purgeCache(86400); // 24 hours (forced)
$params = [$url];
$cache->setParameters($params);
$cache->setKey($params);
// Determine if cached file is within duration
$time = $cache->getTime();