diff --git a/README.md b/README.md index 7ed74d81..530f90fe 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,11 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 * [da2x](https://github.com/da2x) * [Daiyousei](https://github.com/Daiyousei) * [disk0x](https://github.com/disk0x) + * [DJCrashdummy](https://github.com/DJCrashdummy) * [Djuuu](https://github.com/Djuuu) + * [DnAp](https://github.com/DnAp) * [Draeli](https://github.com/Draeli) + * [Dreckiger-Dan](https://github.com/Dreckiger-Dan) * [em92](https://github.com/em92) * [eMerzh](https://github.com/eMerzh) * [EtienneM](https://github.com/EtienneM) @@ -179,6 +182,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 * [nel50n](https://github.com/nel50n) * [niawag](https://github.com/niawag) * [Nono-m0le](https://github.com/Nono-m0le) + * [ObsidianWitch](https://github.com/ObsidianWitch) * [ORelio](https://github.com/ORelio) * [PaulVayssiere](https://github.com/PaulVayssiere) * [pellaeon](https://github.com/pellaeon) @@ -186,6 +190,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 * [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf) * [pitchoule](https://github.com/pitchoule) * [pmaziere](https://github.com/pmaziere) + * [Pofilo](https://github.com/Pofilo) * [prysme01](https://github.com/prysme01) * [quentinus95](https://github.com/quentinus95) * [regisenguehard](https://github.com/regisenguehard) @@ -200,6 +205,7 @@ https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 * [sysadminstory](https://github.com/sysadminstory) * [tameroski](https://github.com/tameroski) * [teromene](https://github.com/teromene) + * [thefranke](https://github.com/thefranke) * [TheRadialActive](https://github.com/TheRadialActive) * [triatic](https://github.com/triatic) * [WalterBarrett](https://github.com/WalterBarrett) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index b223b757..a1b106f5 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -86,9 +86,9 @@ class DisplayAction extends ActionAbstract { // Initialize cache $cache = Cache::create(Configuration::getConfig('cache', 'type')); - $cache->setPath(PATH_CACHE); + $cache->setScope(''); $cache->purgeCache(86400); // 24 hours - $cache->setParameters($cache_params); + $cache->setKey($cache_params); $items = array(); $infos = array(); diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php new file mode 100644 index 00000000..9a3b5c8f --- /dev/null +++ b/bridges/AO3Bridge.php @@ -0,0 +1,121 @@ + array( + 'url' => array( + 'name' => 'url', + 'required' => true, + // Example: F/F tag, complete works only + 'exampleValue' => self::URI + . 'works?work_search[complete]=T&tag_id=F*s*F', + ), + ), + 'Bookmarks' => array( + 'user' => array( + 'name' => 'user', + 'required' => true, + // Example: Nyaaru's bookmarks + 'exampleValue' => 'Nyaaru', + ), + ), + 'Work' => array( + 'id' => array( + 'name' => 'id', + 'required' => true, + // Example: latest chapters from A Better Past by LysSerris + 'exampleValue' => '18181853', + ), + ) + ); + + // Feed for lists of works (e.g. recent works, search results, filtered tags, + // bookmarks, series, collections). + private function collectList($url) { + $html = getSimpleHTMLDOM($url) + or returnServerError('could not request AO3'); + $html = defaultLinkTo($html, self::URI); + + foreach($html->find('.index.group > li') as $element) { + $item = array(); + + $title = $element->find('div h4 a', 0); + if (!isset($title)) continue; // discard deleted works + $item['title'] = $title->plaintext; + $item['content'] = $element; + $item['uri'] = $title->href; + + $strdate = $element->find('div p.datetime', 0)->plaintext; + $item['timestamp'] = strtotime($strdate); + + $chapters = $element->find('dl dd.chapters', 0); + // bookmarked series and external works do not have a chapters count + $chapters = (isset($chapters) ? $chapters->plaintext : 0); + $item['uid'] = $item['uri'] . "/$strdate/$chapters"; + + $this->items[] = $item; + } + } + + // Feed for recent chapters of a specific work. + private function collectWork($id) { + $url = self::URI . "/works/$id/navigate"; + $html = getSimpleHTMLDOM($url) + or returnServerError('could not request AO3'); + $html = defaultLinkTo($html, self::URI); + + $this->title = $html->find('h2 a', 0)->plaintext; + + foreach($html->find('ol.index.group > li') as $element) { + $item = array(); + + $item['title'] = $element->find('a', 0)->plaintext; + $item['content'] = $element; + $item['uri'] = $element->find('a', 0)->href; + + $strdate = $element->find('span.datetime', 0)->plaintext; + $strdate = str_replace('(', '', $strdate); + $strdate = str_replace(')', '', $strdate); + $item['timestamp'] = strtotime($strdate); + + $item['uid'] = $item['uri'] . "/$strdate"; + + $this->items[] = $item; + } + + $this->items = array_reverse($this->items); + } + + public function collectData() { + switch($this->queriedContext) { + case 'Bookmarks': + $user = $this->getInput('user'); + $this->title = $user; + $url = self::URI + . '/users/' . $user + . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date'; + return $this->collectList($url); + case 'List': return $this->collectList( + $this->getInput('url') + ); + case 'Work': return $this->collectWork( + $this->getInput('id') + ); + } + } + + public function getName() { + $name = parent::getName() . " $this->queriedContext"; + if (isset($this->title)) $name .= " - $this->title"; + return $name; + } + + public function getIcon() { + return self::URI . '/favicon.ico'; + } +} diff --git a/bridges/ArtStationBridge.php b/bridges/ArtStationBridge.php new file mode 100644 index 00000000..9c12add5 --- /dev/null +++ b/bridges/ArtStationBridge.php @@ -0,0 +1,93 @@ + array( + 'q' => array( + 'name' => 'Search term', + 'required' => true + ) + ) + ); + + public function getIcon() { + return 'https://www.artstation.com/assets/favicon-58653022bc38c1905ac7aa1b10bffa6b.ico'; + } + + public function getName() { + return self::NAME . ': ' . $this->getInput('q'); + } + + private function fetchSearch($searchQuery) { + $data = '{"query":"' . $searchQuery . '","page":1,"per_page":50,"sorting":"date",'; + $data .= '"pro_first":"1","filters":[],"additional_fields":[]}'; + + $header = array( + 'Content-Type: application/json', + 'Accept: application/json' + ); + + $opts = array( + CURLOPT_POST => true, + CURLOPT_POSTFIELDS => $data, + CURLOPT_RETURNTRANSFER => true + ); + + $jsonSearchURL = self::URI . '/api/v2/search/projects.json'; + $jsonSearchStr = getContents($jsonSearchURL, $header, $opts) + or returnServerError('Could not fetch JSON for search query.'); + return json_decode($jsonSearchStr); + } + + private function fetchProject($hashID) { + $jsonProjectURL = self::URI . '/projects/' . $hashID . '.json'; + $jsonProjectStr = getContents($jsonProjectURL) + or returnServerError('Could not fetch JSON for project.'); + return json_decode($jsonProjectStr); + } + + public function collectData() { + $searchTerm = $this->getInput('q'); + $jsonQuery = $this->fetchSearch($searchTerm); + + foreach($jsonQuery->data as $media) { + // get detailed info about media item + $jsonProject = $this->fetchProject($media->hash_id); + + // create item + $item = array(); + $item['title'] = $media->title; + $item['uri'] = $media->url; + $item['timestamp'] = strtotime($jsonProject->published_at); + $item['author'] = $media->user->full_name; + $item['categories'] = implode(',', $jsonProject->tags); + + $item['content'] = '

' + . $jsonProject->description + . '

'; + + $numAssets = count($jsonProject->assets); + + if ($numAssets > 1) + $item['content'] .= '

Project contains ' + . ($numAssets - 1) + . ' more item(s).

'; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } +} diff --git a/bridges/BadDragonBridge.php b/bridges/BadDragonBridge.php new file mode 100644 index 00000000..d606c4e1 --- /dev/null +++ b/bridges/BadDragonBridge.php @@ -0,0 +1,435 @@ + array( + ), + 'Clearance' => array( + 'ready_made' => array( + 'name' => 'Ready Made', + 'type' => 'checkbox' + ), + 'flop' => array( + 'name' => 'Flops', + 'type' => 'checkbox' + ), + 'skus' => array( + 'name' => 'Products', + 'exampleValue' => 'chanceflared, crackers', + 'title' => 'Comma separated list of product SKUs' + ), + 'onesize' => array( + 'name' => 'One-Size', + 'type' => 'checkbox' + ), + 'mini' => array( + 'name' => 'Mini', + 'type' => 'checkbox' + ), + 'small' => array( + 'name' => 'Small', + 'type' => 'checkbox' + ), + 'medium' => array( + 'name' => 'Medium', + 'type' => 'checkbox' + ), + 'large' => array( + 'name' => 'Large', + 'type' => 'checkbox' + ), + 'extralarge' => array( + 'name' => 'Extra Large', + 'type' => 'checkbox' + ), + 'category' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All' => 'all', + 'Accessories' => 'accessories', + 'Merchandise' => 'merchandise', + 'Dildos' => 'insertable', + 'Masturbators' => 'penetrable', + 'Packers' => 'packer', + 'Lil\' Squirts' => 'shooter', + 'Lil\' Vibes' => 'vibrator', + 'Wearables' => 'wearable' + ), + 'defaultValue' => 'all', + ), + 'soft' => array( + 'name' => 'Soft Firmness', + 'type' => 'checkbox' + ), + 'med_firm' => array( + 'name' => 'Medium Firmness', + 'type' => 'checkbox' + ), + 'firm' => array( + 'name' => 'Firm', + 'type' => 'checkbox' + ), + 'split' => array( + 'name' => 'Split Firmness', + 'type' => 'checkbox' + ), + 'maxprice' => array( + 'name' => 'Max Price', + 'type' => 'number', + 'required' => true, + 'defaultValue' => 300 + ), + 'minprice' => array( + 'name' => 'Min Price', + 'type' => 'number', + 'defaultValue' => 0 + ), + 'cumtube' => array( + 'name' => 'Cumtube', + 'type' => 'checkbox' + ), + 'suctionCup' => array( + 'name' => 'Suction Cup', + 'type' => 'checkbox' + ), + 'noAccessories' => array( + 'name' => 'No Accessories', + 'type' => 'checkbox' + ) + ) + ); + + /* + * This sets index $strFrom (or $strTo if set) in $outArr to 'on' if + * $inArr[$param] contains $strFrom. + * It is used for translating BD's shop filter URLs into something we can use. + * + * For the query '?type[]=ready_made&type[]=flop' we would have an array like: + * Array ( + * [type] => Array ( + * [0] => ready_made + * [1] => flop + * ) + * ) + * which could be translated into: + * Array ( + * [ready_made] => on + * [flop] => on + * ) + * */ + private function setParam($inArr, &$outArr, $param, $strFrom, $strTo = null) { + if(isset($inArr[$param]) && in_array($strFrom, $inArr[$param])) { + $outArr[($strTo ?: $strFrom)] = 'on'; + } + } + + public function detectParameters($url) { + $params = array(); + + // Sale + $regex = '/^(https?:\/\/)?bad-dragon\.com\/sales/'; + if(preg_match($regex, $url, $matches) > 0) { + return $params; + } + + // Clearance + $regex = '/^(https?:\/\/)?bad-dragon\.com\/shop\/clearance/'; + if(preg_match($regex, $url, $matches) > 0) { + parse_str(parse_url($url, PHP_URL_QUERY), $urlParams); + + $this->setParam($urlParams, $params, 'type', 'ready_made'); + $this->setParam($urlParams, $params, 'type', 'flop'); + + if(isset($urlParams['skus'])) { + $skus = array(); + foreach($urlParams['skus'] as $sku) { + is_string($sku) && $skus[] = $sku; + is_array($sku) && $skus[] = $sku[0]; + } + $params['skus'] = implode(',', $skus); + } + + $this->setParam($urlParams, $params, 'sizes', 'onesize'); + $this->setParam($urlParams, $params, 'sizes', 'mini'); + $this->setParam($urlParams, $params, 'sizes', 'small'); + $this->setParam($urlParams, $params, 'sizes', 'medium'); + $this->setParam($urlParams, $params, 'sizes', 'large'); + $this->setParam($urlParams, $params, 'sizes', 'extralarge'); + + if(isset($urlParams['category'])) { + $params['category'] = strtolower($urlParams['category']); + } else{ + $params['category'] = 'all'; + } + + $this->setParam($urlParams, $params, 'firmnessValues', 'soft'); + $this->setParam($urlParams, $params, 'firmnessValues', 'medium', 'med_firm'); + $this->setParam($urlParams, $params, 'firmnessValues', 'firm'); + $this->setParam($urlParams, $params, 'firmnessValues', 'split'); + + if(isset($urlParams['price'])) { + isset($urlParams['price']['max']) + && $params['maxprice'] = $urlParams['price']['max']; + isset($urlParams['price']['min']) + && $params['minprice'] = $urlParams['price']['min']; + } + + isset($urlParams['cumtube']) + && $urlParams['cumtube'] === '1' + && $params['cumtube'] = 'on'; + isset($urlParams['suctionCup']) + && $urlParams['suctionCup'] === '1' + && $params['suctionCup'] = 'on'; + isset($urlParams['noAccessories']) + && $urlParams['noAccessories'] === '1' + && $params['noAccessories'] = 'on'; + + return $params; + } + + return null; + } + + public function getName() { + switch($this->queriedContext) { + case 'Sales': + return 'Bad Dragon Sales'; + case 'Clearance': + return 'Bad Dragon Clearance Search'; + default: + return parent::getName(); + } + } + + public function getURI() { + switch($this->queriedContext) { + case 'Sales': + return self::URI . 'sales'; + case 'Clearance': + return $this->inputToURL(); + default: + return parent::getURI(); + } + } + + public function collectData() { + switch($this->queriedContext) { + case 'Sales': + $sales = json_decode(getContents(self::URI . 'api/sales')) + or returnServerError('Failed to query BD API'); + + foreach($sales as $sale) { + $item = array(); + + $item['title'] = $sale->title; + $item['timestamp'] = strtotime($sale->startDate); + + $item['uri'] = $this->getURI() . '/' . $sale->slug; + + $contentHTML = '

'; + if(isset($sale->endDate)) { + $contentHTML .= '

This promotion ends on ' + . gmdate('M j, Y \a\t g:i A T', strtotime($sale->endDate)) + . '

'; + } else { + $contentHTML .= '

This promotion never ends

'; + } + $ul = false; + $content = json_decode($sale->content); + foreach($content->blocks as $block) { + switch($block->type) { + case 'header-one': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'header-two': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'header-three': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'unordered-list-item': + if(!$ul) { + $contentHTML .= ''; + $ul = false; + } + $contentHTML .= '

' . $block->text . '

'; + break; + } + } + $item['content'] = $contentHTML; + + $this->items[] = $item; + } + break; + case 'Clearance': + $toyData = json_decode(getContents($this->inputToURL(true))) + or returnServerError('Failed to query BD API'); + + $productList = json_decode(getContents(self::URI + . 'api/inventory-toy/product-list')) + or returnServerError('Failed to query BD API'); + + foreach($toyData->toys as $toy) { + $item = array(); + + $item['uri'] = $this->getURI() + . '#' + . $toy->id; + $item['timestamp'] = strtotime($toy->created); + + foreach($productList as $product) { + if($product->sku == $toy->sku) { + $item['title'] = $product->name; + break; + } + } + + // images + $content = '

'; + foreach($toy->images as $image) { + $content .= ''; + } + // price + $content .= '

Price: $' + . $toy->price + // size + . '
Size: ' + . $toy->size + // color + . '
Color: ' + . $toy->color + // features + . '
Features: ' + . ($toy->suction_cup ? 'Suction cup' : '') + . ($toy->suction_cup && $toy->cumtube ? ', ' : '') + . ($toy->cumtube ? 'Cumtube' : '') + . ($toy->suction_cup || $toy->cumtube ? '' : 'None'); + // firmness + $firmnessTexts = array( + '2' => 'Extra soft', + '3' => 'Soft', + '5' => 'Medium', + '8' => 'Firm' + ); + $firmnesses = explode('/', $toy->firmness); + if(count($firmnesses) === 2) { + $content .= '
Firmness: ' + . $firmnessTexts[$firmnesses[0]] + . ', ' + . $firmnessTexts[$firmnesses[1]]; + } else{ + $content .= '
Firmness: ' + . $firmnessTexts[$firmnesses[0]]; + } + // flop + if($toy->type === 'flop') { + $content .= '
Flop reason: ' + . $toy->flop_reason; + } + $content .= '

'; + $item['content'] = $content; + + $enclosures = array(); + foreach($toy->images as $image) { + $enclosures[] = $image->fullFilename; + } + $item['enclosures'] = $enclosures; + + $categories = array(); + $categories[] = $toy->sku; + $categories[] = $toy->type; + $categories[] = $toy->size; + if($toy->cumtube) { + $categories[] = 'cumtube'; + } + if($toy->suction_cup) { + $categories[] = 'suction_cup'; + } + $item['categories'] = $categories; + + $this->items[] = $item; + } + break; + } + } + + private function inputToURL($api = false) { + $url = self::URI; + $url .= ($api ? 'api/inventory-toys?' : 'shop/clearance?'); + + // Default parameters + $url .= 'limit=60'; + $url .= '&page=1'; + $url .= '&sort[field]=created'; + $url .= '&sort[direction]=desc'; + + // Product types + $url .= ($this->getInput('ready_made') ? '&type[]=ready_made' : ''); + $url .= ($this->getInput('flop') ? '&type[]=flop' : ''); + + // Product names + foreach(array_filter(explode(',', $this->getInput('skus'))) as $sku) { + $url .= '&skus[]=' . urlencode(trim($sku)); + } + + // Size + $url .= ($this->getInput('onesize') ? '&sizes[]=onesize' : ''); + $url .= ($this->getInput('mini') ? '&sizes[]=mini' : ''); + $url .= ($this->getInput('small') ? '&sizes[]=small' : ''); + $url .= ($this->getInput('medium') ? '&sizes[]=medium' : ''); + $url .= ($this->getInput('large') ? '&sizes[]=large' : ''); + $url .= ($this->getInput('extralarge') ? '&sizes[]=extralarge' : ''); + + // Category + $url .= ($this->getInput('category') ? '&category=' + . urlencode($this->getInput('category')) : ''); + + // Firmness + if($api) { + $url .= ($this->getInput('soft') ? '&firmnessValues[]=3' : ''); + $url .= ($this->getInput('med_firm') ? '&firmnessValues[]=5' : ''); + $url .= ($this->getInput('firm') ? '&firmnessValues[]=8' : ''); + if($this->getInput('split')) { + $url .= '&firmnessValues[]=3/5'; + $url .= '&firmnessValues[]=3/8'; + $url .= '&firmnessValues[]=8/3'; + $url .= '&firmnessValues[]=5/8'; + $url .= '&firmnessValues[]=8/5'; + } + } else{ + $url .= ($this->getInput('soft') ? '&firmnessValues[]=soft' : ''); + $url .= ($this->getInput('med_firm') ? '&firmnessValues[]=medium' : ''); + $url .= ($this->getInput('firm') ? '&firmnessValues[]=firm' : ''); + $url .= ($this->getInput('split') ? '&firmnessValues[]=split' : ''); + } + + // Price + $url .= ($this->getInput('maxprice') ? '&price[max]=' + . $this->getInput('maxprice') : '&price[max]=300'); + $url .= ($this->getInput('minprice') ? '&price[min]=' + . $this->getInput('minprice') : '&price[min]=0'); + + // Features + $url .= ($this->getInput('cumtube') ? '&cumtube=1' : ''); + $url .= ($this->getInput('suctionCup') ? '&suctionCup=1' : ''); + $url .= ($this->getInput('noAccessories') ? '&noAccessories=1' : ''); + + return $url; + } +} diff --git a/bridges/BakaUpdatesMangaReleasesBridge.php b/bridges/BakaUpdatesMangaReleasesBridge.php index cde9be84..27eca280 100644 --- a/bridges/BakaUpdatesMangaReleasesBridge.php +++ b/bridges/BakaUpdatesMangaReleasesBridge.php @@ -68,7 +68,7 @@ class BakaUpdatesMangaReleasesBridge extends BridgeAbstract { $item['title'] = implode(' ', $title); $item['uri'] = $this->getURI(); - $item['uid'] = hash('sha1', $item['title']); + $item['uid'] = $this->getSanitizedHash($item['title']); $this->items[] = $item; } @@ -89,8 +89,12 @@ class BakaUpdatesMangaReleasesBridge extends BridgeAbstract { return parent::getName(); } + private function getSanitizedHash($string) { + return hash('sha1', preg_replace('/[^a-zA-Z0-9\-\.]/', '', ucwords(strtolower($string)))); + } + private function filterText($text) { - return rtrim($text, '*'); + return rtrim($text, '* '); } private function filterHTML($text) { diff --git a/bridges/CourrierInternationalBridge.php b/bridges/CourrierInternationalBridge.php index 1e7c93e8..1b754e3d 100644 --- a/bridges/CourrierInternationalBridge.php +++ b/bridges/CourrierInternationalBridge.php @@ -3,7 +3,7 @@ class CourrierInternationalBridge extends BridgeAbstract { const MAINTAINER = 'teromene'; const NAME = 'Courrier International Bridge'; - const URI = 'http://CourrierInternational.com/'; + const URI = 'https://www.courrierinternational.com/'; const CACHE_TIMEOUT = 300; // 5 min const DESCRIPTION = 'Courrier International bridge'; diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index 45d33a53..1f66edc3 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -121,8 +121,8 @@ class ElloBridge extends BridgeAbstract { private function getAPIKey() { $cache = Cache::create(Configuration::getConfig('cache', 'type')); - $cache->setPath(PATH_CACHE); - $cache->setParameters(['key']); + $cache->setScope(get_called_class()); + $cache->setKey(['key']); $key = $cache->loadData(); if($key == null) { diff --git a/bridges/GooglePlusPostBridge.php b/bridges/GooglePlusPostBridge.php deleted file mode 100644 index 7911eaf4..00000000 --- a/bridges/GooglePlusPostBridge.php +++ /dev/null @@ -1,208 +0,0 @@ - array( - 'name' => 'username or Id', - 'required' => true - ), - 'include_media' => array( - 'name' => 'Include media', - 'type' => 'checkbox', - 'title' => 'Enable to include media in the feed content' - ) - )); - - public function getIcon() { - return 'https://ssl.gstatic.com/images/branding/product/ico/google_plus_alldp.ico'; - } - - public function collectData(){ - - $username = $this->getInput('username'); - - // Usernames start with a + if it's not an ID - if(!is_numeric($username) && substr($username, 0, 1) !== '+') { - $username = '+' . $username; - } - - $html = getSimpleHTMLDOM(static::URI . '/' . urlencode($username) . '/posts') - or returnServerError('No results for this query.'); - - $html = defaultLinkTo($html, static::URI); - - $this->title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); - $this->url = $html->find('meta[property=og:url]', 0)->getAttribute('content'); - - foreach($html->find('div[jsname=WsjYwc]') as $post) { - - $item = array(); - - $item['author'] = $post->find('div div div div a', 0)->innertext; - $item['uri'] = $post->find('div div div a', 1)->href; - - $timestamp = $post->find('a.qXj2He span', 0); - - if($timestamp) { - $item['timestamp'] = strtotime('+' . preg_replace( - '/[^0-9A-Za-z]/', - '', - $timestamp->getAttribute('aria-label'))); - } - - $message = $post->find('div[jsname=EjRJtf]', 0); - - // Empty messages are not supported right now - if(!$message) { - continue; - } - - $item['content'] = '
' - . $item['author'] - . '
' - . trim(strip_tags($message, '

')) - . '
'; - - // Make title at least 50 characters long, but don't add '...' if it is shorter! - if(strlen($message->plaintext) > 50) { - $end = strpos($message->plaintext, ' ', 50) ?: strlen($message->plaintext); - } else { - $end = strlen($message->plaintext); - } - - if(strlen(substr($message->plaintext, 0, $end)) === strlen($message->plaintext)) { - $item['title'] = $message->plaintext; - } else { - $item['title'] = substr($message->plaintext, 0, $end) . '...'; - } - - $media = $post->find('[jsname="MTOxpb"]', 0); - - if($media) { - - $item['enclosures'] = array(); - - foreach($media->find('img') as $img) { - $item['enclosures'][] = $this->fixImage($img)->src; - } - - if($this->getInput('include_media') === true && count($item['enclosures'] > 0)) { - $item['content'] .= '
'; - } - - } - - // Add custom parameters (only useful for JSON or Plaintext) - $item['fullname'] = $item['author']; - $item['avatar'] = $post->find('div img', 0)->src; - $item['id'] = $post->find('div div div', 0)->getAttribute('id'); - $item['content_simple'] = $message->plaintext; - - $this->items[] = $item; - - } - - } - - public function getName(){ - return $this->title ?: 'Google Plus Post Bridge'; - } - - public function getURI(){ - return $this->url ?: parent::getURI(); - } - - private function fixImage($img) { - - // There are certain images like .gif which link to a static picture and - // get replaced dynamically via JS in the browser. If we want the "real" - // image we need to account for that. - - $urlparts = parse_url($img->src); - - if(array_key_exists('host', $urlparts)) { - - // For some reason some URIs don't contain the scheme, assume https - if(!array_key_exists('scheme', $urlparts)) { - $urlparts['scheme'] = 'https'; - } - - $pathelements = explode('/', $urlparts['path']); - - switch($urlparts['host']) { - - case 'lh3.googleusercontent.com': - - if(pathinfo(end($pathelements), PATHINFO_EXTENSION)) { - - // The second to last element of the path specifies the - // image format. The URL is still valid if we remove it. - unset($pathelements[count($pathelements) - 2]); - - } elseif(strrpos(end($pathelements), '=') !== false) { - - // Some images go throug a proxy. For those images they - // add size information after an equal sign. - // Example: '=w530-h298-n'. Again this can safely be - // removed to get the original image. - $pathelements[count($pathelements) - 1] = substr( - end($pathelements), - 0, - strrpos(end($pathelements), '=') - ); - - } - - break; - - } - - $urlparts['path'] = implode('/', $pathelements); - - } - - $img->src = $this->build_url($urlparts); - return $img; - - } - - /** - * From: https://gist.github.com/Ellrion/f51ba0d40ae1d62eeae44fd1adf7b704 - * slightly adjusted to work with PHP < 7.0 - * @param array $parts - * @return string - */ - private function build_url(array $parts) - { - - $scheme = isset($parts['scheme']) ? ($parts['scheme'] . '://') : ''; - $host = isset($parts['host']) ? $parts['host'] : ''; - $port = isset($parts['port']) ? (':' . $parts['port']) : ''; - $user = isset($parts['user']) ? $parts['user'] : ''; - $pass = isset($parts['pass']) ? (':' . $parts['pass']) : ''; - $pass = ($user || $pass) ? ($pass . '@') : ''; - $path = isset($parts['path']) ? $parts['path'] : ''; - $query = isset($parts['query']) ? ('?' . $parts['query']) : ''; - $fragment = isset($parts['fragment']) ? ('#' . $parts['fragment']) : ''; - - return implode('', [$scheme, $user, $pass, $host, $port, $path, $query, $fragment]); - - } -} diff --git a/bridges/RoadAndTrackBridge.php b/bridges/RoadAndTrackBridge.php index f277b660..b3f0acc0 100644 --- a/bridges/RoadAndTrackBridge.php +++ b/bridges/RoadAndTrackBridge.php @@ -6,91 +6,60 @@ class RoadAndTrackBridge extends BridgeAbstract { const CACHE_TIMEOUT = 86400; // 24h const DESCRIPTION = 'Returns the latest news from Road & Track.'; - const PARAMETERS = array( - array( - 'new-cars' => array( - 'name' => 'New Cars', - 'type' => 'checkbox', - 'exampleValue' => 'checked', - 'title' => 'Activate to load New Cars articles' - ), - 'motorsports' => array( - 'name' => 'Motorsports', - 'type' => 'checkbox', - 'exampleValue' => 'checked', - 'title' => 'Activate to load Motorsports articles' - ), - 'car-culture' => array( - 'name' => 'Car Culture', - 'type' => 'checkbox', - 'exampleValue' => 'checked', - 'title' => 'Activate to load Car Culture articles' - ), - 'car-shows' => array( - 'name' => 'Car shows', - 'type' => 'checkbox', - 'exampleValue' => 'checked', - 'title' => 'Activate to load Car shows articles' - ) - ) - ); - - const API_TOKEN = '2e18e904-d9cd-4911-b30c-1817b1e0b04b'; - const SIG_URL = 'https://cloud.mazdigital.com/feeds/production/comboapp/204/api/v3/'; - const GSIG_URL = 'https://dashboard.mazsystems.com/services/cf_access?app_id=204&app_type=comboapp&api_token='; - public function collectData() { - $signVal = json_decode(getContents(self::GSIG_URL . self::API_TOKEN)); - $signVal = $signVal->signature; + $page = getSimpleHTMLDOM(self::URI); - $newsElements = array(); - if($this->getInput('new-cars')) { - $newsElements = array_merge($newsElements, - json_decode(getContents(self::SIG_URL . '7591/item_feed' . $signVal)) - ); - } - if($this->getInput('motorsports')) { - $newsElements = array_merge($newsElements, - json_decode(getContents(self::SIG_URL . '7590/item_feed' . $signVal)) - ); - } - if($this->getInput('car-culture')) { - $newsElements = array_merge($newsElements, - json_decode(getContents(self::SIG_URL . '7588/item_feed' . $signVal)) - ); - } - if($this->getInput('car-shows')) { - $newsElements = array_merge($newsElements, - json_decode(getContents(self::SIG_URL . '7589/item_feed' . $signVal)) - ); - } - - usort($newsElements, function($a, $b) { - return $b->published - $a->published; - }); + //Process the first element + $firstArticleLink = $page->find('.custom-promo-title', 0)->href; + $this->items[] = $this->fetchArticle($firstArticleLink); $limit = 19; - foreach($newsElements as $element) { - - $item = array(); - $item['uri'] = $element->sourceUrl; - $item['timestamp'] = $element->published; - $item['enclosures'] = array($element->cover->url); - $item['title'] = $element->title; - $item['content'] = $this->getArticleContent($element); - $this->items[] = $item; - - if($limit > 0) { - $limit--; - } else { - break; - } - + foreach($page->find('.full-item-title') as $article) { + $this->items[] = $this->fetchArticle($article->href); + $limit -= 1; + if($limit == 0) break; } } + private function fixImages($content) { + + $enclosures = []; + foreach($content->find('img') as $image) { + $image->src = explode('?', $image->getAttribute('data-src'))[0]; + $enclosures[] = $image->src; + } + + foreach($content->find('.embed-image-wrap, .content-lede-image-wrap') as $imgContainer) { + $imgContainer->style = ''; + } + + return $enclosures; + + } + + private function fetchArticle($articleLink) { + + $articleLink = self::URI . $articleLink; + $article = getSimpleHTMLDOM($articleLink); + $item = array(); + + $item['title'] = $article->find('.content-hed', 0)->innertext; + $item['author'] = $article->find('.byline-name', 0)->innertext; + $item['timestamp'] = strtotime($article->find('.content-info-date', 0)->getAttribute('datetime')); + + $content = $article->find('.content-container', 0); + if($content->find('.content-rail', 0) !== null) + $content->find('.content-rail', 0)->innertext = ''; + $enclosures = $this->fixImages($content); + + $item['enclosures'] = $enclosures; + $item['content'] = $content; + return $item; + + } + private function getArticleContent($article) { return getContents($article->contentUrl); diff --git a/bridges/WordPressPluginUpdateBridge.php b/bridges/WordPressPluginUpdateBridge.php index 51ddd5b7..9101c4ee 100644 --- a/bridges/WordPressPluginUpdateBridge.php +++ b/bridges/WordPressPluginUpdateBridge.php @@ -71,16 +71,4 @@ class WordPressPluginUpdateBridge extends BridgeAbstract { return parent::getName(); } - - private function getCachedDate($url){ - Debug::log('getting pubdate from url ' . $url . ''); - // Initialize cache - $cache = Cache::create(Configuration::getConfig('cache', 'type')); - $cache->setPath(PATH_CACHE . 'pages/'); - $params = [$url]; - $cache->setParameters($params); - // Get cachefile timestamp - $time = $cache->getTime(); - return ($time !== false ? $time : time()); - } } diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 67e95668..d2a45128 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -229,7 +229,7 @@ class YoutubeBridge extends BridgeAbstract { $url_listing = self::URI . 'playlist?list=' . urlencode($this->request); $html = $this->ytGetSimpleHTMLDOM($url_listing) or returnServerError("Could not request YouTube. Tried:\n - $url_listing"); - $item_count = $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a', false); + $item_count = $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a', true); if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { $this->ytBridgeParseXmlFeed($xml); } else { diff --git a/caches/FileCache.php b/caches/FileCache.php index 04d08a25..166ecdb5 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -3,20 +3,21 @@ * Cache with file system */ class FileCache implements CacheInterface { - protected $path; - protected $param; + protected $key; public function loadData(){ if(file_exists($this->getCacheFile())) { return unserialize(file_get_contents($this->getCacheFile())); } + + return null; } - public function saveData($datas){ + public function saveData($data){ // Notice: We use plain serialize() here to reduce memory footprint on // large input data. - $writeStream = file_put_contents($this->getCacheFile(), serialize($datas)); + $writeStream = file_put_contents($this->getCacheFile(), serialize($data)); if($writeStream === false) { throw new \Exception('Cannot write the cache... Do you have the right permissions ?'); @@ -29,13 +30,14 @@ class FileCache implements CacheInterface { $cacheFile = $this->getCacheFile(); clearstatcache(false, $cacheFile); if(file_exists($cacheFile)) { - return filemtime($cacheFile); + $time = filemtime($cacheFile); + return ($time !== false) ? $time : null; } - return false; + return null; } - public function purgeCache($duration){ + public function purgeCache($seconds){ $cachePath = $this->getPath(); if(file_exists($cachePath)) { $cacheIterator = new RecursiveIteratorIterator( @@ -47,7 +49,7 @@ class FileCache implements CacheInterface { if(in_array($cacheFile->getBasename(), array('.', '..', '.gitkeep'))) continue; elseif($cacheFile->isFile()) { - if(filemtime($cacheFile->getPathname()) < time() - $duration) + if(filemtime($cacheFile->getPathname()) < time() - $seconds) unlink($cacheFile->getPathname()); } } @@ -55,34 +57,34 @@ class FileCache implements CacheInterface { } /** - * Set cache path + * Set scope * @return self */ - public function setPath($path){ - if(is_null($path) || !is_string($path)) { - throw new \Exception('The given path is invalid!'); + public function setScope($scope){ + if(is_null($scope) || !is_string($scope)) { + throw new \Exception('The given scope is invalid!'); } - $this->path = $path; - - // Make sure path ends with '/' or '\' - $lastchar = substr($this->path, -1, 1); - if($lastchar !== '/' && $lastchar !== '\\') - $this->path .= '/'; - - if(!is_dir($this->path)) - mkdir($this->path, 0755, true); + $this->path = PATH_CACHE . trim($scope, " \t\n\r\0\x0B\\\/") . '/'; return $this; } /** - * Set HTTP GET parameters + * Set key * @return self */ - public function setParameters(array $param){ - $this->param = array_map('strtolower', $param); + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; return $this; } @@ -90,9 +92,15 @@ class FileCache implements CacheInterface { * Return cache path (and create if not exist) * @return string Cache path */ - protected function getPath(){ + private function getPath(){ if(is_null($this->path)) { - throw new \Exception('Call "setPath" first!'); + throw new \Exception('Call "setScope" first!'); + } + + if(!is_dir($this->path)) { + if (mkdir($this->path, 0755, true) !== true) { + throw new \Exception('Unable to create ' . $this->path); + } } return $this->path; @@ -102,7 +110,7 @@ class FileCache implements CacheInterface { * Get the file name use for cache store * @return string Path to the file cache */ - protected function getCacheFile(){ + private function getCacheFile(){ return $this->getPath() . $this->getCacheName(); } @@ -110,13 +118,11 @@ class FileCache implements CacheInterface { * Determines file name for store the cache * return string */ - protected function getCacheName(){ - if(is_null($this->param)) { - throw new \Exception('Call "setParameters" first!'); + private function getCacheName(){ + if(is_null($this->key)) { + throw new \Exception('Call "setKey" first!'); } - // Change character when making incompatible changes to prevent loading - // errors due to incompatible file contents \|/ - return hash('md5', http_build_query($this->param) . 'A') . '.cache'; + return hash('md5', $this->key) . '.cache'; } } diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php new file mode 100644 index 00000000..42291790 --- /dev/null +++ b/caches/MemcachedCache.php @@ -0,0 +1,115 @@ + 65535) { + returnServerError('"port" param is invalid for ' . get_called_class() . '. Please check your config.ini.php'); + } + + $conn = new Memcached(); + $conn->addServer($host, $port) or returnServerError('Could not connect to memcached server'); + $this->conn = $conn; + } + + public function loadData(){ + if ($this->data) return $this->data; + $result = $this->conn->get($this->getCacheKey()); + if ($result === false) { + return false; + } + + $this->time = $result['time']; + $this->data = $result['data']; + return $result['data']; + } + + public function saveData($datas){ + $time = time(); + $object_to_save = array( + 'data' => $datas, + 'time' => $time, + ); + $result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration); + + if($result === false) { + returnServerError('Cannot write the cache to memcached server'); + } + + $this->time = $time; + + return $this; + } + + public function getTime(){ + if ($this->time === false) { + $this->loadData(); + } + return $this->time; + } + + public function purgeCache($duration){ + // Note: does not purges cache right now + // Just sets cache expiration and leave cache purging for memcached itself + $this->expiration = $duration; + } + + /** + * Set scope + * @return self + */ + public function setScope($scope){ + $this->scope = $scope; + return $this; + } + + /** + * Set key + * @return self + */ + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; + return $this; + } + + private function getCacheKey(){ + if(is_null($this->key)) { + returnServerError('Call "setKey" first!'); + } + + return 'rss_bridge_cache_' . hash('md5', $this->scope . $this->key . 'A'); + } +} diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index 5cbb3772..7d0f584f 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -3,16 +3,25 @@ * Cache based on SQLite 3 */ class SQLiteCache implements CacheInterface { - protected $path; - protected $param; + protected $scope; + protected $key; private $db = null; public function __construct() { - if (!extension_loaded('sqlite3')) + if (!extension_loaded('sqlite3')) { die('"sqlite3" extension not loaded. Please check "php.ini"'); + } - $file = PATH_CACHE . 'cache.sqlite'; + $file = Configuration::getConfig(get_called_class(), 'file'); + if (empty($file)) { + die('Configuration for ' . get_called_class() . ' missing. Please check your config.ini.php'); + } + if (dirname($file) == '.') { + $file = PATH_CACHE . $file; + } elseif (!is_dir(dirname($file))) { + die('Invalid configuration for ' . get_called_class() . '. Please check your config.ini.php'); + } if (!is_file($file)) { $this->db = new SQLite3($file); @@ -39,10 +48,10 @@ class SQLiteCache implements CacheInterface { return null; } - public function saveData($datas){ + public function saveData($data){ $Qupdate = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)'); $Qupdate->bindValue(':key', $this->getCacheKey()); - $Qupdate->bindValue(':value', serialize($datas)); + $Qupdate->bindValue(':value', serialize($data)); $Qupdate->bindValue(':updated', time()); $Qupdate->execute(); @@ -60,40 +69,53 @@ class SQLiteCache implements CacheInterface { } } - return false; + return null; } - public function purgeCache($duration){ + public function purgeCache($seconds){ $Qdelete = $this->db->prepare('DELETE FROM storage WHERE updated < :expired'); - $Qdelete->bindValue(':expired', time() - $duration); + $Qdelete->bindValue(':expired', time() - $seconds); $Qdelete->execute(); } /** - * Set cache path + * Set scope * @return self */ - public function setPath($path){ - $this->path = $path; + public function setScope($scope){ + if(is_null($scope) || !is_string($scope)) { + throw new \Exception('The given scope is invalid!'); + } + + $this->scope = $scope; return $this; } /** - * Set HTTP GET parameters + * Set key * @return self */ - public function setParameters(array $param){ - $this->param = array_map('strtolower', $param); + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; return $this; } //////////////////////////////////////////////////////////////////////////// - protected function getCacheKey(){ - if(is_null($this->param)) { - throw new \Exception('Call "setParameters" first!'); + private function getCacheKey(){ + if(is_null($this->key)) { + throw new \Exception('Call "setKey" first!'); } - return hash('sha1', $this->path . http_build_query($this->param), true); + return hash('sha1', $this->scope . $this->key, true); } } diff --git a/config.default.ini.php b/config.default.ini.php index 394658d6..5f4a75f6 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -52,3 +52,12 @@ username = "" ; The password for authentication. Insert this password when prompted for login. ; Use a strong password to prevent others from guessing your login! password = "" + +; --- Cache specific configuration --------------------------------------------- + +[SQLiteCache] +file = "cache.sqlite" + +[MemcachedCache] +host = "localhost" +port = 11211 diff --git a/index.php b/index.php index 819b5a52..771e3379 100644 --- a/index.php +++ b/index.php @@ -37,7 +37,6 @@ $whitelist_default = array( 'DuckDuckGoBridge', 'FacebookBridge', 'FlickrBridge', - 'GooglePlusPostBridge', 'GoogleSearchBridge', 'IdenticaBridge', 'InstagramBridge', diff --git a/lib/CacheInterface.php b/lib/CacheInterface.php index a74fc0dd..091c5f02 100644 --- a/lib/CacheInterface.php +++ b/lib/CacheInterface.php @@ -13,38 +13,54 @@ /** * The cache interface - * - * @todo Add missing function to the interface - * @todo Explain parameters and return values in more detail - * @todo Return self more often (to allow call chaining) */ interface CacheInterface { + /** + * Set scope of the current cache + * + * If $scope is an empty string, the cache is set to a global context. + * + * @param string $scope The scope the data is related to + */ + public function setScope($scope); + + /** + * Set key to assign the current data + * + * Since $key can be anything, the cache implementation must ensure to + * assign the related data reliably; most commonly by serializing and + * hashing the key in an appropriate way. + * + * @param array $key The key the data is related to + */ + public function setKey($key); + /** * Loads data from cache * - * @return mixed The cache data + * @return mixed The cached data or null */ public function loadData(); /** * Stores data to the cache * - * @param mixed $datas The data to store + * @param mixed $data The data to store * @return self The cache object */ - public function saveData($datas); + public function saveData($data); /** - * Returns the timestamp for the curent cache file + * Returns the timestamp for the curent cache data * - * @return int Timestamp + * @return int Timestamp or null */ public function getTime(); /** - * Removes any data that is older than the specified duration from cache + * Removes any data that is older than the specified age from cache * - * @param int $duration The cache duration in seconds + * @param int $seconds The cache age in seconds */ - public function purgeCache($duration); + public function purgeCache($seconds); } diff --git a/lib/Configuration.php b/lib/Configuration.php index 2b736119..be9315c8 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -28,7 +28,7 @@ final class Configuration { * * @todo Replace this property by a constant. */ - public static $VERSION = 'dev.2019-03-17'; + public static $VERSION = 'dev.2019-05-08'; /** * Holds the configuration data. diff --git a/lib/contents.php b/lib/contents.php index 4740f5c2..c65d6dfb 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -46,11 +46,11 @@ function getContents($url, $header = array(), $opts = array()){ // Initialize cache $cache = Cache::create(Configuration::getConfig('cache', 'type')); - $cache->setPath(PATH_CACHE . 'server/'); + $cache->setScope('server'); $cache->purgeCache(86400); // 24 hours (forced) $params = [$url]; - $cache->setParameters($params); + $cache->setKey($params); // Use file_get_contents if in CLI mode with no root certificates defined if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) { @@ -271,11 +271,11 @@ function getSimpleHTMLDOMCached($url, // Initialize cache $cache = Cache::create(Configuration::getConfig('cache', 'type')); - $cache->setPath(PATH_CACHE . 'pages/'); + $cache->setScope('pages'); $cache->purgeCache(86400); // 24 hours (forced) $params = [$url]; - $cache->setParameters($params); + $cache->setKey($params); // Determine if cached file is within duration $time = $cache->getTime();