'
- . $finish_name . ' ' . $serie . '
'
+ . $finish_name . ' ' . $serie . '
{$fullArticle}
EOD; } else { @@ -85,11 +77,21 @@ EOD; EOD; } - $item['categories'] = array_map(function($e){ return $e->plaintext; }, $article->find('div.tags span.tag')); + // categories + foreach ($article->find('a.crayons-tag') as $tag) { + $item['categories'][] = str_replace('#', '', $tag->plaintext); + } $this->items[] = $item; } + } + public function getName() { + if (!is_null($this->getInput('tag'))) { + return ucfirst($this->getInput('tag')) . ' - dev.to'; + } + + return parent::getName(); } private function getFullArticle($url) { @@ -98,6 +100,10 @@ EOD; $html = defaultLinkTo($html, static::URI); + if ($html->find('div.crayons-article__cover', 0)) { + return $html->find('div.crayons-article__cover', 0) . $html->find('[id="article-body"]', 0); + } + return $html->find('[id="article-body"]', 0); } } diff --git a/bridges/DiarioDeNoticiasBridge.php b/bridges/DiarioDeNoticiasBridge.php new file mode 100644 index 00000000..887eb117 --- /dev/null +++ b/bridges/DiarioDeNoticiasBridge.php @@ -0,0 +1,84 @@ + array( + 'n' => array( + 'name' => 'Tag Name', + 'exampleValue' => 'rogerio-casanova', + ) + ) + ); + + const MONPT = array( + 'jan', + 'fev', + 'mar', + 'abr', + 'mai', + 'jun', + 'jul', + 'ago', + 'set', + 'out', + 'nov', + 'dez', + ); + + public function getIcon() { + return 'https://static.globalnoticias.pt/dn/common/images/favicons/favicon-128.png'; + } + + public function getName() { + switch($this->queriedContext) { + case 'Tag': + $name = self::NAME . ' | Tag | ' . $this->getInput('n'); + break; + default: + $name = self::NAME; + } + return $name; + } + + public function getURI() { + switch($this->queriedContext) { + case 'Tag': + $url = self::URI . '/tag/' . $this->getInput('n') . '.html'; + break; + default: + $url = self::URI; + } + return $url; + } + + public function collectData() { + $archives = self::getURI(); + $html = getSimpleHTMLDOMCached($archives) + or returnServerError('Could not load content'); + + foreach($html->find('article') as $element) { + $item = array(); + + $title = $element->find('.t-am-title', 0); + $link = $element->find('a.t-am-text', 0); + + $item['title'] = $title->plaintext; + $item['uri'] = self::URI . $link->href; + + $snippet = $element->find('.t-am-lead', 0); + if ($snippet) { + $item['content'] = $snippet->plaintext; + } + preg_match('|edicao-do-dia\\/(?P' . $description . '
+Lieu : ' . $city . ' - ' . $region . '
+Par : ' . $author . '
+Date : ' . $date . '
+' - . $description + . urldecode($description) . '
'; $item['enclosures'] = $this->extractEnclosures($model); @@ -98,6 +148,46 @@ class FlickrBridge extends BridgeAbstract { } + public function getURI() { + + switch($this->queriedContext) { + case 'Explore': + return self::URI . 'explore'; + break; + case 'By keyword': + return self::URI . 'search/?q=' . urlencode($this->getInput('q')) + . '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media'); + break; + case 'By username': + return self::URI . 'search/?user_id=' . urlencode($this->getInput('u')) + . '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media'); + break; + + default: + return parent::getURI(); + } + } + + public function getName() { + + switch($this->queriedContext) { + case 'Explore': + return 'Explore - ' . self::NAME; + break; + case 'By keyword': + return $this->getInput('q') . ' - keyword - ' . self::NAME; + break; + case 'By username': + return $this->username . ' - ' . self::NAME; + break; + + default: + return parent::getName(); + } + + return parent::getName(); + } + private function extractJsonModel($html) { // Find SCRIPT containing JSON data diff --git a/bridges/FolhaDeSaoPauloBridge.php b/bridges/FolhaDeSaoPauloBridge.php new file mode 100644 index 00000000..9a9717cb --- /dev/null +++ b/bridges/FolhaDeSaoPauloBridge.php @@ -0,0 +1,52 @@ + array( + 'name' => 'Feed sub-URL', + 'type' => 'text', + 'title' => 'Select the sub-feed (see https://www1.folha.uol.com.br/feed/)', + 'exampleValue' => 'emcimadahora/rss091.xml', + ) + ) + ); + + protected function parseItem($item){ + $item = parent::parseItem($item); + + $articleHTMLContent = getSimpleHTMLDOMCached($item['uri']); + if($articleHTMLContent) { + foreach ($articleHTMLContent->find('div.c-news__body .is-hidden') as $toRemove) { + $toRemove->innertext = ''; + } + $item_content = $articleHTMLContent->find('div.c-news__body', 0); + if ($item_content) { + $text = $item_content->innertext; + $text = strip_tags($text, ''; + $item['categories'] = array_unique($item['categories']); + $this->items[] = $item; } } @@ -144,25 +194,37 @@ class TwitchBridge extends BridgeAbstract { $seconds % 60); } - /* - * Ideally the new 'helix' API should be used as v5/'kraken' is deprecated. - * The new API however still misses many features (markers, played game..) of - * the old one, so let's use the old one for as long as it's available. - */ - private function apiGet($endpoint, $query_data = array()) { - $query_data['api_version'] = 5; - $url = 'https://api.twitch.tv/kraken/' - . $endpoint - . '?' - . http_build_query($query_data); + // GraphQL: https://graphql.org/ + // Tool for developing/testing queries: https://github.com/skevy/graphiql-app + private function apiRequest($query, $variables) { + $request = array( + 'query' => $query, + 'variables' => $variables + ); $header = array( 'Client-ID: ' . self::CLIENT_ID ); + $opts = array( + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => json_encode($request) + ); - $data = json_decode(getContents($url, $header)) - or returnServerError('API request to "' . $url . '" failed.'); + Debug::log("Sending GraphQL query:\n" . $query); + Debug::log("Sending GraphQL variables:\n" + . json_encode($variables, JSON_PRETTY_PRINT)); - return $data; + $response = json_decode(getContents(self::API_ENDPOINT, $header, $opts)) + or returnServerError('API request to "' . self::API_ENDPOINT . '" failed.'); + + Debug::log("Got GraphQL response:\n" + . json_encode($response, JSON_PRETTY_PRINT)); + + if(isset($response->errors)) { + $messages = array_column($response->errors, 'message'); + returnServerError('API error(s): ' . implode("\n", $messages)); + } + + return $response->data; } public function getName(){ diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 2f5565b1..c62a6595 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -2,6 +2,9 @@ class TwitterBridge extends BridgeAbstract { const NAME = 'Twitter Bridge'; const URI = 'https://twitter.com/'; + const API_URI = 'https://api.twitter.com'; + const GUEST_TOKEN_USES = 100; + const GUEST_TOKEN_EXPIRY = 300; // 5min const CACHE_TIMEOUT = 300; // 5min const DESCRIPTION = 'returns tweets'; const MAINTAINER = 'pmaziere'; @@ -72,6 +75,12 @@ EOD 'required' => false, 'type' => 'checkbox', 'title' => 'Hide retweets' + ), + 'nopinned' => array( + 'name' => 'Without pinned tweet', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Hide pinned tweet' ) ), 'By list' => array( @@ -92,6 +101,20 @@ EOD 'required' => false, 'title' => 'Specify term to search for' ) + ), + 'By list ID' => array( + 'listid' => array( + 'name' => 'List ID', + 'exampleValue' => '31748', + 'required' => true, + 'title' => 'Insert the list id' + ), + 'filter' => array( + 'name' => 'Filter', + 'exampleValue' => '#rss-bridge', + 'required' => false, + 'title' => 'Specify term to search for' + ) ) ); @@ -142,6 +165,8 @@ EOD break; case 'By list': return $this->getInput('list') . ' - Twitter list by ' . $this->getInput('user'); + case 'By list ID': + return 'Twitter List #' . $this->getInput('listid'); default: return parent::getName(); } return 'Twitter ' . $specific . $this->getInput($param); @@ -164,22 +189,46 @@ EOD . urlencode($this->getInput('user')) . '/lists/' . str_replace(' ', '-', strtolower($this->getInput('list'))); + case 'By list ID': + return self::URI + . 'i/lists/' + . urlencode($this->getInput('listid')); default: return parent::getURI(); } } + private function getApiURI() { + switch($this->queriedContext) { + case 'By keyword or hashtag': + return self::API_URI + . '/2/search/adaptive.json?q=' + . urlencode($this->getInput('q')) + . '&tweet_mode=extended&tweet_search_mode=live'; + case 'By username': + return self::API_URI + . '/2/timeline/profile/' + . $this->getRestId($this->getInput('u')) + . '.json?tweet_mode=extended'; + case 'By list': + return self::API_URI + . '/2/timeline/list.json?list_id=' + . $this->getListId($this->getInput('user'), $this->getInput('list')) + . '&tweet_mode=extended'; + case 'By list ID': + return self::API_URI + . '/2/timeline/list.json?list_id=' + . $this->getInput('listid') + . '&tweet_mode=extended'; + default: returnServerError('Invalid query context !'); + } + } + public function collectData(){ $html = ''; $page = $this->getURI(); + $data = json_decode($this->getApiContents($this->getApiURI())); - if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) { - $cookies = $this->getCookies($page); - $html = getSimpleHTMLDOM($page, array("Cookie: $cookies")); - } else { - $html = getSimpleHTMLDOM($page, array(), array(CURLOPT_COOKIEFILE => '')); - } - - if(!$html) { + if(!$data) { switch($this->queriedContext) { case 'By keyword or hashtag': returnServerError('No results for this query.'); @@ -192,75 +241,93 @@ EOD $hidePictures = $this->getInput('nopic'); - foreach($html->find('div.js-stream-tweet') as $tweet) { + $promotedTweetIds = array_reduce($data->timeline->instructions[0]->addEntries->entries, function($carry, $entry) { + if (!isset($entry->content->item)) { + return $carry; + } + $tweet = $entry->content->item->content->tweet; + if (isset($tweet->promotedMetadata)) { + $carry[] = $tweet->id; + } + return $carry; + }, array()); - // Skip retweets? - if($this->getInput('noretweet') - && $tweet->find('div.context span.js-retweet-text a', 0)) { + $hidePinned = $this->getInput('nopinned'); + if ($hidePinned) { + $pinnedTweetId = null; + if (isset($data->timeline->instructions[1]) && isset($data->timeline->instructions[1]->pinEntry)) { + $pinnedTweetId = $data->timeline->instructions[1]->pinEntry->entry->content->item->content->tweet->id; + } + } + + foreach($data->globalObjects->tweets as $tweet) { + + /* Debug::log('>>> ' . json_encode($tweet)); */ + // Skip spurious retweets + if (isset($tweet->retweeted_status_id_str) && substr($tweet->full_text, 0, 4) === 'RT @') { continue; } - // remove 'invisible' content - foreach($tweet->find('.invisible') as $invisible) { - $invisible->outertext = ''; + // Skip promoted tweets + if (in_array($tweet->id_str, $promotedTweetIds)) { + continue; } - // Skip protmoted tweets - $heading = $tweet->previousSibling(); - if(!is_null($heading) && - $heading->getAttribute('class') === 'promoted-tweet-heading' - ) { + // Skip pinned tweet + if ($hidePinned && $tweet->id_str === $pinnedTweetId) { continue; } $item = array(); // extract username and sanitize - $item['username'] = htmlspecialchars_decode($tweet->getAttribute('data-screen-name'), ENT_QUOTES); - // extract fullname (pseudonym) - $item['fullname'] = htmlspecialchars_decode($tweet->getAttribute('data-name'), ENT_QUOTES); - // get author + $user_info = $this->getUserInformation($tweet->user_id_str, $data->globalObjects); + + $item['username'] = $user_info->screen_name; + $item['fullname'] = $user_info->name; $item['author'] = $item['fullname'] . ' (@' . $item['username'] . ')'; - if($rt = $tweet->find('div.context span.js-retweet-text a', 0)) { - $item['author'] .= ' RT: @' . $rt->plaintext; + if (null !== $this->getInput('u') && strtolower($item['username']) != strtolower($this->getInput('u'))) { + $item['author'] .= ' RT: @' . $this->getInput('u'); } - // get avatar link - $item['avatar'] = $tweet->find('img', 0)->src; - // get TweetID - $item['id'] = $tweet->getAttribute('data-tweet-id'); - // get tweet link - $item['uri'] = self::URI . substr($tweet->find('a.js-permalink', 0)->getAttribute('href'), 1); + $item['avatar'] = $user_info->profile_image_url_https; + + $item['id'] = $tweet->id_str; + $item['uri'] = self::URI . $item['username'] . '/status/' . $item['id']; // extract tweet timestamp - $item['timestamp'] = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time'); - // generate the title - $item['title'] = strip_tags($this->fixAnchorSpacing(htmlspecialchars_decode( - $tweet->find('p.js-tweet-text', 0), ENT_QUOTES), '')); + $item['timestamp'] = $tweet->created_at; - switch($this->queriedContext) { - case 'By list': - // Check if filter applies to list (using raw content) - if($this->getInput('filter')) { - if(stripos($tweet->find('p.js-tweet-text', 0)->plaintext, $this->getInput('filter')) === false) { - continue 2; // switch + for-loop! - } - } - break; - default: + // Convert plain text URLs into HTML hyperlinks + $cleanedTweet = $tweet->full_text; + $foundUrls = false; + + if (isset($tweet->entities->media)) { + foreach($tweet->entities->media as $media) { + $cleanedTweet = str_replace($media->url, + '' . $media->display_url . '', + $cleanedTweet); + $foundUrls = true; + } } + if (isset($tweet->entities->urls)) { + foreach($tweet->entities->urls as $url) { + $cleanedTweet = str_replace($url->url, + '' . $url->display_url . '', + $cleanedTweet); + $foundUrls = true; + } + } + if ($foundUrls === false) { + // fallback to regex'es + $reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/'; + if(preg_match($reg_ex, $tweet->full_text, $url)) { + $cleanedTweet = preg_replace($reg_ex, + "{$url[0]} ", + $cleanedTweet); + } + } + // generate the title + $item['title'] = strip_tags($cleanedTweet); - $this->processContentLinks($tweet); - $this->processEmojis($tweet); - - // get tweet text - $cleanedTweet = str_replace( - 'href="/', - 'href="' . self::URI, - $tweet->find('p.js-tweet-text', 0)->innertext - ); - - // fix anchors missing spaces in-between - $cleanedTweet = $this->fixAnchorSpacing($cleanedTweet); - - // Add picture to content + // Add avatar $picture_html = ''; if(!$hidePictures) { $picture_html = <<{$cleanedTweet}
{$image_html}+
{$media_html}
{$cleanedQuotedTweet}-
{$quotedImage_html}-
' + . $app['Overview'] + . '
'; + + if(array_key_exists('Project', $app)) + $item['uri'] = $app['Project']; + + if(array_key_exists('Registry', $app)) + $item['content'] .= '') + { + $markup = $trimmedMarkup; + $markup = substr($markup, 3); + + $position = strpos($markup, "
"); + + $markup = substr_replace($markup, '', $position, 4); + } + + return $markup; + } + + # + # Deprecated Methods + # + + function parse($text) + { + $markup = $this->text($text); + + return $markup; + } + + protected function sanitiseElement(array $Element) + { + static $goodAttribute = '/^[a-zA-Z0-9][a-zA-Z0-9-_]*+$/'; + static $safeUrlNameToAtt = array( + 'a' => 'href', + 'img' => 'src', + ); + + if (isset($safeUrlNameToAtt[$Element['name']])) + { + $Element = $this->filterUnsafeUrlInAttribute($Element, $safeUrlNameToAtt[$Element['name']]); + } + + if ( ! empty($Element['attributes'])) + { + foreach ($Element['attributes'] as $att => $val) + { + # filter out badly parsed attribute + if ( ! preg_match($goodAttribute, $att)) + { + unset($Element['attributes'][$att]); + } + # dump onevent attribute + elseif (self::striAtStart($att, 'on')) + { + unset($Element['attributes'][$att]); + } + } + } + + return $Element; + } + + protected function filterUnsafeUrlInAttribute(array $Element, $attribute) + { + foreach ($this->safeLinksWhitelist as $scheme) + { + if (self::striAtStart($Element['attributes'][$attribute], $scheme)) + { + return $Element; + } + } + + $Element['attributes'][$attribute] = str_replace(':', '%3A', $Element['attributes'][$attribute]); + + return $Element; + } + + # + # Static Methods + # + + protected static function escape($text, $allowQuotes = false) + { + return htmlspecialchars($text, $allowQuotes ? ENT_NOQUOTES : ENT_QUOTES, 'UTF-8'); + } + + protected static function striAtStart($string, $needle) + { + $len = strlen($needle); + + if ($len > strlen($string)) + { + return false; + } + else + { + return strtolower(substr($string, 0, $len)) === strtolower($needle); + } + } + + static function instance($name = 'default') + { + if (isset(self::$instances[$name])) + { + return self::$instances[$name]; + } + + $instance = new static(); + + self::$instances[$name] = $instance; + + return $instance; + } + + private static $instances = array(); + + # + # Fields + # + + protected $DefinitionData; + + # + # Read-Only + + protected $specialCharacters = array( + '\\', '`', '*', '_', '{', '}', '[', ']', '(', ')', '>', '#', '+', '-', '.', '!', '|', + ); + + protected $StrongRegex = array( + '*' => '/^[*]{2}((?:\\\\\*|[^*]|[*][^*]*[*])+?)[*]{2}(?![*])/s', + '_' => '/^__((?:\\\\_|[^_]|_[^_]*_)+?)__(?!_)/us', + ); + + protected $EmRegex = array( + '*' => '/^[*]((?:\\\\\*|[^*]|[*][*][^*]+?[*][*])+?)[*](?![*])/s', + '_' => '/^_((?:\\\\_|[^_]|__[^_]*__)+?)_(?!_)\b/us', + ); + + protected $regexHtmlAttribute = '[a-zA-Z_:][\w:.-]*(?:\s*=\s*(?:[^"\'=<>`\s]+|"[^"]*"|\'[^\']*\'))?'; + + protected $voidElements = array( + 'area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', + ); + + protected $textLevelElements = array( + 'a', 'br', 'bdo', 'abbr', 'blink', 'nextid', 'acronym', 'basefont', + 'b', 'em', 'big', 'cite', 'small', 'spacer', 'listing', + 'i', 'rp', 'del', 'code', 'strike', 'marquee', + 'q', 'rt', 'ins', 'font', 'strong', + 's', 'tt', 'kbd', 'mark', + 'u', 'xm', 'sub', 'nobr', + 'sup', 'ruby', + 'var', 'span', + 'wbr', 'time', + ); +}
link to a iframe (could be a video): ' - . $element->src - . '
'; - } + $html = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Could not request: ' . $item['uri']); - $text = strip_tags($text, '
Played games:
'; + if(count($video->moments->edges) > 0) { + foreach($video->moments->edges as $edge) { + $moment = $edge->node; + + $item['categories'][] = $moment->description; $item['content'] .= '- '
- . $this->formatTimestampTime($game_change->time)
+ . $this->formatTimestampTime($moment->positionMilliseconds / 1000)
. ' - '
- . $game_change->label
+ . $moment->description
. '
';
}
+ } else {
+ $item['content'] .= '- 00:00:00 - '
+ . ($video->game ? $video->game->displayName : 'No Game')
+ . '
';
}
$item['content'] .= '