diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php deleted file mode 100644 index 4a7ec9d2..00000000 --- a/bridges/TwitterBridge.php +++ /dev/null @@ -1,80 +0,0 @@ -maintainer = "mitsukarenai"; - $this->name = "Twitter Bridge"; - $this->uri = "http://twitter.com/"; - $this->description = "Returns user timelines or keyword/hashtag search results (without using their API)."; - $this->update = "2016-08-09"; - - $this->parameters["By keyword or hashtag"] = - '[ - { - "name" : "Keyword or #hashtag", - "identifier" : "q" - } - ]'; - - $this->parameters["By username"] = - '[ - { - "name" : "username", - "identifier" : "u" - } - ]'; - - } - - public function collectData(array $param){ - $html = ''; - if (isset($param['q'])) { /* keyword search mode */ - $this->request = $param['q']; - $html = $this->file_get_html('https://twitter.com/search?q='.urlencode($this->request).'&f=tweets') or $this->returnError('No results for this query.', 404); - } - elseif (isset($param['u'])) { /* user timeline mode */ - $this->request = $param['u']; - $html = $this->file_get_html('http://twitter.com/'.urlencode($this->request)) or $this->returnError('Requested username can\'t be found.', 404); - } - else { - $this->returnError('You must specify a keyword (?q=...) or a Twitter username (?u=...).', 400); - } - - foreach($html->find('div.js-stream-tweet') as $tweet) { - $item = new \Item(); - $item->username = $tweet->getAttribute('data-screen-name'); // extract username and sanitize - $item->fullname = $tweet->getAttribute('data-name'); // extract fullname (pseudonym) - $item->avatar = $tweet->find('img', 0)->src; // get avatar link - $item->id = $tweet->getAttribute('data-tweet-id'); // get TweetID - $item->uri = 'https://twitter.com'.$tweet->find('a.js-permalink', 0)->getAttribute('href'); // get tweet link - $item->timestamp = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time'); // extract tweet timestamp - // processing content links - foreach($tweet->find('a') as $link) { - if($link->hasAttribute('data-expanded-url') ) { - $link->href = $link->getAttribute('data-expanded-url'); - } - $link->removeAttribute('data-expanded-url'); - $link->removeAttribute('data-query-source'); - $link->removeAttribute('rel'); - $link->removeAttribute('class'); - $link->removeAttribute('target'); - $link->removeAttribute('title'); - } - $item->content = str_replace('href="/', 'href="https://twitter.com/', strip_tags($tweet->find('p.js-tweet-text', 0)->innertext, '')); // extract tweet text - $item->title = $item->fullname . ' (@' . $item->username . ') | ' . html_entity_decode(strip_tags($item->content),ENT_QUOTES,'UTF-8'); - $this->items[] = $item; - } - } - - public function getName(){ - return (!empty($this->request) ? $this->request .' - ' : '') .'Twitter Bridge'; - } - - public function getCacheDuration(){ - return 300; // 5 minutes - } -} diff --git a/bridges/TwitterBridgeClean.php b/bridges/TwitterBridgeClean.php deleted file mode 100644 index 235469ce..00000000 --- a/bridges/TwitterBridgeClean.php +++ /dev/null @@ -1,81 +0,0 @@ -maintainer = "vinzv"; - $this->name = "Twitter Bridge Clean"; - $this->uri = "http://twitter.com/"; - $this->description = "Returns user timelines without username in title or search results for keywords/hashtags (without using their API)."; - $this->update = "2016-08-09"; - - $this->parameters["By keyword or hashtag"] = - '[ - { - "name" : "Keyword or #hashtag", - "identifier" : "q" - } - ]'; - - $this->parameters["By username"] = - '[ - { - "name" : "username", - "identifier" : "u" - } - ]'; - - } - - public function collectData(array $param){ - $html = ''; - if (isset($param['q'])) { /* keyword search mode */ - $this->request = $param['q']; - $html = $this->file_get_html('https://twitter.com/search?q='.urlencode($this->request).'&f=tweets') or $this->returnError('No results for this query.', 404); - } - elseif (isset($param['u'])) { /* user timeline mode */ - $this->request = $param['u']; - $html = $this->file_get_html('http://twitter.com/'.urlencode($this->request)) or $this->returnError('Requested username can\'t be found.', 404); - } - else { - $this->returnError('You must specify a keyword (?q=...) or a Twitter username (?u=...).', 400); - } - - foreach($html->find('div.js-stream-tweet') as $tweet) { - $item = new \Item(); - $item->username = $tweet->getAttribute('data-screen-name'); // extract username and sanitize - $item->fullname = $tweet->getAttribute('data-name'); // extract fullname (pseudonym) - $item->avatar = $tweet->find('img', 0)->src; // get avatar link - $item->id = $tweet->getAttribute('data-tweet-id'); // get TweetID - $item->uri = 'https://twitter.com'.$tweet->find('a.js-permalink', 0)->getAttribute('href'); // get tweet link - $item->timestamp = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time'); // extract tweet timestamp - // processing content links - foreach($tweet->find('a') as $link) { - if($link->hasAttribute('data-expanded-url') ) { - $link->href = $link->getAttribute('data-expanded-url'); - } - $link->removeAttribute('data-expanded-url'); - $link->removeAttribute('data-query-source'); - $link->removeAttribute('rel'); - $link->removeAttribute('class'); - $link->removeAttribute('target'); - $link->removeAttribute('title'); - $link->removeAttribute('dir'); - } - $item->content = str_replace('pic.twitter.com', 'https://pic.twitter.com', strip_tags($tweet->find('p.js-tweet-text', 0)->innertext)); // extract tweet text - $item->title = html_entity_decode($item->content,ENT_QUOTES,'UTF-8'); - $this->items[] = $item; - } - } - - public function getName(){ - return (!empty($this->request) ? $this->request .' - ' : '') .'Twitter Bridge'; - } - - public function getCacheDuration(){ - return 300; // 5 minutes - } -} diff --git a/bridges/TwitterBridgeCleanExtended.php b/bridges/TwitterBridgeCleanExtended.php deleted file mode 100644 index 1be4c7b0..00000000 --- a/bridges/TwitterBridgeCleanExtended.php +++ /dev/null @@ -1,86 +0,0 @@ -maintainer = "Max Mehl"; - $this->name = "Twitter Bridge Clean Extended"; - $this->uri = "http://twitter.com/"; - $this->description = "Returns user timelines showing RTs correctly or search results for keywords/hashtags (without using their API)."; - $this->update = "2016-08-09"; - - $this->parameters["By keyword or hashtag"] = - '[ - { - "name" : "Keyword or #hashtag", - "identifier" : "q" - } - ]'; - - $this->parameters["By username"] = - '[ - { - "name" : "username", - "identifier" : "u" - } - ]'; - - } - - public function collectData(array $param){ - $html = ''; - if (isset($param['q'])) { /* keyword search mode */ - $this->request = $param['q']; - $html = $this->file_get_html('https://twitter.com/search?q='.urlencode($this->request).'&f=tweets') or $this->returnError('No results for this query.', 404); - } - elseif (isset($param['u'])) { /* user timeline mode */ - $this->request = $param['u']; - $html = $this->file_get_html('http://twitter.com/'.urlencode($this->request)) or $this->returnError('Requested username can\'t be found.', 404); - } - else { - $this->returnError('You must specify a keyword (?q=...) or a Twitter username (?u=...).', 400); - } - - foreach($html->find('div.js-stream-tweet') as $tweet) { - $item = new \Item(); - $item->username = $tweet->getAttribute('data-screen-name'); // extract username and sanitize - $item->fullname = $tweet->getAttribute('data-name'); // extract fullname (pseudonym) - $item->avatar = $tweet->find('img', 0)->src; // get avatar link - $item->id = $tweet->getAttribute('data-tweet-id'); // get TweetID - $item->uri = 'https://twitter.com'.$tweet->find('a.js-permalink', 0)->getAttribute('href'); // get tweet link - $item->timestamp = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time'); // extract tweet timestamp - // processing content links - foreach($tweet->find('a') as $link) { - if($link->hasAttribute('data-expanded-url') ) { - $link->href = $link->getAttribute('data-expanded-url'); - } - $link->removeAttribute('data-expanded-url'); - $link->removeAttribute('data-query-source'); - $link->removeAttribute('rel'); - $link->removeAttribute('class'); - $link->removeAttribute('target'); - $link->removeAttribute('title'); - $link->removeAttribute('dir'); - } - $item->content = str_replace('pic.twitter.com', 'https://pic.twitter.com', strip_tags($tweet->find('p.js-tweet-text', 0)->innertext)); // extract tweet text - if(isset($param['u'])) { - if($item->username != $param['u']) { - $item->content = '♻ @' . $item->username . ': ' . $item->content; - } - } - $item->title = html_entity_decode($item->content,ENT_QUOTES,'UTF-8'); - $this->items[] = $item; - } - } - - public function getName(){ - return (!empty($this->request) ? $this->request : ''); - } - - public function getCacheDuration(){ - return 300; // 5 minutes - } -} diff --git a/bridges/TwitterBridgeTweaked.php b/bridges/TwitterBridgeTweaked.php deleted file mode 100644 index 5c1e53c8..00000000 --- a/bridges/TwitterBridgeTweaked.php +++ /dev/null @@ -1,153 +0,0 @@ -maintainer = "kraoc"; - $this->name = "Twitter Bridge Tweaked"; - $this->uri = "https://twitter.com/"; - $this->description = "(same as Twitter Bridge Extended, but with cleaned title & content)"; - $this->update = "2016-08-09"; - - $this->parameters["By keyword or hashtag"] = - '[ - { - "name" : "Keyword or #hashtag", - "identifier" : "q" - } - ]'; - - $this->parameters["By username"] = - '[ - { - "name" : "username", - "identifier" : "u" - } - ]'; - - } - - private function containsTLD($string) { - preg_match( - "/(AC($|\/)|\.AD($|\/)|\.AE($|\/)|\.AERO($|\/)|\.AF($|\/)|\.AG($|\/)|\.AI($|\/)|\.AL($|\/)|\.AM($|\/)|\.AN($|\/)|\.AO($|\/)|\.AQ($|\/)|\.AR($|\/)|\.ARPA($|\/)|\.AS($|\/)|\.ASIA($|\/)|\.AT($|\/)|\.AU($|\/)|\.AW($|\/)|\.AX($|\/)|\.AZ($|\/)|\.BA($|\/)|\.BB($|\/)|\.BD($|\/)|\.BE($|\/)|\.BF($|\/)|\.BG($|\/)|\.BH($|\/)|\.BI($|\/)|\.BIZ($|\/)|\.BJ($|\/)|\.BM($|\/)|\.BN($|\/)|\.BO($|\/)|\.BR($|\/)|\.BS($|\/)|\.BT($|\/)|\.BV($|\/)|\.BW($|\/)|\.BY($|\/)|\.BZ($|\/)|\.CA($|\/)|\.CAT($|\/)|\.CC($|\/)|\.CD($|\/)|\.CF($|\/)|\.CG($|\/)|\.CH($|\/)|\.CI($|\/)|\.CK($|\/)|\.CL($|\/)|\.CM($|\/)|\.CN($|\/)|\.CO($|\/)|\.COM($|\/)|\.COOP($|\/)|\.CR($|\/)|\.CU($|\/)|\.CV($|\/)|\.CX($|\/)|\.CY($|\/)|\.CZ($|\/)|\.DE($|\/)|\.DJ($|\/)|\.DK($|\/)|\.DM($|\/)|\.DO($|\/)|\.DZ($|\/)|\.EC($|\/)|\.EDU($|\/)|\.EE($|\/)|\.EG($|\/)|\.ER($|\/)|\.ES($|\/)|\.ET($|\/)|\.EU($|\/)|\.FI($|\/)|\.FJ($|\/)|\.FK($|\/)|\.FM($|\/)|\.FO($|\/)|\.FR($|\/)|\.GA($|\/)|\.GB($|\/)|\.GD($|\/)|\.GE($|\/)|\.GF($|\/)|\.GG($|\/)|\.GH($|\/)|\.GI($|\/)|\.GL($|\/)|\.GM($|\/)|\.GN($|\/)|\.GOV($|\/)|\.GP($|\/)|\.GQ($|\/)|\.GR($|\/)|\.GS($|\/)|\.GT($|\/)|\.GU($|\/)|\.GW($|\/)|\.GY($|\/)|\.HK($|\/)|\.HM($|\/)|\.HN($|\/)|\.HR($|\/)|\.HT($|\/)|\.HU($|\/)|\.ID($|\/)|\.IE($|\/)|\.IL($|\/)|\.IM($|\/)|\.IN($|\/)|\.INFO($|\/)|\.INT($|\/)|\.IO($|\/)|\.IQ($|\/)|\.IR($|\/)|\.IS($|\/)|\.IT($|\/)|\.JE($|\/)|\.JM($|\/)|\.JO($|\/)|\.JOBS($|\/)|\.JP($|\/)|\.KE($|\/)|\.KG($|\/)|\.KH($|\/)|\.KI($|\/)|\.KM($|\/)|\.KN($|\/)|\.KP($|\/)|\.KR($|\/)|\.KW($|\/)|\.KY($|\/)|\.KZ($|\/)|\.LA($|\/)|\.LB($|\/)|\.LC($|\/)|\.LI($|\/)|\.LK($|\/)|\.LR($|\/)|\.LS($|\/)|\.LT($|\/)|\.LU($|\/)|\.LV($|\/)|\.LY($|\/)|\.MA($|\/)|\.MC($|\/)|\.MD($|\/)|\.ME($|\/)|\.MG($|\/)|\.MH($|\/)|\.MIL($|\/)|\.MK($|\/)|\.ML($|\/)|\.MM($|\/)|\.MN($|\/)|\.MO($|\/)|\.MOBI($|\/)|\.MP($|\/)|\.MQ($|\/)|\.MR($|\/)|\.MS($|\/)|\.MT($|\/)|\.MU($|\/)|\.MUSEUM($|\/)|\.MV($|\/)|\.MW($|\/)|\.MX($|\/)|\.MY($|\/)|\.MZ($|\/)|\.NA($|\/)|\.NAME($|\/)|\.NC($|\/)|\.NE($|\/)|\.NET($|\/)|\.NF($|\/)|\.NG($|\/)|\.NI($|\/)|\.NL($|\/)|\.NO($|\/)|\.NP($|\/)|\.NR($|\/)|\.NU($|\/)|\.NZ($|\/)|\.OM($|\/)|\.ORG($|\/)|\.PA($|\/)|\.PE($|\/)|\.PF($|\/)|\.PG($|\/)|\.PH($|\/)|\.PK($|\/)|\.PL($|\/)|\.PM($|\/)|\.PN($|\/)|\.PR($|\/)|\.PRO($|\/)|\.PS($|\/)|\.PT($|\/)|\.PW($|\/)|\.PY($|\/)|\.QA($|\/)|\.RE($|\/)|\.RO($|\/)|\.RS($|\/)|\.RU($|\/)|\.RW($|\/)|\.SA($|\/)|\.SB($|\/)|\.SC($|\/)|\.SD($|\/)|\.SE($|\/)|\.SG($|\/)|\.SH($|\/)|\.SI($|\/)|\.SJ($|\/)|\.SK($|\/)|\.SL($|\/)|\.SM($|\/)|\.SN($|\/)|\.SO($|\/)|\.SR($|\/)|\.ST($|\/)|\.SU($|\/)|\.SV($|\/)|\.SY($|\/)|\.SZ($|\/)|\.TC($|\/)|\.TD($|\/)|\.TEL($|\/)|\.TF($|\/)|\.TG($|\/)|\.TH($|\/)|\.TJ($|\/)|\.TK($|\/)|\.TL($|\/)|\.TM($|\/)|\.TN($|\/)|\.TO($|\/)|\.TP($|\/)|\.TR($|\/)|\.TRAVEL($|\/)|\.TT($|\/)|\.TV($|\/)|\.TW($|\/)|\.TZ($|\/)|\.UA($|\/)|\.UG($|\/)|\.UK($|\/)|\.US($|\/)|\.UY($|\/)|\.UZ($|\/)|\.VA($|\/)|\.VC($|\/)|\.VE($|\/)|\.VG($|\/)|\.VI($|\/)|\.VN($|\/)|\.VU($|\/)|\.WF($|\/)|\.WS($|\/)|\.XN--0ZWM56D($|\/)|\.XN--11B5BS3A9AJ6G($|\/)|\.XN--80AKHBYKNJ4F($|\/)|\.XN--9T4B11YI5A($|\/)|\.XN--DEBA0AD($|\/)|\.XN--G6W251D($|\/)|\.XN--HGBK6AJ7F53BBA($|\/)|\.XN--HLCJ6AYA9ESC7A($|\/)|\.XN--JXALPDLP($|\/)|\.XN--KGBECHTV($|\/)|\.XN--ZCKZAH($|\/)|\.YE($|\/)|\.YT($|\/)|\.YU($|\/)|\.ZA($|\/)|\.ZM($|\/)|\.ZW)/i", - $string, - $M - ); - $has_tld = (count($M) > 0) ? true : false; - return $has_tld; - } - private function cleaner($url) { - $U = explode(' ', $url); - $W =array(); - foreach ($U as $k => $u) { - if (stristr($u,".")) { //only preg_match if there is a dot - if ($this->containsTLD($u) === true) { - unset($U[$k]); - return $this->cleaner( implode(' ', $U) ); - } - } - } - return implode(' ', $U); - } - - // (c) Kraoc / urlclean - // https://github.com/kraoc/Leed-market/blob/master/urlclean/urlclean.plugin.disabled.php - private function resolve_url($link) { - // fallback to crawl to real url (slowest method and unsecure to privacy) - if (function_exists('curl_init') && !ini_get('safe_mode')) { - curl_setopt($ch, CURLOPT_USERAGENT, $ua); - curl_setopt($ch, CURLOPT_URL, $link); - curl_setopt($ch, CURLOPT_HEADER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - // >>> anonimization - curl_setopt($ch, CURLOPT_COOKIESESSION, true); - curl_setopt($ch, CURLOPT_REFERER, ''); - // <<< anonimization - $ch = curl_init(); - $ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.16 (KHTML, like Gecko) Chrome/24.0.1304.0 Safari/537.16'; - $a = curl_exec($ch); - $link = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); - } - - $link = preg_replace("/[&#?]xtor=(.)+/", "", $link); // remove: xtor - $link = preg_replace("/utm_([^&#]|(&))+&*/", "", $link); // remove: utm_ - - // cleanup end of url - $link = preg_replace("/\?&/", "", $link); - if (isset($link[strlen($link) -1])){ - if ($link[strlen($link) -1] == '?') - $link = substr($link, 0, strlen($link) -1); - } - - return $link; - } - - public function collectData(array $param){ - $html = ''; - if (isset($param['q'])) { /* keyword search mode */ - $html = $this->file_get_html('https://twitter.com/search?q='.urlencode($param['q']).'&f=tweets') or $this->returnError('No results for this query.', 404); - } - elseif (isset($param['u'])) { /* user timeline mode */ - $html = $this->file_get_html('https://twitter.com/'.urlencode($param['u']).'/with_replies') or $this->returnError('Requested username can\'t be found.', 404); - } - else { - $this->returnError('You must specify a keyword (?q=...) or a Twitter username (?u=...).', 400); - } - - foreach($html->find('div.js-stream-tweet') as $tweet) { - $item = new \Item(); - // extract username and sanitize - $item->username = $tweet->getAttribute('data-screen-name'); - // extract fullname (pseudonym) - $item->fullname = $tweet->getAttribute('data-name'); - // get avatar link - $item->avatar = $tweet->find('img', 0)->src; - // get TweetID - $item->id = $tweet->getAttribute('data-tweet-id'); - // get tweet link - $item->uri = 'https://twitter.com'.$tweet->find('a.js-permalink', 0)->getAttribute('href'); - // extract tweet timestamp - $item->timestamp = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time'); - // extract plaintext - $item->content_simple = str_replace('href="/', 'href="https://twitter.com/', html_entity_decode(strip_tags($tweet->find('p.js-tweet-text', 0)->innertext, ''))); - - // processing content links - foreach($tweet->find('a') as $link) { - if($link->hasAttribute('data-expanded-url') ) { - $link->href = $link->getAttribute('data-expanded-url'); - } - $link->removeAttribute('data-expanded-url'); - $link->removeAttribute('data-query-source'); - $link->removeAttribute('rel'); - $link->removeAttribute('class'); - $link->removeAttribute('target'); - $link->removeAttribute('title'); - } - - // get tweet text - $item->content = 'avatar'.$item->username.' '.$item->fullname.'
'.str_replace('href="/', 'href="https://twitter.com/', $tweet->find('p.js-tweet-text', 0)->innertext).'
'; - // generate the title -// $item->title = $item->fullname . ' (@'. $item->username . ') | ' . $item->content_simple; - $item->title = $item->content_simple; - $item->title = preg_replace('|https?://www\.[a-z\.0-9]+|i', '', $item->title); // remove http(s) links - $item->title = preg_replace('|www\.[a-z\.0-9]+|i', '', $item->title); // remove www. links - $item->title = $this->cleaner($item->title); // remove all remaining links - $item->title = trim($item->title); // remove extra spaces at beginning and end - - // convert all content links to real ones - $regex = "/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/"; - $item->content = preg_replace_callback($regex, function($url) { - // do stuff with $url[0] here - return $this->resolve_url($url[0]); - }, $item->content); - - // put out - $this->items[] = $item; - } - } - - public function getCacheDuration(){ - return 300; // 5 minutes - } -}