From ce7b0b6480aa854ee6893f5c889277b0e3b13efc Mon Sep 17 00:00:00 2001 From: ArthurHoaro Date: Wed, 6 Apr 2016 22:00:52 +0200 Subject: [PATCH] Fixes #531 - Title retrieving is failing with multiple use case see https://github.com/shaarli/Shaarli/issues/531 for details --- application/HttpUtils.php | 60 +++++++++++++++++++++++++----- application/LinkUtils.php | 6 +-- application/Url.php | 42 +++++++++++++++++++++ index.php | 8 ++-- tests/HttpUtils/GetHttpUrlTest.php | 27 ++++++++++++++ tests/Url/UrlTest.php | 15 ++++++++ 6 files changed, 142 insertions(+), 16 deletions(-) diff --git a/application/HttpUtils.php b/application/HttpUtils.php index af7cb37..0e1ce87 100644 --- a/application/HttpUtils.php +++ b/application/HttpUtils.php @@ -27,7 +27,9 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) { $urlObj = new Url($url); - if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { + $cleanUrl = $urlObj->indToAscii(); + + if (! filter_var($cleanUrl, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { return array(array(0 => 'Invalid HTTP Url'), false); } @@ -35,22 +37,27 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) 'http' => array( 'method' => 'GET', 'timeout' => $timeout, - 'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)' - .' Gecko/20100101 Firefox/23.0', - 'request_fulluri' => true, + 'user_agent' => 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)' + .' Gecko/20100101 Firefox/45.0', + 'accept_language' => substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3', ) ); - $context = stream_context_create($options); stream_context_set_default($options); + list($headers, $finalUrl) = get_redirected_headers($cleanUrl); + if (! $headers || strpos($headers[0], '200 OK') === false) { + $options['http']['request_fulluri'] = true; + stream_context_set_default($options); + list($headers, $finalUrl) = get_redirected_headers($cleanUrl); + } - list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup()); if (! $headers || strpos($headers[0], '200 OK') === false) { return array($headers, false); } try { // TODO: catch Exception in calling code (thumbnailer) + $context = stream_context_create($options); $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes); } catch (Exception $exc) { return array(array(0 => 'HTTP Error'), $exc->getMessage()); @@ -60,16 +67,19 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) } /** - * Retrieve HTTP headers, following n redirections (temporary and permanent). + * Retrieve HTTP headers, following n redirections (temporary and permanent ones). * - * @param string $url initial URL to reach. - * @param int $redirectionLimit max redirection follow.. + * @param string $url initial URL to reach. + * @param int $redirectionLimit max redirection follow.. * - * @return array + * @return array HTTP headers, or false if it failed. */ function get_redirected_headers($url, $redirectionLimit = 3) { $headers = get_headers($url, 1); + if (!empty($headers['location']) && empty($headers['Location'])) { + $headers['Location'] = $headers['location']; + } // Headers found, redirection found, and limit not reached. if ($redirectionLimit-- > 0 @@ -79,6 +89,7 @@ function get_redirected_headers($url, $redirectionLimit = 3) $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location']; if ($redirection != $url) { + $redirection = getAbsoluteUrl($url, $redirection); return get_redirected_headers($redirection, $redirectionLimit); } } @@ -86,6 +97,35 @@ function get_redirected_headers($url, $redirectionLimit = 3) return array($headers, $url); } +/** + * Get an absolute URL from a complete one, and another absolute/relative URL. + * + * @param string $originalUrl The original complete URL. + * @param string $newUrl The new one, absolute or relative. + * + * @return string Final URL: + * - $newUrl if it was already an absolute URL. + * - if it was relative, absolute URL from $originalUrl path. + */ +function getAbsoluteUrl($originalUrl, $newUrl) +{ + $newScheme = parse_url($newUrl, PHP_URL_SCHEME); + // Already an absolute URL. + if (!empty($newScheme)) { + return $newUrl; + } + + $parts = parse_url($originalUrl); + $final = $parts['scheme'] .'://'. $parts['host']; + $final .= (!empty($parts['port'])) ? $parts['port'] : ''; + $final .= '/'; + if ($newUrl[0] != '/') { + $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/')); + } + $final .= ltrim($newUrl, '/'); + return $final; +} + /** * Returns the server's base URL: scheme://domain.tld[:port] * diff --git a/application/LinkUtils.php b/application/LinkUtils.php index d8dc8b5..2df76ba 100644 --- a/application/LinkUtils.php +++ b/application/LinkUtils.php @@ -9,8 +9,8 @@ */ function html_extract_title($html) { - if (preg_match('!(.*?)!is', $html, $matches)) { - return trim(str_replace("\n", ' ', $matches[1])); + if (preg_match('!(.*?)!is', $html, $matches)) { + return trim(str_replace("\n", '', $matches[1])); } return false; } @@ -70,7 +70,7 @@ function headers_extract_charset($headers) function html_extract_charset($html) { // Get encoding specified in HTML header. - preg_match('#/]+)"? */?>#Usi', $html, $enc); + preg_match('#/]+)["\']? */?>#Usi', $html, $enc); if (!empty($enc[1])) { return strtolower($enc[1]); } diff --git a/application/Url.php b/application/Url.php index af38c4d..61a30a7 100644 --- a/application/Url.php +++ b/application/Url.php @@ -62,7 +62,21 @@ function add_trailing_slash($url) { return $url . (!endsWith($url, '/') ? '/' : ''); } +/** + * Converts an URL with an IDN host to a ASCII one. + * + * @param string $url Input URL. + * + * @return string converted URL. + */ +function url_with_idn_to_ascii($url) +{ + $parts = parse_url($url); + $parts['host'] = idn_to_ascii($parts['host']); + $httpUrl = new \http\Url($parts); + return $httpUrl->toString(); +} /** * URL representation and cleanup utilities * @@ -220,6 +234,22 @@ class Url return $this->toString(); } + /** + * Converts an URL with an International Domain Name host to a ASCII one. + * This requires PHP-intl. If it's not available, just returns this->cleanup(). + * + * @return string converted cleaned up URL. + */ + public function indToAscii() + { + $out = $this->cleanup(); + if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) { + return $out; + } + $asciiHost = idn_to_ascii($this->parts['host']); + return str_replace($this->parts['host'], $asciiHost, $out); + } + /** * Get URL scheme. * @@ -232,6 +262,18 @@ class Url return $this->parts['scheme']; } + /** + * Get URL host. + * + * @return string the URL host or false if none is provided. + */ + public function getHost() { + if (empty($this->parts['host'])) { + return false; + } + return $this->parts['host']; + } + /** * Test if the Url is an HTTP one. * diff --git a/index.php b/index.php index dfc00fb..41a42cf 100644 --- a/index.php +++ b/index.php @@ -1516,7 +1516,7 @@ function renderPage() // -------- User want to post a new link: Display link edit form. if (isset($_GET['post'])) { - $url = cleanup_url(escape($_GET['post'])); + $url = cleanup_url($_GET['post']); $link_is_new = false; // Check if URL is not already in database (in this case, we will edit the existing link) @@ -1541,8 +1541,8 @@ function renderPage() // Extract title. $title = html_extract_title($content); // Re-encode title in utf-8 if necessary. - if (! empty($title) && $charset != 'utf-8') { - $title = mb_convert_encoding($title, $charset, 'utf-8'); + if (! empty($title) && strtolower($charset) != 'utf-8') { + $title = mb_convert_encoding($title, 'utf-8', $charset); } } } @@ -1551,6 +1551,8 @@ function renderPage() $url = '?' . smallHash($linkdate); $title = 'Note: '; } + $url = escape($url); + $title = escape($title); $link = array( 'linkdate' => $linkdate, diff --git a/tests/HttpUtils/GetHttpUrlTest.php b/tests/HttpUtils/GetHttpUrlTest.php index fd29350..ea53de5 100644 --- a/tests/HttpUtils/GetHttpUrlTest.php +++ b/tests/HttpUtils/GetHttpUrlTest.php @@ -35,4 +35,31 @@ class GetHttpUrlTest extends PHPUnit_Framework_TestCase $this->assertFalse($headers); $this->assertFalse($content); } + + /** + * Test getAbsoluteUrl with relative target URL. + */ + public function testGetAbsoluteUrlWithRelative() + { + $origin = 'http://non.existent/blabla/?test'; + $target = '/stuff.php'; + + $expected = 'http://non.existent/stuff.php'; + $this->assertEquals($expected, getAbsoluteUrl($origin, $target)); + + $target = 'stuff.php'; + $expected = 'http://non.existent/blabla/stuff.php'; + $this->assertEquals($expected, getAbsoluteUrl($origin, $target)); + } + + /** + * Test getAbsoluteUrl with absolute target URL. + */ + public function testGetAbsoluteUrlWithAbsolute() + { + $origin = 'http://non.existent/blabla/?test'; + $target = 'http://other.url/stuff.php'; + + $this->assertEquals($target, getAbsoluteUrl($origin, $target)); + } } diff --git a/tests/Url/UrlTest.php b/tests/Url/UrlTest.php index a64a73e..5fdc861 100644 --- a/tests/Url/UrlTest.php +++ b/tests/Url/UrlTest.php @@ -181,4 +181,19 @@ class UrlTest extends PHPUnit_Framework_TestCase $url = new Url('ftp://save.tld/mysave'); $this->assertFalse($url->isHttp()); } + + /** + * Test IndToAscii. + */ + function testIndToAscii() + { + $ind = 'http://www.académie-française.fr/'; + $expected = 'http://www.xn--acadmie-franaise-npb1a.fr/'; + $url = new Url($ind); + $this->assertEquals($expected, $url->indToAscii()); + + $notInd = 'http://www.academie-francaise.fr/'; + $url = new Url($notInd); + $this->assertEquals($notInd, $url->indToAscii()); + } }