Fixes #531 - Title retrieving is failing with multiple use case

see https://github.com/shaarli/Shaarli/issues/531 for details

application/HttpUtils.php

@@ -27,7 +27,9 @@
 function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 {
     $urlObj = new Url($url);
-    if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
+    $cleanUrl = $urlObj->indToAscii();
+
+    if (! filter_var($cleanUrl, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
         return array(array(0 => 'Invalid HTTP Url'), false);
     }
 
@@ -35,22 +37,27 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
         'http' => array(
             'method' => 'GET',
             'timeout' => $timeout,
-            'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)'
+            'user_agent' => 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
-                         .' Gecko/20100101 Firefox/23.0',
+                         .' Gecko/20100101 Firefox/45.0',
-            'request_fulluri' => true,
+            'accept_language' => substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3',
         )
     );
 
-    $context = stream_context_create($options);
     stream_context_set_default($options);
+    list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+    if (! $headers || strpos($headers[0], '200 OK') === false) {
+        $options['http']['request_fulluri'] = true;
+        stream_context_set_default($options);
+        list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+    }
 
-    list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup());
     if (! $headers || strpos($headers[0], '200 OK') === false) {
         return array($headers, false);
     }
 
     try {
         // TODO: catch Exception in calling code (thumbnailer)
+        $context = stream_context_create($options);
         $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
     } catch (Exception $exc) {
         return array(array(0 => 'HTTP Error'), $exc->getMessage());
@@ -60,16 +67,19 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 }
 
 /**
- * Retrieve HTTP headers, following n redirections (temporary and permanent).
+ * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
  *
  * @param string $url              initial URL to reach.
  * @param int    $redirectionLimit max redirection follow..
  *
- * @return array
+ * @return array HTTP headers, or false if it failed.
  */
 function get_redirected_headers($url, $redirectionLimit = 3)
 {
     $headers = get_headers($url, 1);
+    if (!empty($headers['location']) && empty($headers['Location'])) {
+        $headers['Location'] = $headers['location'];
+    }
 
     // Headers found, redirection found, and limit not reached.
     if ($redirectionLimit-- > 0
@@ -79,6 +89,7 @@ function get_redirected_headers($url, $redirectionLimit = 3)
 
         $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
         if ($redirection != $url) {
+            $redirection = getAbsoluteUrl($url, $redirection);
             return get_redirected_headers($redirection, $redirectionLimit);
         }
     }
@@ -86,6 +97,35 @@ function get_redirected_headers($url, $redirectionLimit = 3)
     return array($headers, $url);
 }
 
+/**
+ * Get an absolute URL from a complete one, and another absolute/relative URL.
+ *
+ * @param string $originalUrl The original complete URL.
+ * @param string $newUrl      The new one, absolute or relative.
+ *
+ * @return string Final URL:
+ *   - $newUrl if it was already an absolute URL.
+ *   - if it was relative, absolute URL from $originalUrl path.
+ */
+function getAbsoluteUrl($originalUrl, $newUrl)
+{
+    $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
+    // Already an absolute URL.
+    if (!empty($newScheme)) {
+        return $newUrl;
+    }
+
+    $parts = parse_url($originalUrl);
+    $final = $parts['scheme'] .'://'. $parts['host'];
+    $final .= (!empty($parts['port'])) ? $parts['port'] : '';
+    $final .= '/';
+    if ($newUrl[0] != '/') {
+        $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
+    }
+    $final .= ltrim($newUrl, '/');
+    return $final;
+}
+
 /**
  * Returns the server's base URL: scheme://domain.tld[:port]
  *

application/LinkUtils.php

@@ -9,7 +9,7 @@
  */
 function html_extract_title($html)
 {
-    if (preg_match('!<title>(.*?)</title>!is', $html, $matches)) {
+    if (preg_match('!<title.*?>(.*?)</title>!is', $html, $matches)) {
         return trim(str_replace("\n", '', $matches[1]));
     }
     return false;
@@ -70,7 +70,7 @@ function headers_extract_charset($headers)
 function html_extract_charset($html)
 {
     // Get encoding specified in HTML header.
-    preg_match('#<meta .*charset="?([^">/]+)"? */?>#Usi', $html, $enc);
+    preg_match('#<meta .*charset=["\']?([^";\'>/]+)["\']? */?>#Usi', $html, $enc);
     if (!empty($enc[1])) {
         return strtolower($enc[1]);
     }

application/Url.php

@@ -62,7 +62,21 @@ function add_trailing_slash($url)
 {
     return $url . (!endsWith($url, '/') ? '/' : '');
 }
+/**
+ * Converts an URL with an IDN host to a ASCII one.
+ *
+ * @param string $url Input URL.
+ *
+ * @return string converted URL.
+ */
+function url_with_idn_to_ascii($url)
+{
+    $parts = parse_url($url);
+    $parts['host'] = idn_to_ascii($parts['host']);
 
+    $httpUrl = new \http\Url($parts);
+    return $httpUrl->toString();
+}
 /**
  * URL representation and cleanup utilities
  *
@@ -220,6 +234,22 @@ public function cleanup()
         return $this->toString();
     }
 
+    /**
+     * Converts an URL with an International Domain Name host to a ASCII one.
+     * This requires PHP-intl. If it's not available, just returns this->cleanup().
+     *
+     * @return string converted cleaned up URL.
+     */
+    public function indToAscii()
+    {
+        $out = $this->cleanup();
+        if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) {
+            return $out;
+        }
+        $asciiHost = idn_to_ascii($this->parts['host']);
+        return str_replace($this->parts['host'], $asciiHost, $out);
+    }
+
     /**
      * Get URL scheme.
      *
@@ -232,6 +262,18 @@ public function getScheme() {
         return $this->parts['scheme'];
     }
 
+    /**
+     * Get URL host.
+     *
+     * @return string the URL host or false if none is provided.
+     */
+    public function getHost() {
+        if (empty($this->parts['host'])) {
+            return false;
+        }
+        return $this->parts['host'];
+    }
+
     /**
      * Test if the Url is an HTTP one.
      *

index.php

@@ -1516,7 +1516,7 @@ function renderPage()
 
     // -------- User want to post a new link: Display link edit form.
     if (isset($_GET['post'])) {
-        $url = cleanup_url(escape($_GET['post']));
+        $url = cleanup_url($_GET['post']);
 
         $link_is_new = false;
         // Check if URL is not already in database (in this case, we will edit the existing link)
@@ -1541,8 +1541,8 @@ function renderPage()
                     // Extract title.
                     $title = html_extract_title($content);
                     // Re-encode title in utf-8 if necessary.
-                    if (! empty($title) && $charset != 'utf-8') {
+                    if (! empty($title) && strtolower($charset) != 'utf-8') {
-                        $title = mb_convert_encoding($title, $charset, 'utf-8');
+                        $title = mb_convert_encoding($title, 'utf-8', $charset);
                     }
                 }
             }
@@ -1551,6 +1551,8 @@ function renderPage()
                 $url = '?' . smallHash($linkdate);
                 $title = 'Note: ';
             }
+            $url = escape($url);
+            $title = escape($title);
 
             $link = array(
                 'linkdate' => $linkdate,

tests/HttpUtils/GetHttpUrlTest.php

@@ -35,4 +35,31 @@ public function testGetInvalidRemoteUrl()
         $this->assertFalse($headers);
         $this->assertFalse($content);
     }
+
+    /**
+     * Test getAbsoluteUrl with relative target URL.
+     */
+    public function testGetAbsoluteUrlWithRelative()
+    {
+        $origin = 'http://non.existent/blabla/?test';
+        $target = '/stuff.php';
+
+        $expected = 'http://non.existent/stuff.php';
+        $this->assertEquals($expected, getAbsoluteUrl($origin, $target));
+
+        $target = 'stuff.php';
+        $expected = 'http://non.existent/blabla/stuff.php';
+        $this->assertEquals($expected, getAbsoluteUrl($origin, $target));
+    }
+
+    /**
+     * Test getAbsoluteUrl with absolute target URL.
+     */
+    public function testGetAbsoluteUrlWithAbsolute()
+    {
+        $origin = 'http://non.existent/blabla/?test';
+        $target = 'http://other.url/stuff.php';
+
+        $this->assertEquals($target, getAbsoluteUrl($origin, $target));
+    }
 }

tests/Url/UrlTest.php

@@ -181,4 +181,19 @@ function testUrlIsNotHttp()
         $url = new Url('ftp://save.tld/mysave');
         $this->assertFalse($url->isHttp());
     }
+
+    /**
+     * Test IndToAscii.
+     */
+    function testIndToAscii()
+    {
+        $ind = 'http://www.académie-française.fr/';
+        $expected = 'http://www.xn--acadmie-franaise-npb1a.fr/';
+        $url = new Url($ind);
+        $this->assertEquals($expected, $url->indToAscii());
+
+        $notInd = 'http://www.academie-francaise.fr/';
+        $url = new Url($notInd);
+        $this->assertEquals($notInd, $url->indToAscii());
+    }
 }