Merge pull request #455 from ArthurHoaro/improved-search-454

Improved search: combine AND, exact terms and exclude search.

application/LinkFilter.php

@@ -120,7 +120,9 @@ private function filterSmallHash($smallHash)
      *
      * Searches:
      *  - in the URLs, title and description;
-     *  - are case-insensitive.
+     *  - are case-insensitive;
+     *  - terms surrounded by quotes " are exact terms search.
+     *  - terms starting with a dash - are excluded (except exact terms).
      *
      * Example:
      *    print_r($mydb->filterFulltext('hollandais'));
@@ -136,19 +138,30 @@ private function filterSmallHash($smallHash)
      */
     private function filterFulltext($searchterms, $privateonly = false)
     {
+        $filtered = array();
         $search = mb_convert_case(html_entity_decode($searchterms), MB_CASE_LOWER, 'UTF-8');
-        $explodedSearch = explode(' ', trim($search));
+        $exactRegex = '/"([^"]+)"/';
-        $keys = array('title', 'description', 'url', 'tags');
+        // Retrieve exact search terms.
-        $found = true;
+        preg_match_all($exactRegex, $search, $exactSearch);
-        $searchExactPhrase = false;
+        $exactSearch = array_values(array_filter($exactSearch[1]));
 
-        // Check if we're using double-quotes to search for the exact string
+        // Remove exact search terms to get AND terms search.
-        if ($search[0] == '"' && $search[strlen($search) - 1] == '"') {
+        $explodedSearchAnd = explode(' ', trim(preg_replace($exactRegex, '', $search)));
-            $searchExactPhrase = true;
+        $explodedSearchAnd = array_values(array_filter($explodedSearchAnd));
-            
+
-            // Remove the double-quotes as they are not what we search for
+        // Filter excluding terms and update andSearch.
-            $search = substr($search, 1, -1);
+        $excludeSearch = array();
+        $andSearch = array();
+        foreach ($explodedSearchAnd as $needle) {
+            if ($needle[0] == '-' && strlen($needle) > 1) {
+                $excludeSearch[] = substr($needle, 1);
+            } else {
+                $andSearch[] = $needle;
+            }
         }
+
+        $keys = array('title', 'description', 'url', 'tags');
+
         // Iterate over every stored link.
         foreach ($this->links as $link) {
 
@@ -157,35 +170,32 @@ private function filterFulltext($searchterms, $privateonly = false)
                 continue;
             }
 
-            // Iterate over searchable link fields.
+            // Concatenate link fields to search across fields.
+            // Adds a '\' separator for exact search terms.
+            $content = '';
             foreach ($keys as $key) {
-                // Be optimistic
+                $content .= mb_convert_case($link[$key], MB_CASE_LOWER, 'UTF-8') . '\\';
-                $found = true;
-                
-                // FIXME: Find a better word for where you're searching in 
-                $haystack = mb_convert_case($link[$key], MB_CASE_LOWER, 'UTF-8');
-
-                // When searching for the phrase, check if it's in the haystack...
-                if ( $searchExactPhrase && strpos($haystack, $search) !== false) {
-                    break;
-                }
-                else {
-                    // Iterate over keywords, if keyword is not found,
-                    // no need to check for the others. We want all or nothing.
-                    foreach($explodedSearch as $keyword) {
-                         if(strpos($haystack, $keyword) === false) {
-                           $found = false;
-                           break;
-                      }
-                    }
-                }
-                
-                // One of the fields of the link matches, no need to check the other.
-                if ($found) {
-                    break;
-                }
             }
-            
+
+            // Be optimistic
+            $found = true;
+
+            // First, we look for exact term search
+            for ($i = 0; $i < count($exactSearch) && $found; $i++) {
+                $found = strpos($content, $exactSearch[$i]) !== false;
+            }
+
+            // Iterate over keywords, if keyword is not found,
+            // no need to check for the others. We want all or nothing.
+            for ($i = 0; $i < count($andSearch) && $found; $i++) {
+                $found = strpos($content, $andSearch[$i]) !== false;
+            }
+
+            // Exclude terms.
+            for ($i = 0; $i < count($excludeSearch) && $found; $i++) {
+                $found = strpos($content, $excludeSearch[$i]) === false;
+            }
+
             if ($found) {
                 $filtered[$link['linkdate']] = $link;
             }

tests/LinkDBTest.php

@@ -278,6 +278,7 @@ public function testAllTags()
                 'stallman' => 1,
                 'free' => 1,
                 '-exclude' => 1,
+                'stuff' => 2,
             ),
             self::$publicLinkDB->allTags()
         );
@@ -297,6 +298,7 @@ public function testAllTags()
                 'w3c' => 1,
                 'css' => 1,
                 'Mercurial' => 1,
+                'stuff' => 2,
                 '-exclude' => 1,
                 '.hidden' => 1,
             ),

tests/LinkFilterTest.php

@@ -27,7 +27,7 @@ public static function setUpBeforeClass()
     public function testFilter()
     {
         $this->assertEquals(
-            6,
+            7,
             count(self::$linkFilter->filter('', ''))
         );
 
@@ -164,6 +164,17 @@ public function testFilterUnknownSmallHash()
         );
     }
 
+    /**
+     * Full-text search - no result found.
+     */
+    public function testFilterFullTextNoResult()
+    {
+        $this->assertEquals(
+            0,
+            count(self::$linkFilter->filter(LinkFilter::$FILTER_TEXT, 'azertyuiop'))
+        );
+    }
+
     /**
      * Full-text search - result from a link's URL
      */
@@ -222,7 +233,7 @@ public function testFilterFullTextDescription()
         );
         
         $this->assertEquals(
-            2,
+            3,
             count(self::$linkFilter->filter(LinkFilter::$FILTER_TEXT, '"free software"'))
         );        
     }
@@ -250,11 +261,71 @@ public function testFilterFullTextTags()
     public function testFilterFullTextMixed()
     {
         $this->assertEquals(
-            2,
+            3,
             count(self::$linkFilter->filter(LinkFilter::$FILTER_TEXT, 'free software'))
         );
     }
 
+    /**
+     * Full-text search - test exclusion with '-'.
+     */
+    public function testExcludeSearch()
+    {
+        $this->assertEquals(
+            1,
+            count(self::$linkFilter->filter(LinkFilter::$FILTER_TEXT, 'free -gnu'))
+        );
+
+        $this->assertEquals(
+            6,
+            count(self::$linkFilter->filter(LinkFilter::$FILTER_TEXT, '-revolution'))
+        );
+    }
+
+    /**
+     * Full-text search - test AND, exact terms and exclusion combined, across fields.
+     */
+    public function testMultiSearch()
+    {
+        $this->assertEquals(
+            2,
+            count(self::$linkFilter->filter(
+                LinkFilter::$FILTER_TEXT,
+                '"Free Software " stallman "read this" @website stuff'
+            ))
+        );
+
+        $this->assertEquals(
+            1,
+            count(self::$linkFilter->filter(
+                LinkFilter::$FILTER_TEXT,
+                '"free software " stallman "read this" -beard @website stuff'
+            ))
+        );
+    }
+
+    /**
+     * Full-text search - make sure that exact search won't work across fields.
+     */
+    public function testSearchExactTermMultiFieldsKo()
+    {
+        $this->assertEquals(
+            0,
+            count(self::$linkFilter->filter(
+                LinkFilter::$FILTER_TEXT,
+                '"designer naming"'
+            ))
+        );
+
+        $this->assertEquals(
+            0,
+            count(self::$linkFilter->filter(
+                LinkFilter::$FILTER_TEXT,
+                '"designernaming"'
+            ))
+        );
+    }
+
     /**
      * Tag search with exclusion.
      */
@@ -266,7 +337,7 @@ public function testTagFilterWithExclusion()
         );
 
         $this->assertEquals(
-            5,
+            6,
             count(self::$linkFilter->filter(LinkFilter::$FILTER_TAG, '-free'))
         );
     }

tests/utils/ReferenceLinkDB.php

@@ -14,12 +14,21 @@ class ReferenceLinkDB
     function __construct()
     {
         $this->addLink(
-            'Free as in Freedom 2.0',
+            'Free as in Freedom 2.0 @website',
             'https://static.fsf.org/nosvn/faif-2.0.pdf',
-            'Richard Stallman and the Free Software Revolution',
+            'Richard Stallman and the Free Software Revolution. Read this.',
             0,
             '20150310_114633',
-            'free gnu software stallman -exclude'
+            'free gnu software stallman -exclude stuff'
+        );
+
+        $this->addLink(
+            'Link title: @website',
+            'local',
+            'Stallman has a beard and is part of the Free Software Foundation (or not). Seriously, read this.',
+            0,
+            '20150310_114651',
+            'stuff'
         );
 
         $this->addLink(