Don't write History for link import

With large imports it has a large impact on performances and isn't really useful.

Instead, write an IMPORT event, which let client using the history service resync its DB.

-> 15k link import done in 6 seconds.

Fixes #985

application/History.php

@@ -16,6 +16,7 @@
  *   - UPDATED: link updated
  *   - DELETED: link deleted
  *   - SETTINGS: the settings have been updated through the UI.
+ *   - IMPORT: bulk links import
  *
  * Note: new events are put at the beginning of the file and history array.
  */
@@ -41,6 +42,11 @@ class History
      */
     const SETTINGS = 'SETTINGS';
 
+    /**
+     * @var string Action key: a bulk import has been processed.
+     */
+    const IMPORT = 'IMPORT';
+
     /**
      * @var string History file path.
      */
@@ -121,6 +127,16 @@ public function updateSettings()
         $this->addEvent(self::SETTINGS);
     }
 
+    /**
+     * Add Event: bulk import.
+     *
+     * Note: we don't store links add/update one by one since it can have a huge impact on performances.
+     */
+    public function importLinks()
+    {
+        $this->addEvent(self::IMPORT);
+    }
+
     /**
      * Save a new event and write it in the history file.
      *

application/NetscapeBookmarkUtils.php

@@ -66,6 +66,7 @@ public static function filterAndFormat($linkDb, $selection, $prependNoteUrl, $in
      * @param int    $importCount    how many links were imported
      * @param int    $overwriteCount how many links were overwritten
      * @param int    $skipCount      how many links were skipped
+     * @param int    $duration       how many seconds did the import take
      *
      * @return string Summary of the bookmark import status
      */
@@ -74,14 +75,16 @@ private static function importStatus(
         $filesize,
         $importCount=0,
         $overwriteCount=0,
-        $skipCount=0
+        $skipCount=0,
+        $duration=0
     )
     {
         $status = 'File '.$filename.' ('.$filesize.' bytes) ';
         if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) {
             $status .= 'has an unknown file format. Nothing was imported.';
         } else {
-            $status .= 'was successfully processed: '.$importCount.' links imported, ';
+            $status .= 'was successfully processed in '. $duration .' seconds: ';
+            $status .= $importCount.' links imported, ';
             $status .= $overwriteCount.' links overwritten, ';
             $status .= $skipCount.' links skipped.';
         }
@@ -101,6 +104,7 @@ private static function importStatus(
      */
     public static function import($post, $files, $linkDb, $conf, $history)
     {
+        $start = time();
         $filename = $files['filetoupload']['name'];
         $filesize = $files['filetoupload']['size'];
         $data = file_get_contents($files['filetoupload']['tmp_name']);
@@ -184,7 +188,6 @@ public static function import($post, $files, $linkDb, $conf, $history)
                 $linkDb[$existingLink['id']] = $newLink;
                 $importCount++;
                 $overwriteCount++;
-                $history->updateLink($newLink);
                 continue;
             }
 
@@ -196,16 +199,19 @@ public static function import($post, $files, $linkDb, $conf, $history)
             $newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']);
             $linkDb[$newLink['id']] = $newLink;
             $importCount++;
-            $history->addLink($newLink);
         }
 
         $linkDb->save($conf->get('resource.page_cache'));
+        $history->importLinks();
+
+        $duration = time() - $start;
         return self::importStatus(
             $filename,
             $filesize,
             $importCount,
             $overwriteCount,
-            $skipCount
+            $skipCount,
+            $duration
         );
     }
 }

tests/NetscapeBookmarkUtils/BookmarkImportTest.php

@@ -132,8 +132,8 @@ public function testImportNoDoctype()
     public function testImportInternetExplorerEncoding()
     {
         $files = file2array('internet_explorer_encoding.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File internet_explorer_encoding.htm (356 bytes) was successfully processed:'
+            'File internet_explorer_encoding.htm (356 bytes) was successfully processed in %d seconds:'
             .' 1 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -161,8 +161,8 @@ public function testImportInternetExplorerEncoding()
     public function testImportNested()
     {
         $files = file2array('netscape_nested.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_nested.htm (1337 bytes) was successfully processed:'
+            'File netscape_nested.htm (1337 bytes) was successfully processed in %d seconds:'
             .' 8 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -283,8 +283,8 @@ public function testImportNested()
     public function testImportDefaultPrivacyNoPost()
     {
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -328,8 +328,8 @@ public function testImportKeepPrivacy()
     {
         $post = array('privacy' => 'default');
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -372,8 +372,8 @@ public function testImportAsPublic()
     {
         $post = array('privacy' => 'public');
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -396,8 +396,8 @@ public function testImportAsPrivate()
     {
         $post = array('privacy' => 'private');
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -422,8 +422,8 @@ public function testOverwriteAsPublic()
 
         // import links as private
         $post = array('privacy' => 'private');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -442,8 +442,8 @@ public function testOverwriteAsPublic()
             'privacy' => 'public',
             'overwrite' => 'true'
         );
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 2 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -468,8 +468,8 @@ public function testOverwriteAsPrivate()
 
         // import links as public
         $post = array('privacy' => 'public');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -489,8 +489,8 @@ public function testOverwriteAsPrivate()
             'privacy' => 'private',
             'overwrite' => 'true'
         );
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 2 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -513,8 +513,8 @@ public function testSkipOverwrite()
     {
         $post = array('privacy' => 'public');
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -523,8 +523,8 @@ public function testSkipOverwrite()
 
         // re-import as private, DO NOT enable overwriting
         $post = array('privacy' => 'private');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 0 links imported, 0 links overwritten, 2 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -542,8 +542,8 @@ public function testSetDefaultTags()
             'default_tags' => 'tag1,tag2 tag3'
         );
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -569,8 +569,8 @@ public function testSanitizeDefaultTags()
             'default_tags' => 'tag1&,tag2 "tag3"'
         );
         $files = file2array('netscape_basic.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File netscape_basic.htm (482 bytes) was successfully processed:'
+            'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
             .' 2 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -594,8 +594,8 @@ public function testSanitizeDefaultTags()
     public function testImportSameDate()
     {
         $files = file2array('same_date.htm');
-        $this->assertEquals(
+        $this->assertStringMatchesFormat(
-            'File same_date.htm (453 bytes) was successfully processed:'
+            'File same_date.htm (453 bytes) was successfully processed in %d seconds:'
             .' 3 links imported, 0 links overwritten, 0 links skipped.',
             NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->conf, $this->history)
         );
@@ -622,24 +622,19 @@ public function testImportCreateUpdateHistory()
             'overwrite' => 'true',
         ];
         $files = file2array('netscape_basic.htm');
-        $nbLinks = 2;
         NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
         $history = $this->history->getHistory();
-        $this->assertEquals($nbLinks, count($history));
+        $this->assertEquals(1, count($history));
-        foreach ($history as $value) {
+        $this->assertEquals(History::IMPORT, $history[0]['event']);
-            $this->assertEquals(History::CREATED, $value['event']);
+        $this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
-            $this->assertTrue(new DateTime('-5 seconds') < $value['datetime']);
-            $this->assertTrue(is_int($value['id']));
-        }
 
         // re-import as private, enable overwriting
         NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
         $history = $this->history->getHistory();
-        $this->assertEquals($nbLinks * 2, count($history));
+        $this->assertEquals(2, count($history));
-        for ($i = 0 ; $i < $nbLinks ; $i++) {
+        $this->assertEquals(History::IMPORT, $history[0]['event']);
-            $this->assertEquals(History::UPDATED, $history[$i]['event']);
+        $this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
-            $this->assertTrue(new DateTime('-5 seconds') < $history[$i]['datetime']);
+        $this->assertEquals(History::IMPORT, $history[1]['event']);
-            $this->assertTrue(is_int($history[$i]['id']));
+        $this->assertTrue(new DateTime('-5 seconds') < $history[1]['datetime']);
-        }
     }
 }