Don't write History for link import
With large imports it has a large impact on performances and isn't really useful. Instead, write an IMPORT event, which let client using the history service resync its DB. -> 15k link import done in 6 seconds. Fixes #985
This commit is contained in:
ArthurHoaro
parent
78865393a6
commit
66e74d50d3
3 changed files with 65 additions and 48 deletions
|
|
@ -16,6 +16,7 @@
|
|||
* - UPDATED: link updated
|
||||
* - DELETED: link deleted
|
||||
* - SETTINGS: the settings have been updated through the UI.
|
||||
* - IMPORT: bulk links import
|
||||
*
|
||||
* Note: new events are put at the beginning of the file and history array.
|
||||
*/
|
||||
|
|
@ -41,6 +42,11 @@ class History
|
|||
*/
|
||||
const SETTINGS = 'SETTINGS';
|
||||
|
||||
/**
|
||||
* @var string Action key: a bulk import has been processed.
|
||||
*/
|
||||
const IMPORT = 'IMPORT';
|
||||
|
||||
/**
|
||||
* @var string History file path.
|
||||
*/
|
||||
|
|
@ -121,6 +127,16 @@ public function updateSettings()
|
|||
$this->addEvent(self::SETTINGS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add Event: bulk import.
|
||||
*
|
||||
* Note: we don't store links add/update one by one since it can have a huge impact on performances.
|
||||
*/
|
||||
public function importLinks()
|
||||
{
|
||||
$this->addEvent(self::IMPORT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a new event and write it in the history file.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ public static function filterAndFormat($linkDb, $selection, $prependNoteUrl, $in
|
|||
* @param int $importCount how many links were imported
|
||||
* @param int $overwriteCount how many links were overwritten
|
||||
* @param int $skipCount how many links were skipped
|
||||
* @param int $duration how many seconds did the import take
|
||||
*
|
||||
* @return string Summary of the bookmark import status
|
||||
*/
|
||||
|
|
@ -74,14 +75,16 @@ private static function importStatus(
|
|||
$filesize,
|
||||
$importCount=0,
|
||||
$overwriteCount=0,
|
||||
$skipCount=0
|
||||
$skipCount=0,
|
||||
$duration=0
|
||||
)
|
||||
{
|
||||
$status = 'File '.$filename.' ('.$filesize.' bytes) ';
|
||||
if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) {
|
||||
$status .= 'has an unknown file format. Nothing was imported.';
|
||||
} else {
|
||||
$status .= 'was successfully processed: '.$importCount.' links imported, ';
|
||||
$status .= 'was successfully processed in '. $duration .' seconds: ';
|
||||
$status .= $importCount.' links imported, ';
|
||||
$status .= $overwriteCount.' links overwritten, ';
|
||||
$status .= $skipCount.' links skipped.';
|
||||
}
|
||||
|
|
@ -101,6 +104,7 @@ private static function importStatus(
|
|||
*/
|
||||
public static function import($post, $files, $linkDb, $conf, $history)
|
||||
{
|
||||
$start = time();
|
||||
$filename = $files['filetoupload']['name'];
|
||||
$filesize = $files['filetoupload']['size'];
|
||||
$data = file_get_contents($files['filetoupload']['tmp_name']);
|
||||
|
|
@ -184,7 +188,6 @@ public static function import($post, $files, $linkDb, $conf, $history)
|
|||
$linkDb[$existingLink['id']] = $newLink;
|
||||
$importCount++;
|
||||
$overwriteCount++;
|
||||
$history->updateLink($newLink);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -196,16 +199,19 @@ public static function import($post, $files, $linkDb, $conf, $history)
|
|||
$newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']);
|
||||
$linkDb[$newLink['id']] = $newLink;
|
||||
$importCount++;
|
||||
$history->addLink($newLink);
|
||||
}
|
||||
|
||||
$linkDb->save($conf->get('resource.page_cache'));
|
||||
$history->importLinks();
|
||||
|
||||
$duration = time() - $start;
|
||||
return self::importStatus(
|
||||
$filename,
|
||||
$filesize,
|
||||
$importCount,
|
||||
$overwriteCount,
|
||||
$skipCount
|
||||
$skipCount,
|
||||
$duration
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -132,8 +132,8 @@ public function testImportNoDoctype()
|
|||
public function testImportInternetExplorerEncoding()
|
||||
{
|
||||
$files = file2array('internet_explorer_encoding.htm');
|
||||
$this->assertEquals(
|
||||
'File internet_explorer_encoding.htm (356 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File internet_explorer_encoding.htm (356 bytes) was successfully processed in %d seconds:'
|
||||
.' 1 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -161,8 +161,8 @@ public function testImportInternetExplorerEncoding()
|
|||
public function testImportNested()
|
||||
{
|
||||
$files = file2array('netscape_nested.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_nested.htm (1337 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_nested.htm (1337 bytes) was successfully processed in %d seconds:'
|
||||
.' 8 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -283,8 +283,8 @@ public function testImportNested()
|
|||
public function testImportDefaultPrivacyNoPost()
|
||||
{
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import([], $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -328,8 +328,8 @@ public function testImportKeepPrivacy()
|
|||
{
|
||||
$post = array('privacy' => 'default');
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -372,8 +372,8 @@ public function testImportAsPublic()
|
|||
{
|
||||
$post = array('privacy' => 'public');
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -396,8 +396,8 @@ public function testImportAsPrivate()
|
|||
{
|
||||
$post = array('privacy' => 'private');
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -422,8 +422,8 @@ public function testOverwriteAsPublic()
|
|||
|
||||
// import links as private
|
||||
$post = array('privacy' => 'private');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -442,8 +442,8 @@ public function testOverwriteAsPublic()
|
|||
'privacy' => 'public',
|
||||
'overwrite' => 'true'
|
||||
);
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 2 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -468,8 +468,8 @@ public function testOverwriteAsPrivate()
|
|||
|
||||
// import links as public
|
||||
$post = array('privacy' => 'public');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -489,8 +489,8 @@ public function testOverwriteAsPrivate()
|
|||
'privacy' => 'private',
|
||||
'overwrite' => 'true'
|
||||
);
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 2 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -513,8 +513,8 @@ public function testSkipOverwrite()
|
|||
{
|
||||
$post = array('privacy' => 'public');
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -523,8 +523,8 @@ public function testSkipOverwrite()
|
|||
|
||||
// re-import as private, DO NOT enable overwriting
|
||||
$post = array('privacy' => 'private');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 0 links imported, 0 links overwritten, 2 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -542,8 +542,8 @@ public function testSetDefaultTags()
|
|||
'default_tags' => 'tag1,tag2 tag3'
|
||||
);
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -569,8 +569,8 @@ public function testSanitizeDefaultTags()
|
|||
'default_tags' => 'tag1&,tag2 "tag3"'
|
||||
);
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$this->assertEquals(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File netscape_basic.htm (482 bytes) was successfully processed in %d seconds:'
|
||||
.' 2 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -594,8 +594,8 @@ public function testSanitizeDefaultTags()
|
|||
public function testImportSameDate()
|
||||
{
|
||||
$files = file2array('same_date.htm');
|
||||
$this->assertEquals(
|
||||
'File same_date.htm (453 bytes) was successfully processed:'
|
||||
$this->assertStringMatchesFormat(
|
||||
'File same_date.htm (453 bytes) was successfully processed in %d seconds:'
|
||||
.' 3 links imported, 0 links overwritten, 0 links skipped.',
|
||||
NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->conf, $this->history)
|
||||
);
|
||||
|
|
@ -622,24 +622,19 @@ public function testImportCreateUpdateHistory()
|
|||
'overwrite' => 'true',
|
||||
];
|
||||
$files = file2array('netscape_basic.htm');
|
||||
$nbLinks = 2;
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
|
||||
$history = $this->history->getHistory();
|
||||
$this->assertEquals($nbLinks, count($history));
|
||||
foreach ($history as $value) {
|
||||
$this->assertEquals(History::CREATED, $value['event']);
|
||||
$this->assertTrue(new DateTime('-5 seconds') < $value['datetime']);
|
||||
$this->assertTrue(is_int($value['id']));
|
||||
}
|
||||
$this->assertEquals(1, count($history));
|
||||
$this->assertEquals(History::IMPORT, $history[0]['event']);
|
||||
$this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
|
||||
|
||||
// re-import as private, enable overwriting
|
||||
NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->conf, $this->history);
|
||||
$history = $this->history->getHistory();
|
||||
$this->assertEquals($nbLinks * 2, count($history));
|
||||
for ($i = 0 ; $i < $nbLinks ; $i++) {
|
||||
$this->assertEquals(History::UPDATED, $history[$i]['event']);
|
||||
$this->assertTrue(new DateTime('-5 seconds') < $history[$i]['datetime']);
|
||||
$this->assertTrue(is_int($history[$i]['id']));
|
||||
}
|
||||
$this->assertEquals(2, count($history));
|
||||
$this->assertEquals(History::IMPORT, $history[0]['event']);
|
||||
$this->assertTrue(new DateTime('-5 seconds') < $history[0]['datetime']);
|
||||
$this->assertEquals(History::IMPORT, $history[1]['event']);
|
||||
$this->assertTrue(new DateTime('-5 seconds') < $history[1]['datetime']);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue