Skip to content

Make import command resumabe #242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 13, 2023
Merged

Make import command resumabe #242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions src/Command/MeilisearchImportCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ protected function configure(): void
'Update settings related to indices to the search engine'
)
->addOption('batch-size', null, InputOption::VALUE_REQUIRED)
->addOption(
'skip-batches',
null,
InputOption::VALUE_REQUIRED,
'Skip the first N batches and start importing from the N+1 batch',
0
)
->addOption(
'response-timeout',
't',
Expand Down Expand Up @@ -86,7 +93,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
}

$entitiesToIndex = array_unique($indexes->all(), SORT_REGULAR);
$batchSize = $input->getOption('batch-size');
$batchSize = $input->getOption('batch-size') ?? '';
$batchSize = ctype_digit($batchSize) ? (int) $batchSize : $config->get('batchSize');
$responseTimeout = ((int) $input->getOption('response-timeout')) ?: self::DEFAULT_RESPONSE_TIMEOUT;

Expand All @@ -97,29 +104,47 @@ protected function execute(InputInterface $input, OutputInterface $output): int
continue;
}

$totalIndexed = 0;

$manager = $this->managerRegistry->getManagerForClass($entityClassName);
$repository = $manager->getRepository($entityClassName);
$classMetadata = $manager->getClassMetadata($entityClassName);
$entityIdentifiers = $classMetadata->getIdentifierFieldNames();
$sortByIdentifiersParam = array_combine($entityIdentifiers, array_fill(0, count($entityIdentifiers), 'ASC'));

$output->writeln('<info>Importing for index '.$entityClassName.'</info>');

$page = 0;
$page = max(0, (int) $input->getOption('skip-batches'));

if ($page > 0) {
$output->writeln(
sprintf(
'<info>Skipping first <comment>%d</comment> batches (<comment>%d</comment> records)</info>',
$page,
$page * $batchSize,
)
);
}

do {
$entities = $repository->findBy(
[],
null,
$sortByIdentifiersParam,
$batchSize,
$batchSize * $page
);

$responses = $this->formatIndexingResponse($this->searchService->index($manager, $entities), $responseTimeout);
$totalIndexed += count($entities);
foreach ($responses as $indexName => $numberOfRecords) {
$output->writeln(
sprintf(
'Indexed <comment>%s / %s</comment> %s entities into %s index',
'Indexed a batch of <comment>%d / %d</comment> %s entities into %s index (%d indexed since start)',
$numberOfRecords,
count($entities),
$entityClassName,
'<info>'.$indexName.'</info>'
'<info>'.$indexName.'</info>',
$totalIndexed,
)
);
}
Expand Down
69 changes: 46 additions & 23 deletions tests/Integration/CommandsTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,25 +74,25 @@ public function testSearchImportAndClearAndDeleteWithoutIndices(): void

$this->assertSame(<<<'EOD'
Importing for index Meilisearch\Bundle\Tests\Entity\Post
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index (6 indexed since start)
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index (6 indexed since start)
Settings updated.
Settings updated.
Settings updated.
Importing for index Meilisearch\Bundle\Tests\Entity\Comment
Importing for index Meilisearch\Bundle\Tests\Entity\Tag
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (6 indexed since start)
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index (6 indexed since start)
Importing for index Meilisearch\Bundle\Tests\Entity\Link
Importing for index Meilisearch\Bundle\Tests\Entity\Page
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (6 indexed since start)
Importing for index Meilisearch\Bundle\Tests\Entity\SelfNormalizable
Importing for index Meilisearch\Bundle\Tests\Entity\Post
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index (6 indexed since start)
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index (6 indexed since start)
Importing for index Meilisearch\Bundle\Tests\Entity\Tag
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index
Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (6 indexed since start)
Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index (6 indexed since start)
Done!

EOD, $importOutput);
Expand Down Expand Up @@ -150,12 +150,12 @@ public function testSearchImportWithCustomBatchSize(): void

$this->assertSame(<<<'EOD'
Importing for index Meilisearch\Bundle\Tests\Entity\Page
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (2 indexed since start)
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (4 indexed since start)
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (6 indexed since start)
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (8 indexed since start)
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (10 indexed since start)
Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (11 indexed since start)
Done!

EOD, $importOutput);
Expand All @@ -176,7 +176,7 @@ public function testSearchImportWithCustomResponseTimeout(): void
$output = $importCommandTester->getDisplay();

$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);

Expand All @@ -197,7 +197,7 @@ public function testSearchImportWithCustomResponseTimeout(): void
$output = $importCommandTester->getDisplay();

$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);
}
Expand All @@ -221,8 +221,8 @@ public function testImportDifferentEntitiesIntoSameIndex(): void

$output = $commandTester->getDisplay();
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Tag', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index', $output);
$this->assertStringContainsString('Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Link entities into sf_phpunit__tags index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Link entities into sf_phpunit__tags index (2 indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);

/** @var SearchResult $searchResult */
Expand All @@ -245,7 +245,30 @@ public function testSearchImportAggregator(): void

$output = $commandTester->getDisplay();
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);
}

public function testSearchImportWithSkipBatches(): void
{
for ($i = 0; $i < 10; ++$i) {
$this->createPage($i);
}

$command = $this->application->find('meili:import');
$commandTester = new CommandTester($command);
$return = $commandTester->execute([
'--indices' => 'pages',
'--batch-size' => '3',
'--skip-batches' => '2',
]);

$output = $commandTester->getDisplay();
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
$this->assertStringContainsString('Skipping first 2 batches (6 records)', $output);
$this->assertStringContainsString('Indexed a batch of 3 / 3 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (3 indexed since start)', $output);
$this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (4 indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);
}
Expand All @@ -264,7 +287,7 @@ public function testImportingIndexNameWithAndWithoutPrefix(): void

$output = $commandTester->getDisplay();
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);

Expand All @@ -283,7 +306,7 @@ public function testImportingIndexNameWithAndWithoutPrefix(): void

$output = $commandTester->getDisplay();
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);
$this->assertSame(0, $return);
}
Expand Down Expand Up @@ -352,7 +375,7 @@ public function testImportsSelfNormalizable(): void

$this->assertSame(<<<'EOD'
Importing for index Meilisearch\Bundle\Tests\Entity\SelfNormalizable
Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\SelfNormalizable entities into sf_phpunit__self_normalizable index
Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\SelfNormalizable entities into sf_phpunit__self_normalizable index (2 indexed since start)
Done!

EOD, $importOutput);
Expand Down
8 changes: 4 additions & 4 deletions tests/Integration/SearchTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ public function testSearchImportAggregator(): void

$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
$this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Tag', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index', $output);
$this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
$this->assertStringContainsString('Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index', $output);
$this->assertStringContainsString('Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__'.self::$indexName.' index', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
$this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (1 indexed since start)', $output);
$this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__'.self::$indexName.' index (1 indexed since start)', $output);
$this->assertStringContainsString('Done!', $output);

$searchTerm = 'Test';
Expand Down