-
-
Notifications
You must be signed in to change notification settings - Fork 4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix metadata storage with sharding #48563
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,22 @@ public function __construct( | |
) { | ||
} | ||
|
||
private function getStorageId(IFilesMetadata $filesMetadata): int { | ||
if ($filesMetadata instanceof FilesMetadata) { | ||
$storage = $filesMetadata->getStorageId(); | ||
if ($storage) { | ||
return $storage; | ||
} | ||
} | ||
// all code paths that lead to saving metadata *should* have the storage id set | ||
// this fallback is there just in case | ||
Comment on lines
+38
to
+39
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We'll need to update applications like Photos |
||
$query = $this->dbConnection->getQueryBuilder(); | ||
$query->select('storage') | ||
->from('filecache') | ||
->where($query->expr()->eq('fileid', $query->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); | ||
return $query->executeQuery()->fetchColumn(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wondering if we should cache it with |
||
} | ||
|
||
/** | ||
* store metadata into database | ||
* | ||
|
@@ -38,6 +54,7 @@ public function __construct( | |
public function store(IFilesMetadata $filesMetadata): void { | ||
$qb = $this->dbConnection->getQueryBuilder(); | ||
$qb->insert(self::TABLE_METADATA) | ||
->hintShardKey('storage', $this->getStorageId($filesMetadata)) | ||
->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) | ||
->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) | ||
->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken())) | ||
|
@@ -134,6 +151,7 @@ public function updateMetadata(IFilesMetadata $filesMetadata): int { | |
$expr = $qb->expr(); | ||
|
||
$qb->update(self::TABLE_METADATA) | ||
->hintShardKey('files_metadata', $this->getStorageId($filesMetadata)) | ||
->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) | ||
->set('sync_token', $qb->createNamedParameter($this->generateSyncToken())) | ||
->set('last_update', $qb->createFunction('NOW()')) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
<?php | ||
|
||
declare(strict_types=1); | ||
/** | ||
* SPDX-FileCopyrightText: 2024 Robin Appelman <[email protected]> | ||
* SPDX-License-Identifier: AGPL-3.0-or-later | ||
*/ | ||
|
||
namespace Test\FilesMetadata; | ||
|
||
use OC\BackgroundJob\JobList; | ||
use OC\Files\Storage\Temporary; | ||
use OC\FilesMetadata\FilesMetadataManager; | ||
use OC\FilesMetadata\Service\IndexRequestService; | ||
use OC\FilesMetadata\Service\MetadataRequestService; | ||
use OCP\EventDispatcher\Event; | ||
use OCP\EventDispatcher\IEventDispatcher; | ||
use OCP\Files\Folder; | ||
use OCP\Files\IRootFolder; | ||
use OCP\FilesMetadata\AMetadataEvent; | ||
use OCP\IAppConfig; | ||
use OCP\IDBConnection; | ||
use OCP\Server; | ||
use Psr\Log\LoggerInterface; | ||
use Test\TestCase; | ||
use Test\Traits\MountProviderTrait; | ||
use Test\Traits\UserTrait; | ||
|
||
/** | ||
* @group DB | ||
*/ | ||
class FilesMetadataManagerTest extends TestCase { | ||
use UserTrait; | ||
use MountProviderTrait; | ||
|
||
private IEventDispatcher $eventDispatcher; | ||
private JobList $jobList; | ||
private IAppConfig $appConfig; | ||
private LoggerInterface $logger; | ||
private MetadataRequestService $metadataRequestService; | ||
private IndexRequestService $indexRequestService; | ||
private FilesMetadataManager $manager; | ||
private IDBConnection $connection; | ||
private Folder $userFolder; | ||
private array $metadata = []; | ||
|
||
protected function setUp(): void { | ||
parent::setUp(); | ||
|
||
$this->jobList = $this->createMock(JobList::class); | ||
$this->eventDispatcher = $this->createMock(IEventDispatcher::class); | ||
$this->eventDispatcher->method('dispatchTyped')->willReturnCallback(function (Event $event) { | ||
if ($event instanceof AMetadataEvent) { | ||
$name = $event->getNode()->getName(); | ||
if (isset($this->metadata[$name])) { | ||
$meta = $event->getMetadata(); | ||
foreach ($this->metadata[$name] as $key => $value) { | ||
$meta->setString($key, $value); | ||
} | ||
} | ||
} | ||
}); | ||
$this->appConfig = $this->createMock(IAppConfig::class); | ||
$this->logger = $this->createMock(LoggerInterface::class); | ||
|
||
$this->connection = Server::get(IDBConnection::class); | ||
$this->metadataRequestService = new MetadataRequestService($this->connection, $this->logger); | ||
$this->indexRequestService = new IndexRequestService($this->connection, $this->logger); | ||
$this->manager = new FilesMetadataManager( | ||
$this->eventDispatcher, | ||
$this->jobList, | ||
$this->appConfig, | ||
$this->logger, | ||
$this->metadataRequestService, | ||
$this->indexRequestService, | ||
); | ||
|
||
$this->createUser('metatest', ''); | ||
$this->registerMount('metatest', new Temporary([]), '/metatest'); | ||
|
||
$rootFolder = Server::get(IRootFolder::class); | ||
$this->userFolder = $rootFolder->getUserFolder('metatest'); | ||
} | ||
|
||
public function testRefreshMetadata(): void { | ||
$this->metadata['test.txt'] = [ | ||
'istest' => 'yes' | ||
]; | ||
$file = $this->userFolder->newFile('test.txt', 'test'); | ||
$stored = $this->manager->refreshMetadata($file); | ||
$this->assertEquals($file->getId(), $stored->getFileId()); | ||
$this->assertEquals('yes', $stored->getString('istest')); | ||
|
||
$retrieved = $this->manager->getMetadata($file->getId()); | ||
$this->assertEquals($file->getId(), $retrieved->getFileId()); | ||
$this->assertEquals('yes', $retrieved->getString('istest')); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Typing is not working?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Things are typed to return this interface, but only the implementation has the
setStorageId
method because I didn't want to touch the public api