From a683a1742964fd9787bdbaac4bde55ad99c6130a Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Thu, 18 Jul 2024 15:44:51 +1000 Subject: [PATCH] feat: add object tagging --- TAGGING.md | 86 +++++ classes/check/tagging_status.php | 62 ++++ classes/local/manager.php | 5 + classes/local/report/objectfs_report.php | 1 + .../local/report/tag_count_report_builder.php | 51 +++ classes/local/store/object_client.php | 27 ++ classes/local/store/object_client_base.php | 36 ++ classes/local/store/object_file_system.php | 56 +++ classes/local/store/s3/client.php | 112 ++++++ classes/local/store/s3/file_system.php | 1 + classes/local/tag/environment_source.php | 61 ++++ classes/local/tag/mime_type_source.php | 65 ++++ classes/local/tag/tag_manager.php | 183 ++++++++++ classes/local/tag/tag_source.php | 50 +++ classes/task/trigger_update_object_tags.php | 48 +++ classes/task/update_object_tags.php | 85 +++++ classes/tests/test_client.php | 37 ++ classes/tests/testcase.php | 3 +- db/install.xml | 20 +- db/tasks.php | 12 + db/upgrade.php | 45 +++ lang/en/tool_objectfs.php | 19 + lib.php | 11 +- settings.php | 42 ++- tests/local/report/object_status_test.php | 2 +- tests/local/tagging_test.php | 340 ++++++++++++++++++ tests/object_file_system_test.php | 104 ++++++ tests/task/populate_objects_filesize_test.php | 1 + .../task/trigger_update_object_tags_test.php | 49 +++ tests/task/update_object_tags_test.php | 170 +++++++++ version.php | 4 +- 31 files changed, 1777 insertions(+), 11 deletions(-) create mode 100644 TAGGING.md create mode 100644 classes/check/tagging_status.php create mode 100644 classes/local/report/tag_count_report_builder.php create mode 100644 classes/local/tag/environment_source.php create mode 100644 classes/local/tag/mime_type_source.php create mode 100644 classes/local/tag/tag_manager.php create mode 100644 classes/local/tag/tag_source.php create mode 100644 classes/task/trigger_update_object_tags.php create mode 100644 classes/task/update_object_tags.php create mode 100644 tests/local/tagging_test.php create mode 100644 tests/task/trigger_update_object_tags_test.php create mode 100644 tests/task/update_object_tags_test.php diff --git a/TAGGING.md b/TAGGING.md new file mode 100644 index 00000000..0c2155e8 --- /dev/null +++ b/TAGGING.md @@ -0,0 +1,86 @@ +# Tagging +Tagging allows extra metadata about your files to be send to the external object store. These sources are defined in code, and currently cannot be configured on/off from the UI. + +Currently, this is only implemented for the S3 file system client. +**Tagging vs metadata** + +Note object tags are different from object metadata. + +Object metadata is immutable, and attached to the object on upload. With metadata, if you wish to update it (for example during a migration, or the sources changed), you have to copy the object with the new metadata, and delete the old object. This is problematic, since deletion is optional in objectfs. + +Object tags are more suitable, since their permissions can be managed separately (e.g. a client can be allowed to modify tags, but not delete objects). + +## File system setup +### S3 +[See the S3 docs for more information about tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html). + +You must allow `s3:GetObjectTagging` and `s3:PutObjectTagging` permission to the objectfs client. + +## Sources +The following sources are implemented currently: +### Environment +What environment the file was uploaded in. Configure the environment using `$CFG->objectfs_environment_name` + +### Mimetype +What mimetype the file is stored as under the `mdl_files` table. + +## Multiple environments pointing to single bucket +It is possible you are using objectfs with multiple environments (e.g. prod, staging) that both point to the same bucket. Since files are referenced by contenthash, it generally does not matter where they come from, so this isn't a problem. However to ensure the tags remain accurate, you should turn off `overwriteobjecttags` in the plugin settings for every environment except production. + +This means that staging is unable to overwrite tags for files uploaded elsewhere, but can set it on files only uploaded only from staging. However, files uploaded from production will always have the correct tags, and will overwrite any existing tags. + +```mermaid +graph LR + subgraph S3 + Object("`**Object** + contenthash: xyz + tags: env=prod`") + end + subgraph Prod + UploadObjectProd["`**Upload object** + contenthash: xyz + tags: env=prod`"] --> Object + end + subgraph Staging + UploadObjectStaging["`**Upload object** + contenthash: xyz + tags: env=staging`"] + end + Blocked["Blocked - does not have permissions\nto overwrite existing object tags"] + UploadObjectStaging --- Blocked + Blocked -.-> Object + + style Object fill:#ffffff00,stroke:#ffa812 + style S3 fill:#ffffff00,stroke:#ffa812 + style Prod fill:#ffffff00,stroke:#26ff4a + style UploadObjectProd fill:#ffffff00,stroke:#26ff4a + style Staging fill:#ffffff00,stroke:#978aff + style UploadObjectStaging fill:#ffffff00,stroke:#978aff + style Blocked fill:#ffffff00,stroke:#ff0000 +``` + +## Migration +If the way a tag was calculated has changed, or new tags are added (or removed) or this feature was turned on for the first time (or turned on after being off), you must do the following: +- Manually run `trigger_update_object_tags` scheduled task from the UI, which queues a `update_object_tags` adhoc task that will process all objects marked as needing sync (default is true) +or +- Call the CLI to execute a `update_object_tags` adhoc task manually. + +## Reporting +There is an additional graph added to the object summary report showing the tag value combinations and counts of each. + +Note, this is only for files that have been uploaded from this environment, and may not be consistent for environments where `overwriteobjecttags` is disabled (because the site does not know if a file was overwritten in the external store by another client). + +## For developers + +### Adding a new source +Note the rules about sources: +- Identifier must be < 32 chars long. +- Value must be < 128 chars long. + +While external providers allow longer key/values, we intentionally limit it to reserve space for future use. These limits may change in the future as the feature matures. + +To add a new source: +- Implement `tag_source` +- Add to the `tag_manager` class +- As part of an upgrade step, mark all objects `tagsyncstatus` to needing sync (using `tag_manager` class, or manually in the DB) +- As part of an upgrade step, queue a `update_object_tags` adhoc task to process the tag migration. \ No newline at end of file diff --git a/classes/check/tagging_status.php b/classes/check/tagging_status.php new file mode 100644 index 00000000..df3e68d5 --- /dev/null +++ b/classes/check/tagging_status.php @@ -0,0 +1,62 @@ +. + +namespace tool_objectfs\check; + +use core\check\check; +use core\check\result; +use tool_objectfs\local\tag\tag_manager; + +/** + * Tagging status check + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_status extends check { + /** + * Link to ObjectFS settings page. + * + * @return \action_link|null + */ + public function get_action_link(): ?\action_link { + $url = new \moodle_url('/admin/category.php', ['category' => 'tool_objectfs']); + return new \action_link($url, get_string('pluginname', 'tool_objectfs')); + } + + /** + * Get result + * @return result + */ + public function get_result(): result { + if (!tag_manager::is_tagging_enabled_and_supported()) { + return new result(result::NA, get_string('check:tagging:na', 'tool_objectfs')); + } + + // Do a tag set test. + $config = \tool_objectfs\local\manager::get_objectfs_config(); + $client = \tool_objectfs\local\manager::get_client($config); + $result = $client->test_set_object_tag(); + + if ($result->success) { + return new result(result::OK, get_string('check:tagging:ok', 'tool_objectfs'), $result->details); + } else { + return new result(result::ERROR, get_string('check:tagging:error', 'tool_objectfs'), $result->details); + } + } +} diff --git a/classes/local/manager.php b/classes/local/manager.php index 5d791f97..188df9fc 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -64,6 +64,7 @@ public static function get_objectfs_config() { $config->batchsize = 10000; $config->useproxy = 0; $config->deleteexternal = 0; + $config->enabletagging = false; $config->filesystem = ''; $config->enablepresignedurls = 0; @@ -329,6 +330,10 @@ public static function get_available_fs_list() { * @return string */ public static function get_client_classname_from_fs($filesystem) { + // Unit tests need to return the test client. + if ($filesystem == '\tool_objectfs\tests\test_file_system') { + return '\tool_objectfs\tests\test_client'; + } $clientclass = str_replace('_file_system', '', $filesystem); return str_replace('tool_objectfs\\', 'tool_objectfs\\local\\store\\', $clientclass.'\\client'); } diff --git a/classes/local/report/objectfs_report.php b/classes/local/report/objectfs_report.php index cc9eb910..56869cd7 100644 --- a/classes/local/report/objectfs_report.php +++ b/classes/local/report/objectfs_report.php @@ -166,6 +166,7 @@ public static function get_report_types() { 'location', 'log_size', 'mime_type', + 'tag_count', ]; } diff --git a/classes/local/report/tag_count_report_builder.php b/classes/local/report/tag_count_report_builder.php new file mode 100644 index 00000000..0364b3fc --- /dev/null +++ b/classes/local/report/tag_count_report_builder.php @@ -0,0 +1,51 @@ +. + +namespace tool_objectfs\local\report; + +/** + * Tag count report builder. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tag_count_report_builder extends objectfs_report_builder { + /** + * Builds report + * @param int $reportid + * @return objectfs_report + */ + public function build_report($reportid) { + global $DB; + $report = new objectfs_report('tag_count', $reportid); + + // Returns counts + sizes of key:value. + $sql = " + SELECT CONCAT(COALESCE(object_tags.tagkey, '(untagged)'), ': ', COALESCE(object_tags.tagvalue, '')) as datakey, + COUNT(objects.id) as objectcount, + SUM(objects.filesize) as objectsum + FROM {tool_objectfs_objects} objects + LEFT JOIN {tool_objectfs_object_tags} object_tags + ON objects.contenthash = object_tags.contenthash + GROUP BY object_tags.tagkey, object_tags.tagvalue + "; + $result = $DB->get_records_sql($sql); + $report->add_rows($result); + return $report; + } +} diff --git a/classes/local/store/object_client.php b/classes/local/store/object_client.php index 9121a824..edb8b9c0 100644 --- a/classes/local/store/object_client.php +++ b/classes/local/store/object_client.php @@ -25,6 +25,8 @@ namespace tool_objectfs\local\store; +use stdClass; + interface object_client { /** @@ -137,6 +139,31 @@ public function proxy_range_request(\stored_file $file, $ranges); */ public function test_range_request($filesystem); + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass; + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags); + + /** + * Returns given objects tags queried from the external store. External object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array; + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool; } diff --git a/classes/local/store/object_client_base.php b/classes/local/store/object_client_base.php index 4f03418d..f2d5dd42 100644 --- a/classes/local/store/object_client_base.php +++ b/classes/local/store/object_client_base.php @@ -25,6 +25,8 @@ namespace tool_objectfs\local\store; +use stdClass; + /** * [Description object_client_base] */ @@ -187,4 +189,38 @@ public function test_connection() { public function test_permissions($testdelete) { return (object)['success' => false, 'details' => '']; } + + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass { + return (object)['success' => false, 'details' => '']; + } + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags) { + return []; + } + + /** + * Returns given objects tags queried from the external store. External object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array { + return []; + } + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool { + return false; + } } diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index fb3da10d..cbdd8306 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -36,7 +36,10 @@ use stored_file; use file_storage; use BlobRestProxy; +use coding_exception; +use Throwable; use tool_objectfs\local\manager; +use tool_objectfs\local\tag\tag_manager; defined('MOODLE_INTERNAL') || die(); @@ -360,6 +363,12 @@ public function copy_object_from_local_to_external_by_hash($contenthash, $object } } + // If tagging is enabled, ensure tags are synced regardless of if object is local or duplicated, etc... + // The file may exist in external store because it was uploaded by another site, but we may want to put our tags onto it. + if (tag_manager::is_tagging_enabled_and_supported()) { + $this->push_object_tags($contenthash); + } + $this->logger->log_object_move('copy_object_from_local_to_external', $initiallocation, $finallocation, @@ -1154,4 +1163,51 @@ private function update_object(array $result): array { return $result; } + + /** + * Pushes tags to the external store (post upload) for a given hash. + * External client must support tagging. + * + * @param string $contenthash file to sync tags for + */ + public function push_object_tags(string $contenthash) { + if (!$this->get_external_client()->supports_object_tagging()) { + throw new coding_exception("Cannot sync tags, external client does not support tagging."); + } + + // Get a lock before syncing, to ensure other parts of objectfs are not moving/interacting with this object. + $lock = $this->acquire_object_lock($contenthash, 10); + + // No lock - just skip it. + if (!$lock) { + throw new coding_exception("Could not get object lock"); + } + + try { + $objectexists = $this->is_file_readable_externally_by_hash($contenthash); + + // Object must exist, and we can overwrite (and not care about existing tags) + // or cannot overwrite, and the tags are empty. + // Avoid unnecessarily checking tags, since this is an extra API call. + $canset = $objectexists && (tag_manager::can_overwrite_object_tags() || + empty($this->get_external_client()->get_object_tags($contenthash))); + + if ($canset) { + $tags = tag_manager::gather_object_tags_for_upload($contenthash); + $this->get_external_client()->set_object_tags($contenthash, $tags); + tag_manager::store_tags_locally($contenthash, $tags); + } + + // Regardless, it has synced. + tag_manager::mark_object_tag_sync_status($contenthash, tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED); + } catch (Throwable $e) { + $lock->release(); + + // Mark object as tag sync error, this should stop it re-trying until fixed manually. + tag_manager::mark_object_tag_sync_status($contenthash, tag_manager::SYNC_STATUS_ERROR); + + throw $e; + } + $lock->release(); + } } diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index a6e2598e..5e73d974 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -25,10 +25,13 @@ namespace tool_objectfs\local\store\s3; +use coding_exception; use tool_objectfs\local\manager; use tool_objectfs\local\store\object_client_base; use tool_objectfs\local\store\signed_url; use local_aws\admin_settings_aws_region; +use stdClass; +use Throwable; define('AWS_API_VERSION', '2006-03-01'); define('AWS_CAN_READ_OBJECT', 0); @@ -875,4 +878,113 @@ public function test_range_request($filesystem) { } return (object)['result' => false, 'error' => get_string('fixturefilemissing', 'tool_objectfs')]; } + + /** + * Tests setting an objects tag. + * @return stdClass containing 'success' and 'details' properties + */ + public function test_set_object_tag(): stdClass { + try { + // First ensure a test object exists to put tags on. + // Note this will override the existing object if exists. + $key = $this->bucketkeyprefix . 'tagging_check_file'; + $this->client->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $key, + 'Body' => 'test content', + ]); + + // Next try to tag it - this will throw an exception if cannot set + // (for example, because it does not have permissions to). + $this->client->putObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $key, + 'Tagging' => [ + 'TagSet' => [ + [ + 'Key' => 'test', + 'Value' => 'test', + ], + ], + ], + ]); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'details' => $e->getMessage(), + ]; + } + + // Success - no exceptions thrown. + return (object) ['success' => true, 'details' => '']; + } + + /** + * Convert key=>value to s3 tag format + * @param array $tags + * @return array tags in s3 format. + */ + private function convert_tags_to_s3_format(array $tags): array { + foreach ($tags as $key => $value) { + $s3tags[] = [ + 'Key' => $key, + 'Value' => $value, + ]; + } + return $s3tags; + } + + /** + * Set the given objects tags in the external store. + * @param string $contenthash file content hash + * @param array $tags array of key=>value pairs to set as tags. + */ + public function set_object_tags(string $contenthash, array $tags) { + $objectkey = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash); + + // Then put onto object. + $this->client->putObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $objectkey, + 'Tagging' => [ + 'TagSet' => $this->convert_tags_to_s3_format($tags), + ], + ]); + } + + /** + * Returns given objects tags queried from the external store. Object must exist. + * @param string $contenthash file content has + * @return array array of key=>value tag pairs + */ + public function get_object_tags(string $contenthash): array { + $key = $this->bucketkeyprefix . $this->get_filepath_from_hash($contenthash); + + // Query from S3. + $result = $this->client->getObjectTagging([ + 'Bucket' => $this->bucket, + 'Key' => $key, + ]); + + // Ensure tags are what we expect, and AWS have not changed the format. + if (!array_key_exists('TagSet', $result->toArray())) { + throw new coding_exception("Unexpected tag format received. Result did not contain a TagSet"); + } + + // Convert from S3 format to key=>value format. + $tagkv = []; + foreach ($result->toArray()['TagSet'] as $tag) { + $tagkv[$tag['Key']] = $tag['Value']; + } + + return $tagkv; + } + + /** + * If the client supports object tagging feature. + * @return bool true if supports, else false + */ + public function supports_object_tagging(): bool { + return true; + } } diff --git a/classes/local/store/s3/file_system.php b/classes/local/store/s3/file_system.php index dd911092..83ac861d 100644 --- a/classes/local/store/s3/file_system.php +++ b/classes/local/store/s3/file_system.php @@ -32,6 +32,7 @@ use tool_objectfs\local\manager; use tool_objectfs\local\store\object_file_system; +use tool_objectfs\local\tag\tag_manager; require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); diff --git a/classes/local/tag/environment_source.php b/classes/local/tag/environment_source.php new file mode 100644 index 00000000..d7715ddb --- /dev/null +++ b/classes/local/tag/environment_source.php @@ -0,0 +1,61 @@ +. + +namespace tool_objectfs\local\tag; + +/** + * Provides environment a file was uploaded in. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class environment_source implements tag_source { + /** + * Identifier used in tagging file. Is the 'key' of the tag. + * @return string + */ + public static function get_identifier(): string { + return 'environment'; + } + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string { + return get_string('tagsource:environment', 'tool_objectfs', self::get_env()); + } + + /** + * Returns current env value from $CFG + * @return string|null string if set, else null + */ + private static function get_env(): ?string { + global $CFG; + return !empty($CFG->objectfs_environment_name) ? $CFG->objectfs_environment_name : null; + } + + /** + * Returns the tag value for the given file contenthash + * @param string $contenthash + * @return string|null mime type for file. + */ + public function get_value_for_contenthash(string $contenthash): ?string { + return self::get_env(); + } +} diff --git a/classes/local/tag/mime_type_source.php b/classes/local/tag/mime_type_source.php new file mode 100644 index 00000000..7546cdcf --- /dev/null +++ b/classes/local/tag/mime_type_source.php @@ -0,0 +1,65 @@ +. + +namespace tool_objectfs\local\tag; + +/** + * Provides mime type of file. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class mime_type_source implements tag_source { + /** + * Identifier used in tagging file. Is the 'key' of the tag. + * @return string + */ + public static function get_identifier(): string { + return 'mimetype'; + } + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string { + return get_string('tagsource:mimetype', 'tool_objectfs'); + } + + /** + * Returns the tag value for the given file contenthash + * @param string $contenthash + * @return string|null mime type for file. + */ + public function get_value_for_contenthash(string $contenthash): ?string { + global $DB; + // Sometimes multiple with same hash are uploaded (e.g. real vs draft), + // in this case, just take the first (mimetype is the same regardless). + $mime = $DB->get_field_sql('SELECT mimetype + FROM {files} + WHERE contenthash = :hash + LIMIT 1', + ['hash' => $contenthash]); + + if (empty($mime)) { + return null; + } + + return $mime; + } +} diff --git a/classes/local/tag/tag_manager.php b/classes/local/tag/tag_manager.php new file mode 100644 index 00000000..36aada41 --- /dev/null +++ b/classes/local/tag/tag_manager.php @@ -0,0 +1,183 @@ +. + +namespace tool_objectfs\local\tag; + +use coding_exception; +use tool_objectfs\local\manager; + +defined('MOODLE_INTERNAL') || die(); +require_once($CFG->dirroot . '/admin/tool/objectfs/lib.php'); + +/** + * Manages object tagging feature. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tag_manager { + + /** + * @var int If object needs sync. These will periodically be picked up by scheduled tasks and queued for syncing. + */ + public const SYNC_STATUS_NEEDS_SYNC = 0; + + /** + * @var int Object does not need sync. Will be essentially ignored in tagging process. + */ + public const SYNC_STATUS_SYNC_NOT_REQUIRED = 1; + + /** + * @var int Object tried to sync but there was an error. Will make it ignored and must be corrected manually. + */ + public const SYNC_STATUS_ERROR = 2; + + /** + * @var array All possible tag sync statuses. + */ + public const SYNC_STATUSES = [ + self::SYNC_STATUS_NEEDS_SYNC, + self::SYNC_STATUS_SYNC_NOT_REQUIRED, + self::SYNC_STATUS_ERROR, + ]; + + /** + * Returns an array of tag_source instances that are currently defined. + * @return array + */ + public static function get_defined_tag_sources(): array { + // All possible tag sources should be defined here. + // Note this should be a maximum of 10 sources, as this is an AWS limit. + return [ + new mime_type_source(), + new environment_source(), + ]; + } + + /** + * Is the tagging feature enabled and supported by the configured fs? + * @return bool + */ + public static function is_tagging_enabled_and_supported(): bool { + $enabledinconfig = !empty(get_config('tool_objectfs', 'taggingenabled')); + + $client = manager::get_client(manager::get_objectfs_config()); + $supportedbyfs = !empty($client) && $client->supports_object_tagging(); + + return $enabledinconfig && $supportedbyfs; + } + + /** + * Gathers the tag values for a given content hash + * @param string $contenthash + * @return array array of key=>value pairs, the tags for the given file. + */ + public static function gather_object_tags_for_upload(string $contenthash): array { + $tags = []; + foreach (self::get_defined_tag_sources() as $source) { + $val = $source->get_value_for_contenthash($contenthash); + + // Null means not set for this object. + if (is_null($val)) { + continue; + } + + $tags[$source->get_identifier()] = $val; + } + return $tags; + } + + /** + * Stores tag records for contenthash locally + * @param string $contenthash + * @param array $tags + */ + public static function store_tags_locally(string $contenthash, array $tags) { + global $DB; + + // Purge any existing tags for this object. + $DB->delete_records('tool_objectfs_object_tags', ['contenthash' => $contenthash]); + + // Record time in var, so that they all have the same time. + $timemodified = time(); + + // Store new records. + $recordstostore = []; + foreach ($tags as $key => $value) { + $recordstostore[] = [ + 'contenthash' => $contenthash, + 'tagkey' => $key, + 'tagvalue' => $value, + 'timemodified' => $timemodified, + ]; + } + $DB->insert_records('tool_objectfs_object_tags', $recordstostore); + } + + /** + * Returns objects that are candidates for tag syncing. + * @param int $limit max number of records to return + * @return array array of contenthashes, which need tags calculated and synced. + */ + public static function get_objects_needing_sync(int $limit) { + global $DB; + + // Find object records where the status is NEEDS_SYNC and is replicated. + [$insql, $inparams] = $DB->get_in_or_equal([ + OBJECT_LOCATION_DUPLICATED, OBJECT_LOCATION_EXTERNAL, OBJECT_LOCATION_ORPHANED], SQL_PARAMS_NAMED); + $inparams['syncstatus'] = self::SYNC_STATUS_NEEDS_SYNC; + $records = $DB->get_records_select('tool_objectfs_objects', 'tagsyncstatus = :syncstatus AND location ' . $insql, + $inparams, '', 'contenthash', 0, $limit); + return array_column($records, 'contenthash'); + } + + /** + * Marks a given object as the given status. + * @param string $contenthash + * @param int $status one of SYNC_STATUS_* constants + */ + public static function mark_object_tag_sync_status(string $contenthash, int $status) { + global $DB; + if (!in_array($status, self::SYNC_STATUSES)) { + throw new coding_exception("Invalid object tag sync status " . $status); + } + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', $status, ['contenthash' => $contenthash]); + } + + /** + * Returns a simple list of all the sources and their descriptions. + * @return string html string + */ + public static function get_tag_summary_html(): string { + $sources = self::get_defined_tag_sources(); + $html = ''; + + foreach ($sources as $source) { + $html .= $source->get_identifier() . ': ' . $source->get_description() . '
'; + } + return $html; + } + + /** + * If the current env is allowed to overwrite tags on objects that already have tags. + * @return bool + */ + public static function can_overwrite_object_tags(): bool { + return (bool) get_config('tool_objectfs', 'overwriteobjecttags'); + } +} diff --git a/classes/local/tag/tag_source.php b/classes/local/tag/tag_source.php new file mode 100644 index 00000000..a915c82a --- /dev/null +++ b/classes/local/tag/tag_source.php @@ -0,0 +1,50 @@ +. + +namespace tool_objectfs\local\tag; + +/** + * Tag source interface + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +interface tag_source { + /** + * Returns an unchanging identifier for this source. + * Must never change, otherwise it will lose connection with the tags replicated to objects. + * If it ever must change, a migration step must be completed to trigger all objects to recalculate their tags. + * Must not exceed 128 chars. + * @return string + */ + public static function get_identifier(): string; + + /** + * Description for source displayed in the admin settings. + * @return string + */ + public static function get_description(): string; + + /** + * Returns the value of this tag for the file with the given content hash. + * This must be deterministic, and should never exceed 128 chars. + * @param string $contenthash + * @return string + */ + public function get_value_for_contenthash(string $contenthash): ?string; +} diff --git a/classes/task/trigger_update_object_tags.php b/classes/task/trigger_update_object_tags.php new file mode 100644 index 00000000..6cf64dd9 --- /dev/null +++ b/classes/task/trigger_update_object_tags.php @@ -0,0 +1,48 @@ +. + +namespace tool_objectfs\task; + +use core\task\manager; +use core\task\scheduled_task; + +/** + * Queues update_object_tags adhoc task periodically, or manually from the frontend. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class trigger_update_object_tags extends scheduled_task { + /** + * Task name + */ + public function get_name() { + return get_string('task:triggerupdateobjecttags', 'tool_objectfs'); + } + /** + * Execute task + */ + public function execute() { + // Queue adhoc task, nothing else. + $task = new update_object_tags(); + $task->set_custom_data([ + 'iteration' => 1, + ]); + manager::queue_adhoc_task($task, true); + } +} diff --git a/classes/task/update_object_tags.php b/classes/task/update_object_tags.php new file mode 100644 index 00000000..cc3bbfcb --- /dev/null +++ b/classes/task/update_object_tags.php @@ -0,0 +1,85 @@ +. + +namespace tool_objectfs\task; + +use core\task\adhoc_task; +use tool_objectfs\local\tag\tag_manager; + +/** + * Calculates and updates an objects tags in the external store. + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class update_object_tags extends adhoc_task { + /** + * Execute task + */ + public function execute() { + if (!tag_manager::is_tagging_enabled_and_supported()) { + mtrace("Tagging feature not enabled or supported by filesystem, exiting."); + return; + } + + // Since this adhoc task can requeue itself, ensure there is a fixed limit on the number + // of times this can happen, to avoid any accidental runaways. + $iterationlimit = get_config('tool_objectfs', 'maxtaggingiterations') ?: 0; + $iteration = !empty($this->get_custom_data()->iteration) ? $this->get_custom_data()->iteration : 0; + + if (empty($iterationlimit) || empty($iteration)) { + mtrace("Invalid number of iterations, exiting."); + return; + } + + if (abs($iteration) > abs($iterationlimit)) { + mtrace("Maximum number of iterations reached: " . $iteration . ", exiting."); + return; + } + + // Get the maximum num of objects to update as configured. + $limit = get_config('tool_objectfs', 'maxtaggingperrun'); + $contenthashes = tag_manager::get_objects_needing_sync($limit); + + if (empty($contenthashes)) { + mtrace("No more objects found that need tagging, exiting."); + return; + } + + // Sanity check that fs is object file system and not anything else. + $fs = get_file_storage()->get_file_system(); + + if (!method_exists($fs, "push_object_tags")) { + mtrace("File system is not object file system, exiting."); + return; + } + + // For each, try to sync their tags. + foreach ($contenthashes as $contenthash) { + $fs->push_object_tags($contenthash); + } + + // Re-queue self to process more in another iteration. + mtrace("Requeing self for another iteration."); + $task = new update_object_tags(); + $task->set_custom_data([ + 'iteration' => $iteration + 1, + ]); + \core\task\manager::queue_adhoc_task($task); + } +} diff --git a/classes/tests/test_client.php b/classes/tests/test_client.php index 7a380c17..a0151ec9 100644 --- a/classes/tests/test_client.php +++ b/classes/tests/test_client.php @@ -16,6 +16,7 @@ namespace tool_objectfs\tests; +use coding_exception; use tool_objectfs\local\store\object_client_base; /** @@ -34,6 +35,11 @@ class test_client extends object_client_base { */ private $bucketpath; + /** + * @var array in-memory tags used for unit tests + */ + public $tags; + /** * string * @param \stdClass $config @@ -157,5 +163,36 @@ public function test_permissions($testdelete) { public function get_maximum_upload_size() { return $this->maxupload; } + + /** + * Sets object tags - uses in-memory store for unit tests + * @param string $contenthash + * @param array $tags + */ + public function set_object_tags(string $contenthash, array $tags) { + global $CFG; + if (!empty($CFG->phpunit_objectfs_simulate_tag_set_error)) { + throw new coding_exception("Simulated tag set error"); + } + $this->tags[$contenthash] = $tags; + } + + /** + * Gets object tags - uses in-memory store for unit tests + * @param string $contenthash + * @return array + */ + public function get_object_tags(string $contenthash): array { + return $this->tags[$contenthash] ?? []; + } + + /** + * Object tagging support, for unit testing + * @return bool + */ + public function supports_object_tagging(): bool { + global $CFG; + return $CFG->phpunit_objectfs_supports_object_tagging; + } } diff --git a/classes/tests/testcase.php b/classes/tests/testcase.php index 6d85a3b5..124ce5d4 100644 --- a/classes/tests/testcase.php +++ b/classes/tests/testcase.php @@ -30,7 +30,6 @@ * @package tool_objectfs */ abstract class testcase extends \advanced_testcase { - /** @var test_file_system Filesystem */ public $filesystem; @@ -45,8 +44,10 @@ protected function setUp(): void { global $CFG; $CFG->alternative_file_system_class = '\\tool_objectfs\\tests\\test_file_system'; $CFG->forced_plugin_settings['tool_objectfs']['deleteexternal'] = false; + $CFG->objectfs_environment_name = 'test'; $this->filesystem = new test_file_system(); $this->logger = new \tool_objectfs\log\null_logger(); + $this->resetAfterTest(true); } diff --git a/db/install.xml b/db/install.xml index 855c9786..5d6b6f4e 100644 --- a/db/install.xml +++ b/db/install.xml @@ -1,5 +1,5 @@ - @@ -11,6 +11,7 @@ + @@ -37,7 +38,7 @@ - + @@ -49,5 +50,20 @@ + + + + + + + + + + + + + + +
diff --git a/db/tasks.php b/db/tasks.php index ac98e3ff..43e3cfa6 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -107,5 +107,17 @@ 'dayofweek' => '*', 'month' => '*', ], + [ + 'classname' => 'tool_objectfs\task\trigger_update_object_tags', + 'blocking' => 0, + 'minute' => 'R', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*', + // Default disabled - intended to be manually run. + // Also, objectfs tagging support is default off. + 'disabled' => true, + ], ]; diff --git a/db/upgrade.php b/db/upgrade.php index 19c70089..26c8840a 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -170,5 +170,50 @@ function xmldb_tool_objectfs_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2023013100, 'tool', 'objectfs'); } + + if ($oldversion < 2023051702) { + + // Define table tool_objectfs_object_tags to be created. + $table = new xmldb_table('tool_objectfs_object_tags'); + + // Adding fields to table tool_objectfs_object_tags. + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('contenthash', XMLDB_TYPE_CHAR, '40', null, XMLDB_NOTNULL, null, null); + $table->add_field('tagkey', XMLDB_TYPE_CHAR, '32', null, XMLDB_NOTNULL, null, null); + $table->add_field('tagvalue', XMLDB_TYPE_CHAR, '128', null, XMLDB_NOTNULL, null, null); + $table->add_field('timemodified', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null); + + // Adding keys to table tool_objectfs_object_tags. + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); + + // Adding indexes to table tool_objectfs_object_tags. + $table->add_index('objecttagkey_idx', XMLDB_INDEX_UNIQUE, ['contenthash', 'tagkey']); + + // Conditionally launch create table for tool_objectfs_object_tags. + if (!$dbman->table_exists($table)) { + $dbman->create_table($table); + } + + // Define field tagsyncstatus to be added to tool_objectfs_objects. + $table = new xmldb_table('tool_objectfs_objects'); + $field = new xmldb_field('tagsyncstatus', XMLDB_TYPE_INTEGER, '2', null, XMLDB_NOTNULL, null, '0', 'filesize'); + + // Conditionally launch add field tagsyncstatus. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + // Changing precision of field datakey on table tool_objectfs_report_data, + // to (255) to allow for tag key + value pairs to fit in. + $table = new xmldb_table('tool_objectfs_report_data'); + $field = new xmldb_field('datakey', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, 'reporttype'); + + // Launch change of precision for field datakey. + $dbman->change_field_precision($table, $field); + + // Objectfs savepoint reached. + upgrade_plugin_savepoint(true, 2023051702, 'tool', 'objectfs'); + } + return true; } diff --git a/lang/en/tool_objectfs.php b/lang/en/tool_objectfs.php index e8f48876..4360da81 100644 --- a/lang/en/tool_objectfs.php +++ b/lang/en/tool_objectfs.php @@ -269,3 +269,22 @@ $string['check:proxyrangerequestsdisabled'] = 'The proxy range request setting is disabled.'; $string['checkproxy_range_request'] = 'Pre-signed URL range request proxy'; + +$string['settings:taggingheader'] = 'Tagging settings'; +$string['settings:taggingenabled'] = 'Tagging enabled'; +$string['checktagging_status'] = 'Object tagging'; +$string['check:tagging:ok'] = 'Object tagging ok'; +$string['check:tagging:na'] = 'Tagging not enabled or is not supported by file system'; +$string['check:tagging:error'] = 'Error trying to tag object'; +$string['settings:maxtaggingperrun'] = 'Object tagging adhoc sync maximum objects per run'; +$string['settings:maxtaggingperrun:desc'] = 'The maximum number of objects to sync tags for per tagging sync adhoc task iteration.'; +$string['settings:maxtaggingiterations'] = 'Object tagging adhoc sync maximum number of iterations '; +$string['settings:maxtaggingiterations:desc'] = 'The maximum number of times the tagging sync adhoc task will requeue itself. To avoid accidental infinite runaway.'; +$string['settings:overrideobjecttags'] = 'Allow object tag override'; +$string['settings:overrideobjecttags:desc'] = 'Allows ObjectFS to overwrite tags on objects that already exist in the external store.'; +$string['tagsource:environment'] = 'Environment defined by $CFG->objectfs_environment_name, currently: "{$a}".'; +$string['tagsource:mimetype'] = 'File mimetype as stored in {files} table'; +$string['settings:tagsources'] = 'Tag sources'; +$string['task:triggerupdateobjecttags'] = 'Queue adhoc task to update object tags'; +$string['settings:tagging:help'] = 'Object tagging allows extra metadata to be attached to objects in the external store. Please read TAGGING.md in the plugin Github repository for detailed setup and considerations. This is currently only supported by the S3 external client.'; +$string['object_status:tag_count'] = 'Object tags'; diff --git a/lib.php b/lib.php index 05e08fb0..7321f499 100644 --- a/lib.php +++ b/lib.php @@ -24,6 +24,7 @@ */ use tool_objectfs\local\object_manipulator\manipulator_builder; +use tool_objectfs\local\tag\tag_manager; define('OBJECTFS_PLUGIN_NAME', 'tool_objectfs'); @@ -120,11 +121,13 @@ function tool_objectfs_pluginfile($course, $cm, context $context, $filearea, arr * @return array */ function tool_objectfs_status_checks() { + $checks = [ + new tool_objectfs\check\tagging_status(), + ]; + if (get_config('tool_objectfs', 'proxyrangerequests')) { - return [ - new tool_objectfs\check\proxy_range_request(), - ]; + $checks[] = new tool_objectfs\check\proxy_range_request(); } - return []; + return $checks; } diff --git a/settings.php b/settings.php index a941e506..10b8bba2 100644 --- a/settings.php +++ b/settings.php @@ -23,6 +23,9 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\task\update_object_tags; + defined('MOODLE_INTERNAL') || die(); require_once(__DIR__ . '/classes/local/manager.php'); @@ -125,7 +128,6 @@ $settings->add(new admin_setting_configduration('tool_objectfs/consistencydelay', new lang_string('settings:consistencydelay', 'tool_objectfs'), '', 10 * MINSECS, MINSECS)); - $settings->add(new admin_setting_heading('tool_objectfs/storagefilesystemselection', new lang_string('settings:clientselection:header', 'tool_objectfs'), '')); @@ -249,4 +251,42 @@ $settings->add(new admin_setting_configcheckbox('tool_objectfs/preferexternal', new lang_string('settings:preferexternal', 'tool_objectfs'), '', '')); + + // Tagging settings. + $settings->add(new admin_setting_heading('tool_objectfs/taggingsettings', + new lang_string('settings:taggingheader', 'tool_objectfs'), '')); + + $settings->add(new admin_setting_description('tool_objectfs/tagginghelp', + '', + get_string('settings:tagging:help', 'tool_objectfs') + )); + + $settings->add(new admin_setting_configcheckbox('tool_objectfs/taggingenabled', + new lang_string('settings:taggingenabled', 'tool_objectfs'), '', 0)); + + $settings->add(new admin_setting_description('tool_objectfs/tagsources', + new lang_string('settings:tagsources', 'tool_objectfs'), + tag_manager::get_tag_summary_html() + )); + + $settings->add(new admin_setting_configtext('tool_objectfs/maxtaggingperrun', + new lang_string('settings:maxtaggingperrun', 'tool_objectfs'), + get_string('settings:maxtaggingperrun:desc', 'tool_objectfs'), + 10000, + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('tool_objectfs/maxtaggingiterations', + new lang_string('settings:maxtaggingiterations', 'tool_objectfs'), + get_string('settings:maxtaggingiterations:desc', 'tool_objectfs'), + 1000, + PARAM_INT + )); + + $settings->add(new admin_setting_configcheckbox('tool_objectfs/overwriteobjecttags', + new lang_string('settings:overrideobjecttags', 'tool_objectfs'), + get_string('settings:overrideobjecttags:desc', 'tool_objectfs'), + 1 + )); + } diff --git a/tests/local/report/object_status_test.php b/tests/local/report/object_status_test.php index bbc895d4..a06caa53 100644 --- a/tests/local/report/object_status_test.php +++ b/tests/local/report/object_status_test.php @@ -66,7 +66,7 @@ public function test_generate_status_report_historic() { public function test_get_report_types() { $reporttypes = objectfs_report::get_report_types(); $this->assertEquals('array', gettype($reporttypes)); - $this->assertEquals(3, count($reporttypes)); + $this->assertEquals(4, count($reporttypes)); } /** diff --git a/tests/local/tagging_test.php b/tests/local/tagging_test.php new file mode 100644 index 00000000..95443451 --- /dev/null +++ b/tests/local/tagging_test.php @@ -0,0 +1,340 @@ +. + +namespace tool_objectfs\local; + +use coding_exception; +use Throwable; +use tool_objectfs\local\manager; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\local\tag\tag_source; +use tool_objectfs\tests\testcase; + +/** + * Tests tagging + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tagging_test extends testcase { + /** + * Tests get_defined_tag_sources + * @covers \tool_objectfs\local\tag_manager::get_defined_tag_sources + */ + public function test_get_defined_tag_sources() { + $sources = tag_manager::get_defined_tag_sources(); + $this->assertIsArray($sources); + + // Both AWS and Azure limit 10 tags per object, so ensure never more than 10 sources defined. + $this->assertLessThanOrEqual(10, count($sources)); + } + + /** + * Provides values to various tag source tests + * @return array + */ + public static function tag_source_provider(): array { + $sources = tag_manager::get_defined_tag_sources(); + $tests = []; + + foreach ($sources as $source) { + $tests[$source->get_identifier()] = [ + 'source' => $source, + ]; + } + + return $tests; + } + + /** + * Tests the source identifier + * @param tag_source $source + * @dataProvider tag_source_provider + * @covers \tool_objectfs\local\tag_source::get_identifier + */ + public function test_tag_sources_identifier(tag_source $source) { + $count = strlen($source->get_identifier()); + + // Ensure < 32 chars, the max length as defined in our docs. + $this->assertLessThan(32, $count); + $this->assertGreaterThan(0, $count); + } + + /** + * Tests the source value + * @param tag_source $source + * @dataProvider tag_source_provider + * @covers \tool_objectfs\local\tag_source::get_value_for_contenthash + */ + public function test_tag_sources_value(tag_source $source) { + $file = $this->create_duplicated_object('tag source value test ' . $source->get_identifier()); + $value = $source->get_value_for_contenthash($file->contenthash); + + // Null value - allowed, but means we cannot test. + if (is_null($value)) { + return; + } + + $count = strlen($value); + + // Ensure < 128 chars, the max length as defined in our docs. + $this->assertLessThan(128, $count); + $this->assertGreaterThan(0, $count); + } + + /** + * Provides values to test_is_tagging_enabled_and_supported + * @return array + */ + public static function is_tagging_enabled_and_supported_provider(): array { + return [ + 'neither config nor fs supports' => [ + 'enabledinconfig' => false, + 'supportedbyfs' => false, + 'expected' => false, + ], + 'enabled in config but fs does not support' => [ + 'enabledinconfig' => true, + 'supportedbyfs' => false, + 'expected' => false, + ], + 'enabled in config and fs does support' => [ + 'enabledinconfig' => true, + 'supportedbyfs' => true, + 'expected' => true, + ], + ]; + } + + /** + * Tests is_tagging_enabled_and_supported + * @param bool $enabledinconfig if tagging feature is turned on + * @param bool $supportedbyfs if the filesystem supports tagging + * @param bool $expected expected return result + * @dataProvider is_tagging_enabled_and_supported_provider + * @covers \tool_objectfs\local\tag_manager::is_tagging_enabled_and_supported + */ + public function test_is_tagging_enabled_and_supported(bool $enabledinconfig, bool $supportedbyfs, bool $expected) { + global $CFG; + // Set config. + set_config('taggingenabled', $enabledinconfig, 'tool_objectfs'); + + // Set supported by fs. + $config = manager::get_objectfs_config(); + $config->taggingenabled = $enabledinconfig; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + $CFG->phpunit_objectfs_supports_object_tagging = $supportedbyfs; + + $this->assertEquals($expected, tag_manager::is_tagging_enabled_and_supported()); + } + + /** + * Tests gather_object_tags_for_upload + * @covers \tool_objectfs\local\tag_manager::gather_object_tags_for_upload + */ + public function test_gather_object_tags_for_upload() { + $object = $this->create_duplicated_object('gather tags for upload test'); + $tags = tag_manager::gather_object_tags_for_upload($object->contenthash); + + $this->assertArrayHasKey('mimetype', $tags); + $this->assertArrayHasKey('environment', $tags); + $this->assertEquals('text', $tags['mimetype']); + $this->assertEquals('test', $tags['environment']); + } + + /** + * Tests store_tags_locally + * @covers \tool_objectfs\local\tag_manager::store_tags_locally + */ + public function test_store_tags_locally() { + global $DB; + + $tags = [ + 'test1' => 'abc', + 'test2' => 'xyz', + ]; + $hash = 'thisisatest'; + + // Ensure no tags for hash intially. + $this->assertEmpty($DB->get_records('tool_objectfs_object_tags', ['contenthash' => $hash])); + + // Store. + tag_manager::store_tags_locally($hash, $tags); + + // Confirm they are stored. + $queriedtags = $DB->get_records('tool_objectfs_object_tags', ['contenthash' => $hash]); + $this->assertCount(2, $queriedtags); + $tagtimebefore = current($queriedtags)->timemodified; + + // Re-store, confirm times changed. + $this->waitForSecond(); + tag_manager::store_tags_locally($hash, $tags); + $queriedtags = $DB->get_records('tool_objectfs_object_tags', ['contenthash' => $hash]); + $tagtimeafter = current($queriedtags)->timemodified; + + $this->assertNotSame($tagtimebefore, $tagtimeafter); + } + + /** + * Provides values to test_get_objects_needing_sync + * @return array + */ + public static function get_objects_needing_sync_provider(): array { + return [ + 'duplicated, needs sync' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => true, + ], + 'remote, needs sync' => [ + 'location' => OBJECT_LOCATION_EXTERNAL, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => true, + ], + 'local, needs sync' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_NEEDS_SYNC, + 'expectedneedssync' => false, + ], + 'duplicated, does not need sync' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED, + 'expectedneedssync' => false, + ], + 'local, does not need sync' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED, + 'expectedneedssync' => false, + ], + 'duplicated, sync error' => [ + 'location' => OBJECT_LOCATION_DUPLICATED, + 'status' => tag_manager::SYNC_STATUS_ERROR, + 'expectedneedssync' => false, + ], + 'local, sync error' => [ + 'location' => OBJECT_LOCATION_LOCAL, + 'status' => tag_manager::SYNC_STATUS_ERROR, + 'expectedneedssync' => false, + ], + ]; + } + + /** + * Tests get_objects_needing_sync + * @param int $location object location + * @param int $syncstatus sync status to set on object record + * @param bool $expectedneedssync if the object should be included in the return of the function + * @dataProvider get_objects_needing_sync_provider + * @covers \tool_objectfs\local\tag_manager::get_objects_needing_sync + */ + public function test_get_objects_needing_sync(int $location, int $syncstatus, bool $expectedneedssync) { + global $DB; + + // Create the test object at the required location. + switch ($location) { + case OBJECT_LOCATION_DUPLICATED: + $object = $this->create_duplicated_object('tagging test object duplicated'); + break; + case OBJECT_LOCATION_LOCAL: + $object = $this->create_local_object('tagging test object local'); + break; + case OBJECT_LOCATION_EXTERNAL: + $object = $this->create_remote_object('tagging test object remote'); + break; + default: + throw new coding_exception("Object location not handled in test"); + } + + // Set the sync status. + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', $syncstatus, ['id' => $object->id]); + + // Check if it is included in the list. + $needssync = tag_manager::get_objects_needing_sync(1); + + if ($expectedneedssync) { + $this->assertContains($object->contenthash, $needssync); + } else { + $this->assertNotContains($object->contenthash, $needssync); + } + } + + /** + * Tests the limit input to get_objects_needing_sync + * @covers \tool_objectfs\local\tag_manager::get_objects_needing_sync + */ + public function test_get_objects_needing_sync_limit() { + global $DB; + + // Create two duplicated objects needing sync. + $object = $this->create_duplicated_object('sync limit test duplicated'); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + $object = $this->create_remote_object('sync limit test remote'); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + + // Ensure a limit of 2 returns 2, and limit of 1 returns 1. + $this->assertCount(2, tag_manager::get_objects_needing_sync(2)); + $this->assertCount(1, tag_manager::get_objects_needing_sync(1)); + } + + /** + * Test get_tag_summary_html + * @covers \tool_objectfs\local\tag_manager::get_tag_summary_html + */ + public function test_get_tag_summary_html() { + // Quick test just to ensure it generates and nothing explodes. + $html = tag_manager::get_tag_summary_html(); + $this->assertIsString($html); + } + + /** + * Tests when fails to sync object tags, that the sync status is updated to SYNC_STATUS_ERROR. + */ + public function test_object_tag_sync_error() { + global $CFG, $DB; + + // Setup FS for tagging. + $config = manager::get_objectfs_config(); + $config->taggingenabled = true; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + manager::set_objectfs_config($config); + $CFG->phpunit_objectfs_supports_object_tagging = true; + $this->assertTrue(tag_manager::is_tagging_enabled_and_supported()); + + // Create a good duplicated object. + $object = $this->create_duplicated_object('sync limit test duplicated'); + $status = $DB->get_field('tool_objectfs_objects', 'tagsyncstatus', ['id' => $object->id]); + $this->assertEquals(tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED, $status); + + // Now try push tags, but trigger a simulated tag set error. + $CFG->phpunit_objectfs_simulate_tag_set_error = true; + $didthrow = false; + try { + $this->filesystem->push_object_tags($object->contenthash); + } catch (Throwable $e) { + $didthrow = true; + } + $this->assertTrue($didthrow); + + // Ensure tag sync status set to error. + $status = $DB->get_field('tool_objectfs_objects', 'tagsyncstatus', ['id' => $object->id]); + $this->assertEquals(tag_manager::SYNC_STATUS_ERROR, $status); + } +} diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 0ccd2a5a..6b8cf9b8 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -16,8 +16,10 @@ namespace tool_objectfs; +use coding_exception; use tool_objectfs\local\store\object_file_system; use tool_objectfs\local\manager; +use tool_objectfs\local\tag\tag_manager; use tool_objectfs\tests\test_file_system; /** @@ -1016,4 +1018,106 @@ public function test_add_file_from_string_update_object_fail() { $this->assertEquals(\core_text::strlen($content), $result[1]); $this->assertTrue($result[2]); } + + /** + * Test syncing tags throws exception when client does not support tagging. + */ + public function test_push_object_tags_not_supported() { + global $CFG; + $CFG->phpunit_objectfs_supports_object_tagging = false; + $this->expectException(coding_exception::class); + $this->expectExceptionMessage('Cannot sync tags, external client does not support tagging'); + $this->filesystem->push_object_tags('123'); + } + + /** + * Tests syncing object tags where the file is not replicated. + */ + public function test_push_object_tags_object_not_replicated() { + global $CFG, $DB; + $CFG->phpunit_objectfs_supports_object_tagging = true; + + // Create object - not replicated to 'external' store yet. + $object = $this->create_local_object('test syncing local'); + + // Sync, this should do nothing but change sync status - cannot sync object tags + // where the object is not replicated. + $this->filesystem->push_object_tags($object->contenthash); + $object = $DB->get_record('tool_objectfs_objects', ['contenthash' => $object->contenthash]); + $this->assertEquals($object->tagsyncstatus, tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED); + } + + /** + * Provides values to push_object_tags_replicated + * @return array + */ + public static function push_object_tags_replicated_provider(): array { + return [ + 'can override' => [ + 'can override' => true, + ], + 'cannot override' => [ + 'cannot override' => false, + ], + ]; + } + + /** + * Tests push_object_tags when the object is replicated. + * @param bool $canoverride if filesystem should be able to overwrite existing objects + * @dataProvider push_object_tags_replicated_provider + */ + public function test_push_object_tags_replicated(bool $canoverride) { + global $CFG, $DB; + $CFG->phpunit_objectfs_supports_object_tagging = true; + + set_config('overwriteobjecttags', $canoverride, 'tool_objectfs'); + $this->assertEquals($canoverride, tag_manager::can_overwrite_object_tags()); + + $object = $this->create_duplicated_object('test syncing replicated'); + + $testtags = [ + 'test' => 123, + 'test2' => 123, + 'test3' => 123, + 'test4' => 123, + ]; + + // Fake set the tags in the external store. + $this->filesystem->get_external_client()->tags[$object->contenthash] = $testtags; + + // Ensure tags are set 'externally'. + $tags = $this->filesystem->get_external_client()->get_object_tags($object->contenthash); + $this->assertCount(count($testtags), $tags); + + // But tags will not be stored locally (yet). + $localtags = $DB->get_records('tool_objectfs_object_tags', ['contenthash' => $object->contenthash]); + $this->assertCount(0, $localtags); + + // Sync the file. + $this->filesystem->push_object_tags($object->contenthash); + + // Tags should now be replicated locally. + $localtags = $DB->get_records('tool_objectfs_object_tags', ['contenthash' => $object->contenthash]); + $externaltags = $this->filesystem->get_external_client()->get_object_tags($object->contenthash); + + if ($canoverride) { + // If can override, we expect it to be overwritten by the tags defined in the sources. + $expectednum = count(tag_manager::get_defined_tag_sources()); + $this->assertCount($expectednum, $localtags); + + // Also expect the external store to be updated. + $this->assertCount($expectednum, $externaltags); + } else { + // If cannot overwrite, no tags should be synced. + $this->assertCount(0, $localtags); + + // External store should not be changed. + $this->assertCount(count($testtags), $externaltags); + } + + // Ensure status changed to not needing sync. + $object = $DB->get_record('tool_objectfs_objects', ['contenthash' => $object->contenthash]); + $this->assertEquals($object->tagsyncstatus, tag_manager::SYNC_STATUS_SYNC_NOT_REQUIRED); + } } diff --git a/tests/task/populate_objects_filesize_test.php b/tests/task/populate_objects_filesize_test.php index 9ff08ee0..dd1402e1 100644 --- a/tests/task/populate_objects_filesize_test.php +++ b/tests/task/populate_objects_filesize_test.php @@ -179,6 +179,7 @@ public function test_that_non_null_values_are_not_updated() { */ public function test_orphaned_objects_are_not_updated() { global $DB; + $objects = $DB->get_records('tool_objectfs_objects'); $file1 = $this->create_local_file("Test 1"); $this->create_local_file("Test 2"); $this->create_local_file("Test 3"); diff --git a/tests/task/trigger_update_object_tags_test.php b/tests/task/trigger_update_object_tags_test.php new file mode 100644 index 00000000..cd3082d8 --- /dev/null +++ b/tests/task/trigger_update_object_tags_test.php @@ -0,0 +1,49 @@ +. + +namespace tool_objectfs\task; + +use advanced_testcase; +use core\task\manager; + +/** + * Tests trigger_update_object_tags + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class trigger_update_object_tags_test extends advanced_testcase { + /** + * Tests executing scheduled task. + */ + public function test_execute() { + $this->resetAfterTest(); + + $task = new trigger_update_object_tags(); + $task->execute(); + + // Ensure it spawned an adhoc task. + $queuedadhoctasks = manager::get_adhoc_tasks(update_object_tags::class); + $this->assertCount(1, $queuedadhoctasks); + + // Ensure the adhoc task spawned has an iteration of 1. + $adhoctask = current($queuedadhoctasks); + $this->assertNotEmpty($adhoctask->get_custom_data()); + $this->assertEquals(1, $adhoctask->get_custom_data()->iteration); + } +} diff --git a/tests/task/update_object_tags_test.php b/tests/task/update_object_tags_test.php new file mode 100644 index 00000000..f4866aba --- /dev/null +++ b/tests/task/update_object_tags_test.php @@ -0,0 +1,170 @@ +. + +namespace tool_objectfs\task; + +use core\task\manager; +use tool_objectfs\local\tag\tag_manager; +use tool_objectfs\tests\testcase; + +/** + * Tests update_object_tags + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class update_object_tags_test extends testcase { + /** + * Enables tagging in config and sets up the filesystem to allow tagging + */ + private function set_tagging_enabled() { + global $CFG; + $config = \tool_objectfs\local\manager::get_objectfs_config(); + $config->taggingenabled = true; + $config->enabletasks = true; + $config->filesystem = '\\tool_objectfs\\tests\\test_file_system'; + \tool_objectfs\local\manager::set_objectfs_config($config); + $CFG->phpunit_objectfs_supports_object_tagging = true; + } + + /** + * Creates object with tags needing to be synced + * @param string $contents contents of object to create. + * @return stdClass object record + */ + private function create_object_needing_tag_sync(string $contents) { + global $DB; + $object = $this->create_duplicated_object($contents); + $DB->set_field('tool_objectfs_objects', 'tagsyncstatus', tag_manager::SYNC_STATUS_NEEDS_SYNC, ['id' => $object->id]); + return $object; + } + + /** + * Tests task exits when the tagging feature is disabled. + */ + public function test_not_enabled() { + $this->resetAfterTest(); + + // By default filesystem does not support and tagging not enabled, so should error. + $task = new update_object_tags(); + + $this->expectOutputString("Tagging feature not enabled or supported by filesystem, exiting.\n"); + $task->execute(); + } + + /** + * Tests handles an invalid iteration limit + */ + public function test_invalid_iteration_limit() { + $this->resetAfterTest(); + $this->set_tagging_enabled(); + + // This should be greater than 1, if zero should error. + set_config('maxtaggingiterations', 0, 'tool_objectfs'); + + // Give it a valid iteration number though. + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 5]); + + $this->expectOutputString("Invalid number of iterations, exiting.\n"); + $task->execute(); + } + + /** + * Tests handles an invalid number of iterations in custom data + */ + public function test_invalid_iteration_number() { + $this->resetAfterTest(); + $this->set_tagging_enabled(); + + // Give it a valid max iteration number. + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + + // But don't set the iteration number on the customdata at all. + $task = new update_object_tags(); + $this->expectOutputString("Invalid number of iterations, exiting.\n"); + $task->execute(); + } + + /** + * Tests exits when there are no more objects needing to be synced + */ + public function test_no_more_objects_to_sync() { + $this->resetAfterTest(); + $this->set_tagging_enabled(); + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 1]); + + $this->expectOutputString("No more objects found that need tagging, exiting.\n"); + $task->execute(); + } + + /** + * Tests maxtaggingiterations is correctly checked + */ + public function test_max_iterations() { + $this->resetAfterTest(); + $this->set_tagging_enabled(); + + // Set max 1 iteration. + set_config('maxtaggingiterations', 1, 'tool_objectfs'); + set_config('maxtaggingperrun', 100, 'tool_objectfs'); + + $task = new update_object_tags(); + + // Give it an iteration number higher. + $task->set_custom_data(['iteration' => 5]); + + $this->expectOutputString("Maximum number of iterations reached: 5, exiting.\n"); + $task->execute(); + } + + /** + * Tests a successful tagging run where it needs to requeue for further processing + */ + public function test_tagging_run_with_requeue() { + $this->resetAfterTest(); + $this->set_tagging_enabled(); + + // Set max 1 object per run. + set_config('maxtaggingperrun', 1, 'tool_objectfs'); + set_config('maxtaggingiterations', 5, 'tool_objectfs'); + + // Create two objects needing sync. + $this->create_object_needing_tag_sync('object 1'); + $this->create_object_needing_tag_sync('object 2'); + $this->assertCount(2, tag_manager::get_objects_needing_sync(100)); + + $task = new update_object_tags(); + $task->set_custom_data(['iteration' => 1]); + + $this->expectOutputString("Requeing self for another iteration.\n"); + $task->execute(); + + // Ensure that 1 object had its sync status updated. + $this->assertCount(1, tag_manager::get_objects_needing_sync(100)); + + // Ensure there is another task that was re-queued with the iteration incremented. + $tasks = manager::get_adhoc_tasks(update_object_tags::class); + $this->assertCount(1, $tasks); + $task = current($tasks); + $this->assertNotEmpty($task->get_custom_data()); + $this->assertEquals(2, $task->get_custom_data()->iteration); + } +} diff --git a/version.php b/version.php index bf606bc1..6e567df5 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023051701; // The current plugin version (Date: YYYYMMDDXX). -$plugin->release = 2023051701; // Same as version. +$plugin->version = 2023051702; // The current plugin version (Date: YYYYMMDDXX). +$plugin->release = 2023051702; // Same as version. $plugin->requires = 2020110900; // Requires Filesystem API. $plugin->component = "tool_objectfs"; $plugin->maturity = MATURITY_STABLE;