diff --git a/classes/azure_blob_storage_file_system.php b/classes/azure_blob_storage_file_system.php new file mode 100644 index 00000000..b13ed108 --- /dev/null +++ b/classes/azure_blob_storage_file_system.php @@ -0,0 +1,33 @@ +. + +namespace tool_objectfs; + +use tool_objectfs\local\store\azure_blob_storage\file_system; + +/** + * File system for Azure Blob Storage. + * This file tells objectfs that this storage system is available for use. + * E.g. via $CFG->alternative_file_storage_class // TODO check this config var name + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class azure_blob_storage_file_system extends file_system { + +} diff --git a/classes/local/manager.php b/classes/local/manager.php index 5d791f97..a6f3d456 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -310,6 +310,7 @@ public static function get_available_fs_list() { $filesystems['\tool_objectfs\digitalocean_file_system'] = '\tool_objectfs\digitalocean_file_system'; $filesystems['\tool_objectfs\s3_file_system'] = '\tool_objectfs\s3_file_system'; $filesystems['\tool_objectfs\swift_file_system'] = '\tool_objectfs\swift_file_system'; + $filesystems['\tool_objectfs\azure_blob_storage_file_system'] = '\tool_objectfs\azure_blob_storage_file_system'; foreach ($filesystems as $filesystem) { $clientclass = self::get_client_classname_from_fs($filesystem); diff --git a/classes/local/store/azure/stream_wrapper.php b/classes/local/store/azure/stream_wrapper.php index 47a67a87..142596a0 100644 --- a/classes/local/store/azure/stream_wrapper.php +++ b/classes/local/store/azure/stream_wrapper.php @@ -35,12 +35,13 @@ use GuzzleHttp\Psr7\Stream; use GuzzleHttp\Psr7\CachingStream; -use GuzzleHttp\Psr7; +use GuzzleHttp\Psr7\Utils; use MicrosoftAzure\Storage\Blob\BlobRestProxy; use MicrosoftAzure\Storage\Blob\Models\BlobProperties; use MicrosoftAzure\Storage\Blob\Models\SetBlobPropertiesOptions; use MicrosoftAzure\Storage\Common\Exceptions\ServiceException; use Psr\Http\Message\StreamInterface; +use local_azureblobstorage\api; /** * stream_wrapper @@ -161,7 +162,11 @@ public function stream_flush() { } $hash = hash_final($this->hash); - $md5 = base64_encode(hex2bin($hash)); + $md5 = hex2bin($hash); + + // TODO get rid of getOptions, get the key directly?. + $params = $this->getOptions(true); + $this->getclient()->put_blob($params['Key'], $this->body, $md5); $params = $this->getOptions(true); $params['Body'] = $this->body; @@ -409,7 +414,7 @@ private function getoption($name) { /** * Gets the client. * - * @return BlobRestProxy + * @return api * @throws \RuntimeException if no client has been configured */ private function getclient() { @@ -443,7 +448,7 @@ private function openreadstream() { try { $blob = $client->getBlob($params['Container'], $params['Key']); - $this->body = Psr7\stream_for($blob->getContentStream()); + $this->body = Utils::streamFor($blob->getContentStream()); } catch (ServiceException $e) { // Prevent the client from keeping the request open when the content cannot be found. $response = $e->getResponse(); diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php new file mode 100644 index 00000000..3030778d --- /dev/null +++ b/classes/local/store/azure_blob_storage/client.php @@ -0,0 +1,230 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use GuzzleHttp\Psr7\Utils; +use tool_objectfs\local\store\object_client_base; + +// TODO does this fail when the plugin is not installed ? +use local_azureblobstorage\api; +use stdClass; +use Throwable; + +/** + * Azure blob storage client + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class client extends object_client_base { + + /** @var api $api Azure API */ + protected api $api; + + /** + * Creates object client + * @param stdClass $config / TODO is this maybe null ? + */ + public function __construct($config) { + if (empty($config)) { + parent::__construct($config); + return; + } + + $this->api = new api($config->azure_accountname, $config->azure_container, $config->azure_sastoken); + } + + public function get_availability() { + // Requires local_azureblobstorage to be installed. + $info = \core\plugin_manager::instance()->get_plugin_info('local_azureblobstorage'); + + // Info is empty if plugin is not installed. + return !empty($info); + } + + /** + * Returns the full path for a given file by contenthash + * @param string $contenthash + * @return string filepath + */ + public function get_fullpath_from_hash($contenthash) { + $filepath = $this->get_filepath_from_hash($contenthash); + $container = $this->api->container; + return "blob://$container/$filepath"; + } + + /** + * Returns the filepath from the contenthash, mimicking the + * structure of the filedir storage system. + * + * @param string $contenthash + * @return string filepath + */ + protected function get_filepath_from_hash($contenthash): string { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + return "$l1/$l2/$contenthash"; + } + + /** + * Returns the blob key (the key used to reference the blob) from a given filepath. + * @param string $filepath + * @return string + */ + protected function get_blob_key_from_path(string $filepath): string { + $container = $this->api->container; + return str_replace("blob://$container/", '', $filepath); + } + + /** + * Deletes a given file + * @param string $fullpath + */ + public function delete_file($fullpath) { + $blobkey = $this->get_blob_key_from_path($fullpath); + // TODO call api to delete. + } + + /** + * Renames a given file + * @param string $currentpath + * @param string $destinationpath + */ + public function rename_file($currentpath, $destinationpath) { + // Azure does not support renaming, instead the file is copied + // and the old one is deleted. + copy($currentpath, $destinationpath); + $this->delete_file($currentpath); + } + + /** + * Verifies an object is uploaded correctly. + * In Azure, this is done by checking the md5 hash of the contents. + * + * @param string $contenthash + * @param string $localpath + * @return bool + */ + public function verify_object($contenthash, $localpath) { + // TODO get blob properties + // and check md5 hash is correct. + + return true; + } + + /** + * Returns a stream context used to handle file IO + * @return resource stream resource + */ + public function get_seekable_stream_context() { + $context = stream_context_create([ + 'blob' => [ + 'seekable' => true, + ], + ]); + return $context; + } + + // TODO test_permissions + + // TODO test_connection + + public function test_permissions($testdelete) { + $key = 'permissions_check_test'; + $file = Utils::streamFor('test permission file'); + $filemd5 = hex2bin(md5('test permission file')); + + // Try create a file. + try { + $this->api->put_blob($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + // TODO can we clean this up... this is awful. + 'messages' => [get_string('settings:writefailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + // Try read the file that was created. + try { + $this->api->get_blob($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + // TODO can we clean this up... this is awful. + 'messages' => [get_string('settings:permissionreadfailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + + // If testing delete, try delete the test file. + if ($testdelete) { + try { + // TODO call delete blob, not implemented yet. + } catch (Throwable $e) { + return (object) [ + 'success' => false, + // TODO can we clean this up... this is awful. + 'messages' => [get_string('settings:deleteerror', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + } + + return (object) [ + 'success' => true, + 'messages' => [get_string('settings:permissioncheckpassed', 'tool_objectfs') => 'notifysuccess'], + ]; + } + + public function test_connection() { + // Try to create a file. + try { + $this->api->put_blob('connection_check_test', Utils::streamFor('test contents'), hex2bin(md5('test contents'))); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'details' => $e->getMessage(), + ]; + } + + return (object) [ + 'success' => true, + 'details' => '', + ]; + } + + public function define_client_section($settings, $config) { + $settings->add(new \admin_setting_heading('tool_objectfs/azure', + new \lang_string('settings:azure:header', 'tool_objectfs'), $this->define_client_check())); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_accountname', + new \lang_string('settings:azure:accountname', 'tool_objectfs'), + new \lang_string('settings:azure:accountname_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_container', + new \lang_string('settings:azure:container', 'tool_objectfs'), + new \lang_string('settings:azure:container_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configpasswordunmask('tool_objectfs/azure_sastoken', + new \lang_string('settings:azure:sastoken', 'tool_objectfs'), + new \lang_string('settings:azure:sastoken_help', 'tool_objectfs'), '')); + + return $settings; + } +} \ No newline at end of file diff --git a/classes/local/store/azure_blob_storage/file_system.php b/classes/local/store/azure_blob_storage/file_system.php new file mode 100644 index 00000000..1160e2d3 --- /dev/null +++ b/classes/local/store/azure_blob_storage/file_system.php @@ -0,0 +1,39 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use tool_objectfs\local\store\azure_blob_storage\client; +use tool_objectfs\local\store\object_file_system; + +/** + * Azure blob store file system + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class file_system extends object_file_system { + /** + * Initialise client + * @param mixed $config + * @return client + */ + protected function initialise_external_client($config) { + return new client($config); + } +} \ No newline at end of file diff --git a/classes/local/store/azure_blob_storage/stream_wrapper.php b/classes/local/store/azure_blob_storage/stream_wrapper.php new file mode 100644 index 00000000..98d11aa2 --- /dev/null +++ b/classes/local/store/azure_blob_storage/stream_wrapper.php @@ -0,0 +1,333 @@ +body = null; + $this->hash = null; + } + + /** + * Opens the stream, based on the given mode. + * + * @param mixed $path filepath + * @param mixed $mode stream mode, see + * @param mixed $options + * @param mixed $opened_path + * + * @return bool + */ + public function stream_open($path, $mode, $options, &$openedpath) { + $this->initProtocol($path); + + // TODO rewrite this part, a bit weirdly written. + $this->params = $this->getContainerKey($path); + + // TODO what does this do ? + $this->mode = rtrim($mode, 'bt'); + + if ($errors = $this->validate($path, $this->mode)) { + return $this->triggerError($errors); + } + + $this->hash = hash_init('md5'); + + return $this->boolCall(function() use ($path) { + switch ($this->mode) { + case 'r': + return $this->open_read_stream(); + case 'a': + return $this->open_append_stream(); + default: + return $this->open_write_stream(); + } + }); + } + + private function open_read_stream() { + $client = $this->getClient(); + $params = $this->getOptions(true); + + // TODO not sure if this is correct. + $promise = $client->get_blob($params['Key'])->wait(); + $this->body = $promise->body; + + // TODO implement checking + + // TODO implement CachingStream + + return true; + } + + private function open_append_stream() { + $client = $this->getClient(); + $params = $this->getOptions(true); + + // TODO not sure if this is correct. + $promise = $client->get_blob($params['Key'])->wait(); + $this->body = $promise->body; + $this->body->seek(0, SEEK_END); + return true; + + // TODO fallback to write stream. + } + + private function open_write_stream() { + // Write stream is actually just writing to a temp php file. + // once the stream is flushed, this is then uploaded + // to the Azure blob storage. + $this->body = new Stream(fopen('php://temp', 'r+')); + return true; + } + + /** + * getContainerKey + * @param string $path + * + * @return array + */ + private function getcontainerkey($path) { + // Remove the protocol. + $parts = explode('://', $path); + // Get the container, key. + $parts = explode('/', $parts[1], 2); + + return [ + 'Container' => $parts[0], + 'Key' => isset($parts[1]) ? $parts[1] : null + ]; + } + + /** + * Parse the protocol out of the given path. + * + * @param string $path + */ + private function initprotocol($path) { + $parts = explode('://', $path, 2); + $this->protocol = $parts[0] ?: 'blob'; + } + + /** + * Validates the provided stream arguments for fopen and returns an array + * of errors. + * @param string $path + * @param string $mode + * + * @return [type] + */ + private function validate($path, $mode) { + $errors = []; + + if (!$this->getOption('Key')) { + $errors[] = 'Cannot open a bucket. You must specify a path in the ' + . 'form of blob://container/key'; + } + + if (!in_array($mode, ['r', 'w', 'a', 'x'])) { + $errors[] = "Mode not supported: {$mode}. " + . "Use one 'r', 'w', 'a', or 'x'."; + } + + // When using mode "x" validate if the file exists before attempting to read. + if ($mode == 'x' && false // TODO + // TODO see if already exists + ) { + $errors[] = "{$path} already exists on Azure Blob Storage"; + } + + // When using mode 'r' we should validate the file exists before opening a handle on it. + if ($mode == 'r' && false //TODO + // TODO see if already exists + ) { + $errors[] = "{$path} does not exist on Azure Blob Storage"; + $this->readable = false; + } + + return $errors; + } + + /** + * Get the stream context options available to the current stream + * + * @param bool $removeContextData Set to true to remove contextual kvp's + * like 'client' from the result. + * + * @return array + */ + private function getoptions($removecontextdata = false) { + // Context is not set when doing things like stat. + if ($this->context === null) { + $options = []; + } else { + $options = stream_context_get_options($this->context); + $options = isset($options[$this->protocol]) + ? $options[$this->protocol] + : []; + } + + $default = stream_context_get_options(stream_context_get_default()); + $default = isset($default[$this->protocol]) + ? $default[$this->protocol] + : []; + $result = $this->params + $options + $default; + + if ($removecontextdata) { + unset($result['client'], $result['seekable']); + } + + return $result; + } + + /** + * Gets a URL stat template with default values + * + * @return array + */ + private function getstattemplate() { + // TODO what is this ? Maybe a doc link might be good. + return [ + 0 => 0, 'dev' => 0, + 1 => 0, 'ino' => 0, + 2 => 0, 'mode' => 0, + 3 => 0, 'nlink' => 0, + 4 => 0, 'uid' => 0, + 5 => 0, 'gid' => 0, + 6 => -1, 'rdev' => -1, + 7 => 0, 'size' => 0, + 8 => 0, 'atime' => 0, + 9 => 0, 'mtime' => 0, + 10 => 0, 'ctime' => 0, + 11 => -1, 'blksize' => -1, + 12 => -1, 'blocks' => -1, + ]; + } + + /** + * Trigger one or more errors + * + * @param string|array $errors Errors to trigger + * @param mixed $flags If set to STREAM_URL_STAT_QUIET, then no + * error or exception occurs + * + * @return bool Returns false + * @throws \RuntimeException if throw_errors is true + */ + private function triggererror($errors, $flags = null) { + // TODO explain this a bit better. + // TODO more docs. + + // This is triggered with things like file_exists(). + if ($flags & STREAM_URL_STAT_QUIET) { + return $flags & STREAM_URL_STAT_LINK + // This is triggered for things like is_link(). + ? $this->getStatTemplate() + : false; + } + + // This is triggered when doing things like lstat() or stat(). + trigger_error(implode("\n", (array) $errors), E_USER_WARNING); + + return false; + } + + /** + * Get a specific stream context option + * + * @param string $name Name of the option to retrieve + * + * @return mixed|null + */ + private function getoption($name) { + $options = $this->getOptions(); + + return isset($options[$name]) ? $options[$name] : null; + } + + /** + * Invokes a callable and triggers an error if an exception occurs while + * calling the function. + * + * @param callable $fn + * @param int $flags + * + * @return bool + */ + private function boolcall(callable $fn, $flags = null) { + try { + return $fn(); + } catch (\Exception $e) { + return $this->triggerError($e->getMessage(), $flags); + } + } + + /** + * Gets the client. + * + * @return api + * @throws \RuntimeException if no client has been configured + */ + private function getclient() { + if (!$client = $this->getOption('client')) { + throw new \RuntimeException('No client in stream context'); + } + + return $client; + } +} \ No newline at end of file