s3fs.module

<?php

/**
 * @file
 * Hook implementations and other primary functionality for S3 File System.
 */

use Aws\Credentials\CredentialProvider;
use Aws\S3\S3Client;
use Aws\S3\Exception\S3Exception;
use Aws\Sdk;

/**
 * Class used to differentiate between known and unknown exception states.
 */
class S3fsException extends Exception {
}

/**
 * Implements hook_stream_wrappers().
 * ().
 *
 * Defines the s3:// stream wrapper.
 */
function s3fs_stream_wrappers() {
  return [
    's3' => [
      'name'        => 'S3 File System',
      'class'       => 'S3fsStreamWrapper',
      'description' => t('Amazon Simple Storage Service'),
      'type'        => STREAM_WRAPPERS_NORMAL,
    ],
  ];
}

/**
 * Implements hook_stream_wrappers_alter().
 * ().
 *
 * If configured to do so, s3fs takes control of the public:// stream wrapper.
 */
function s3fs_stream_wrappers_alter(&$wrappers) {
  $config = config('s3fs.settings');
  if (!empty($config->get('s3fs_use_s3_for_public'))) {
    $wrappers['public'] = [
      'name'        => t('Public files (s3fs)'),
      'class'       => 'S3fsStreamWrapper',
      'description' => t('Public files served from Amazon S3.'),
      'type'        => STREAM_WRAPPERS_NORMAL,
    ];
  }
  if (!empty($config->get('s3fs_use_s3_for_private'))) {
    $wrappers['private'] = [
      'name'        => t('Private files (s3fs)'),
      'class'       => 'S3fsStreamWrapper',
      'description' => t('Private files served from Amazon S3.'),
      'type'        => STREAM_WRAPPERS_NORMAL,
      // Required by the file_entity module to let it know that this is a private stream.
      'private'     => TRUE,
    ];
  }
}

/**
 * Implements hook_libraries_info().
 * ().
 */
function s3fs_libraries_info() {
  return array(
    'awssdk' => array(
      'title' => 'AWS SDK for PHP',
      'vendor url' => 'http://docs.aws.amazon.com/aws-sdk-php/guide/latest/index.html',
      'download url' => 'https://github.com/aws/aws-sdk-php/releases',
      'version arguments' => array(
        'file' => 'Aws/Sdk.php',
        'pattern' => "/const VERSION = '(.*)';/",
        'lines' => 500,
      ),
      'files' => array(
        'php' => array(
          'aws-autoloader.php',
        ),
      ),
    ),
  );
}

/**
 * Implements hook_menu().
 * ().
 */
function s3fs_menu() {
  $items = [];

  $items['admin/config/media/s3fs'] = [
    'title'            => 'S3 File System',
    'description'      => 'Configure S3 File System.',
    'page callback'    => 'backdrop_get_form',
    'page arguments'   => ['s3fs_settings'],
    'access arguments' => ['administer s3fs'],
    'file'             => 's3fs.admin.inc',
    'type'             => MENU_NORMAL_ITEM,
  ];
  $items['admin/config/media/s3fs/settings'] = [
    'title'  => 'Settings',
    'type'   => MENU_DEFAULT_LOCAL_TASK,
    'weight' => 0,
  ];
  $items['admin/config/media/s3fs/actions'] = [
    'title'            => 'Actions',
    'description'      => 'Actions for S3 File System.',
    'page callback'    => 'backdrop_get_form',
    'page arguments'   => ['s3fs_actions'],
    'access arguments' => ['administer s3fs'],
    'file'             => 's3fs.admin.inc',
    'type'             => MENU_LOCAL_TASK,
    'weight'           => 10,
  ];
  $items['admin/config/media/s3fs/actions/copy-images'] = [
    'title'            => 'Copy System Images to S3',
    'description'      => 'Copy system image files from modules, themes, and libraries to S3.',
    'page callback'    => 'backdrop_get_form',
    'page arguments'   => ['s3fs_copy_system_images_confirm_form'],
    'access arguments' => ['administer s3fs'],
    'file'             => 's3fs.admin.inc',
    'type'             => MENU_LOCAL_TASK,
  ];
  // A custom version of system/files/styles/%image_style, based on how the
  // core Image module creates image styles with image_style_deliver().
  $items['s3/files/styles/%image_style'] = [
    'title'           => 'Generate image style in S3',
    'page callback'   => '_s3fs_image_style_deliver',
    'page arguments'  => [3],
    'access callback' => TRUE,
    'type'            => MENU_CALLBACK,
  ];

  return $items;
}

/**
 * Implements hook_permission().
 * ().
 */
function s3fs_permission() {
  return [
    'administer s3fs' => [
      'title' => t('Administer S3 File System'),
    ],
  ];
}


/**
 * Implements hook_form_FORM_ID_alter().
 *
 * Disables the "file system path" fields from the system module's
 * file_system_settings form when s3fs is taking them over. They do have an
 * effect if the user makes use of the s3fs-copy-local bee commands, but
 * we don't want users to think of these fields as being meaningful once s3fs
 * has already taken over.
 */
function s3fs_form_system_file_system_settings_alter(
  &$form,
  &$form_state,
  $form_id
) {
  $config = config('s3fs.settings'); if (!empty($config->get
  ('s3fs_use_s3_for_public'))) {
    $form['file_public_path']['#attributes'] = ['disabled' => 'disabled'];
    $form['file_public_path']['#description']
      = 'S3 File System has taken control of the public:// filesystem, making this setting irrelevant for typical use.';
  }

  if (!empty($config->get('s3fs_use_s3_for_private'))) {
    $form['file_private_path']['#attributes'] = ['disabled' => 'disabled'];
    $form['file_private_path']['#description']
      = 'S3 File System has taken control of the private:// filesystem, making this setting irrelevant for typical use.';
  }
}

///////////////////////////////////////////////////////////////////////////////
//                          INTERNAL FUNCTIONS
///////////////////////////////////////////////////////////////////////////////


/**
 * Generates an image derivative in S3.
 *
 * This is a re-write of the core Image module's image_style_deliver() function.
 * It exists to improve the performance of serving newly-created image
 * derivatives from S3.
 *
 * Note to future maintainers: this function is variatic. It accepts two fixed
 * arguments: $style and $scheme, and any number of further arguments, which
 * represent the path to the file in S3 (split on the slashes).
 */
function _s3fs_image_style_deliver($style, $scheme) {
  $config = config('s3fs.settings');
  // Check that the style is defined and the scheme is valid.
  if (!$style || !file_stream_wrapper_valid_scheme($scheme)) {
    backdrop_add_http_header('Status', '400 Bad Request');
    backdrop_add_http_header('Content-Type', 'text/html; charset=utf-8');
    print t('Invalid request.');
    backdrop_exit();
  }
  // Backdrop's black magic calls this function with the image style as arg0,
  // the scheme as arg1, and the full path to the filename split across arg2+.
  // So we need to use PHP's version of variatic functions to get the complete
  // filename.
  $args = func_get_args();
  $style = array_shift($args);
  $scheme = array_shift($args);
  $filename = implode('/', $args);


  $image_uri = "$scheme://$filename";
  $derivative_uri = image_style_path($style['name'], $image_uri);

  // Confirm that the original source image exists before trying to process it.
  if (!is_file($image_uri)) {
    watchdog('s3fs',
      'Source image at %source_image_path not found while trying to generate derivative image at %derivative_path.',
      [
        '%source_image_path' => $image_uri,
        '%derivative_path'   => $derivative_uri,
      ]
    );
    return MENU_NOT_FOUND;
  }

  // Don't start generating the image if the derivative already exists or if
  // generation is in progress in another thread.
  $generation_lock_name = '_s3fs_image_style_deliver:' . $style['name'] . ':'
    . backdrop_hash_base64($image_uri);
  if (!file_exists($derivative_uri)) {
    $generation_lock_acquired = lock_acquire($generation_lock_name);
    if (!$generation_lock_acquired) {
      // Tell client to retry again in 3 seconds. Currently no browsers are known
      // to support Retry-After.
      backdrop_add_http_header('Status', '503 Service Unavailable');
      backdrop_add_http_header('Content-Type', 'text/html; charset=utf-8');
      backdrop_add_http_header('Retry-After', 3);
      print t('Image generation in progress. Try again shortly.');
      backdrop_exit();
    }
  }

  // Try to generate the image, unless another thread just did it while we were
  // acquiring the lock.
  $success = file_exists($derivative_uri);
  if (!$success) {
    // If we successfully generate the derivative, wait until S3 acknowledges
    // its existence. Otherwise, redirecting to it may cause a 403 error.
    $success = image_style_create_derivative(
      $style,
      $image_uri,
      $derivative_uri
    );
    file_stream_wrapper_get_instance_by_scheme('s3')->waitUntilFileExists(
      $derivative_uri
    );
  }

  if (!empty($lock_acquired)) {
    lock_release($generation_lock_name);
  }


  if ($success) {
    if ($config->get('s3fs_no_redirect_derivatives', FALSE)) {
      // If the site admin doesn't want us to redirect to the new derivative, we upload it to the client, instead.
      $image = image_load($derivative_uri);
      $settings = [
        'Content-Type'   => $image->info['mime_type'],
        'Content-Length' => $image->info['file_size'],
      ];
      file_transfer($image->source, $settings);
    }
    else {
      // Perform a 302 Redirect to the new image derivative in S3.
      backdrop_goto(file_create_url($derivative_uri));
    }
  }
  else {
    watchdog(
      'S3 File System',
      'Unable to generate an image derivative at %path.',
      ['%path' => $derivative_uri]
    );
    backdrop_add_http_header('Status', '500 Internal Server Error');
    backdrop_add_http_header('Content-Type', 'text/html; charset=utf-8');
    print t('Error generating image.');
    backdrop_exit();
  }
}

/**
 * Checks all the configuration options to ensure that they're valid.
 *
 * @param array $config
 *   An s3fs configuration array.
 *
 * @return bool
 *   TRUE if config is good to go, otherwise FALSE.
 */
/**
 * Checks all the configuration options to ensure that they're valid.
 *
 * @param array $config
 *   An s3fs configuration array.
 *
 * @return bool
 *   TRUE if config is good to go, otherwise FALSE.
 */
function _s3fs_validate_config($config) {

  $s3fs_use_customhost = config('s3fs.settings')->get('s3fs_use_customhost');
  $s3fs_hostname = config('s3fs.settings')->get('s3fs_hostname');
  $s3fs_use_cname = config('s3fs.settings')->get('s3fs_use_cname');
  $s3fs_domain = config('s3fs.settings')->get('s3fs_domain');
  $s3fs_domain_root = config('s3fs.settings')->get('s3fs_domain_root');
  $s3fs_root_folder = config('s3fs.settings')->get('s3fs_root_folder');
  $s3fs_public_folder = config('s3fs.settings')->get('s3fs_public_folder');
  $s3fs_bucket = config('s3fs.settings')->get('s3fs_bucket');

  if (!empty($s3fs_use_customhost) && empty($s3fs_hostname)) {
    form_set_error(
      's3fs_hostname',
      'You must specify a Hostname to use the Custom Host feature.'
    );
    return FALSE;
  }
  if (!empty($s3fs_use_cname) && empty($s3fs_domain)) {
    form_set_error(
      's3fs_domain',
      'You must specify a CDN Domain Name to use the CNAME feature.'
    );
    return FALSE;
  }

  switch ($s3fs_domain_root) {
    case 'root':
      if (empty($s3fs_root_folder)) {
        form_set_error(
          's3fs_root_folder',
          'You must specify a Root folder to map the Domain Name to it.'
        );
        return FALSE;
      }
      break;
    case 'public':
      if (empty($s3fs_public_folder)) {
        form_set_error(
          's3fs_public_folder',
          'You must specify a Public folder to map the Domain Name to it.'
        );
        return FALSE;
      }
      elseif (!empty($s3fs_root_folder)) {
        form_set_error(
          's3fs_root_folder',
          'For the Public folder option, the Root folder must be blank. Otherwise, use the "Root & Public folders" option.'
        );
        return FALSE;
      }
      break;
    case 'root_public':
      if (empty($s3fs_root_folder) || empty($s3fs_public_folder)) {
        form_set_error(
          's3fs_domain_root',
          'You must specify both Root and Public folders to map the Domain Name to it.'
        );
        return FALSE;
      }
      break;
  }

  try {
    $s3 = _s3fs_get_amazons3_client($config);
  } catch (S3fsException $e) {
    form_set_error('form', $e->getMessage());
    return FALSE;
  }

  // Test the connection to S3, and the bucket name.
  try {
    $list_obj_args = [
      'Bucket'  => $s3fs_bucket,
      'MaxKeys' => 1,
    ];
    if (!empty($s3fs_root_folder)) {
      // If the root_folder option has been set, retrieve from S3 only those files
      // which reside in the root folder.
      $list_obj_args['Prefix'] = "{$s3fs_root_folder}/";
    }
    // listObjects() will trigger descriptive exceptions if the credentials,
    // bucket name, or region are invalid/mismatched.
    $s3->listObjects($list_obj_args);
  } catch (S3Exception $e) {
    form_set_error(
      'form',
      t(
        'An unexpected %exception occurred, with the following error message:<br>%error',
        ['%exception' => $e->getAwsErrorCode(), '%error' => $e->getMessage()]
      )
    );
    return FALSE;
  }

  return TRUE;
}


/**
 * Refreshes the metadata cache.
 *
 * Iterates over the full list of objects in the s3fs_root_folder within S3
 * bucket (or the entire bucket, if no root folder has been set), caching
 * their metadata in the database.
 *
 * It then caches the ancestor folders for those files, since folders are not
 * normally stored as actual objects in S3.
 *
 * @param array $config
 *   An s3fs configuration array.
 */
function _s3fs_refresh_cache($config) {
  if (!_s3fs_validate_config($config)) {
    form_set_error('s3fs_refresh_cache][refresh', t('Unable to validate S3 configuration settings.'));
    return;
  }

  if (function_exists('bee_message')) {
    bee_message(bt('Getting Amazon S3 client...'), 'log');
  }
  if (function_exists('drush_log')) {
    drush_log('Getting Amazon S3 client...');
  }

  $s3 = _s3fs_get_amazons3_client($config);

  // Load the s3fs configuration settings once before the loop
  $s3fs_config = config('s3fs.settings');
  $bucket = $s3fs_config->get('s3fs_bucket');
  $root_folder = $s3fs_config->get('s3fs_root_folder');
  $public_folder_name = $s3fs_config->get('s3fs_public_folder', 's3fs-public');
  $private_folder_name = $s3fs_config->get('s3fs_private_folder', 's3fs-private');
  $use_versioning = $s3fs_config->get('s3fs_use_versioning');

  $iterator_args = ['Bucket' => $bucket];
  if (!empty($root_folder)) {
    $iterator_args['Prefix'] = $root_folder . '/';
  }

  $op_name = $use_versioning ? 'ListObjectVersions' : 'ListObjects';
  $iterator = $s3->getPaginator($op_name, $iterator_args);

  if (function_exists('bee_message')) {
    bee_message(bt('Creating temporary tables...'), 'log');
  }
  if (function_exists('drush_log')) {
    drush_log('Creating temporary tables...');
  }

  // Create the temp table, into which all the refreshed data will be written.
  // After the full refresh is complete, the temp table will be swapped with
  // the real one.
  module_load_install('s3fs');
  $schema = s3fs_schema();
  try {
    db_create_table('s3fs_file_temp', $schema['s3fs_file']);
  } catch (DatabaseSchemaObjectExistsException $e) {
    // The table already exists, so we can simply truncate it to start fresh.
    db_truncate('s3fs_file_temp')->execute();
  }
  // Create temporary table for folders which will allow for duplicates.
  // Folders will be written at the same time as the file data is written,
  // then will be merged with the files at the end.
  try {
    $folder_schema = $schema['s3fs_file'];
    unset($folder_schema['primary key'], $folder_schema['indexes']);

    db_create_table('s3fs_folder_temp', $folder_schema);
    $options = Database::getConnectionInfo('default');
    switch ($options['default']['driver']) {
      case 'pgsql':
        break;
      case 'sqlite':
        break;
      case 'mysql':
        db_query(
          'ALTER TABLE {s3fs_folder_temp} CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'
        );
        break;
    }
  } catch (DatabaseSchemaObjectExistsException $e) {
    db_truncate('s3fs_folder_temp')->execute();
  }

  // The $folders array is an associative array keyed by folder paths, which
  // is constructed as each filename is written to the DB. After all the files
  // are written, the folder paths are converted to metadata and written.
  $folders = [];
  $file_metadata_list = [];
  // Start by gathering all the existing folders. If we didn't do this, empty
  // folders would be lost, because they'd have no files from which to rebuild
  // themselves.
  $existing_folders = db_select('s3fs_file', 's')
    ->fields('s', ['uri'])
    ->condition('dir', 1, '=');

  $folder_counter = 0;

  foreach ($existing_folders->execute()->fetchCol(0) as $folder_uri) {
    $folders[rtrim($folder_uri, '/')] = TRUE;
    if ($folder_counter++ % 1000 == 0) {
      _s3fs_write_metadata($file_metadata_list, $folders);
    }
  }

  foreach ($iterator as $result) {
    if (!isset($result['Contents'])) {
      continue;
    }
    foreach ($result['Contents'] as $s3_metadata) {
      $key = $s3_metadata['Key'];
      // The root folder is an implementation detail that only appears on S3.
      // Files' URIs are not aware of it, so we need to remove it beforehand.
      if (!empty($root_folder)) {
        $key = str_replace($root_folder . '/', '', $key);
      }


      if (strpos($key, "$public_folder_name/") === 0) {
        $key = str_replace("$public_folder_name/", '', $key);
        $uri = "public://$key";
      } elseif (strpos($key, "$private_folder_name/") === 0) {
        $key = str_replace("$private_folder_name/", '', $key);
        $uri = "private://$key";
      } else {
        $uri = "s3://$key";
      }

      $max_uri_length = $schema['s3fs_file']['fields']['uri']['length'];
      if (strlen($uri) >= $max_uri_length) {
        watchdog(
          's3fs',
          'URI "@uri" is too long, ignoring',
          ['@uri' => $uri],
          WATCHDOG_WARNING
        );
        continue;
      }

      if ($uri[strlen($uri) - 1] == '/') {
        // Treat objects in S3 whose filenames end in a '/' as folders.
        // But don't store the '/' itself as part of the folder's uri.
        $folders[rtrim($uri, '/')] = TRUE;
      }
      else {
        // Only store the metadata for the latest version of the file.
        if (isset($s3_metadata['IsLatest']) && !$s3_metadata['IsLatest']) {
          continue;
        }
        // Files with no StorageClass are actually from the DeleteMarkers list,
        // rather then the Versions list. They represent a file which has been
        // deleted, so don't cache them.
        if (!isset($s3_metadata['StorageClass'])) {
          continue;
        }
        // Buckets with Versioning disabled set all files' VersionIds to "null".
        // If we see that, unset VersionId to prevent "null" from being written
        // to the DB.
        if (isset($s3_metadata['VersionId'])
          && $s3_metadata['VersionId'] == 'null'
        ) {
          unset($s3_metadata['VersionId']);
        }
        $file_metadata_list[] = _s3fs_convert_metadata($uri, $s3_metadata);
      }
      _s3fs_write_metadata($file_metadata_list, $folders);
    }
  }

  // Write folders.
  $query = db_select('s3fs_folder_temp')->distinct();
  $query->fields('s3fs_folder_temp');
  $folder_counter = 0;

  $result = $query->execute();

  $insert_query = db_insert('s3fs_file_temp')
    ->fields(['uri', 'filesize', 'timestamp', 'dir', 'version']);

  foreach ($result as $record) {
    $insert_query->values((array) $record);
    // Flush every 1000 records.
    if ($folder_counter++ % 1000 == 0) {
      $insert_query->execute();
    }
  }

  // Write any remainders.
  $insert_query->execute();
  if (function_exists('bee_message')) {
    bee_message(bt('Flushed @folders folders to the file table.', array(
      '@folders' => $folder_counter,
    )), 'log');
  }
  if (function_exists('drush_log')) {
    drush_log(dt('Flushed @folders folders to the file table.', array(
      '@folders' => $folder_counter,
    )));
  }

  // Cleanup.
  db_drop_table('s3fs_folder_temp');

  // Swap the temp table with the real table.
  db_rename_table('s3fs_file', 's3fs_file_old');
  db_rename_table('s3fs_file_temp', 's3fs_file');
  db_drop_table('s3fs_file_old');


  if (function_exists('drush_log')) {
    drush_log(dt('S3 File System cache refreshed.'));
  }
  if (function_exists('bee_message')) {
    bee_message(bt('S3 File System cache refreshed.'));
  }
}

/**
 * Writes metadata to the temp table in the database.
 *
 * @param array $file_metadata_list
 *   An array passed by reference, which contains the current page of file
 *   metadata. This function empties out $file_metadata_list at the end.
 * @param array $folders
 *   An associative array keyed by folder name, which is populated with the
 *   ancestor folders of each file in $file_metadata_list. Also emptied.
 */
function _s3fs_write_metadata(&$file_metadata_list, &$folders) {
  $insert_query = db_insert('s3fs_file_temp')
    ->fields(['uri', 'filesize', 'timestamp', 'dir', 'version']);

  foreach ($file_metadata_list as $key => $metadata) {
    $uri = backdrop_dirname($metadata['uri']);

    // Check if the URI already exists in the table.
    $exists = db_select('s3fs_file_temp', 'f')
      ->fields('f', ['uri'])
      ->condition('uri', $metadata['uri'])
      ->execute()
      ->fetchField();

    if (!$exists) {
      if (!$metadata['dir']) {
        $insert_query->values($metadata);
      } else {
        $folders[$uri] = $metadata;
        unset($file_metadata_list[$key]);
      }
    }

    $root = file_uri_scheme($uri) . '://';
    while ($uri != $root) {
      $folders[rtrim($uri, '/')] = TRUE;
      $uri = backdrop_dirname($uri);
    }
  }

  if (!empty($file_metadata_list)) {
    $insert_query->execute();
    if (function_exists('bee_message')) {
      bee_message('  ' . bt('Wrote @files file(s).', array('@files' => count($file_metadata_list))), 'log');
    }
    if (function_exists('drush_log')) {
      drush_log('  ' . dt('Wrote @files file(s).', array('@files' => count($file_metadata_list))));
    }
    $file_metadata_list = [];
  }

  // Folder handling
  if (!empty($folders)) {
    $insert_folder_query = db_insert('s3fs_folder_temp')
      ->fields(['uri', 'filesize', 'timestamp', 'dir', 'version']);

    foreach ($folders as $folder_name => $folder_data) {
      if (is_bool($folder_data)) {
        $insert_folder_query->values(_s3fs_convert_metadata($folder_name, []));
      } else {
        $insert_folder_query->values($folder_data);
      }
    }
    $insert_folder_query->execute();
    if (function_exists('bee_message')) {
      bee_message('  ' . bt('Wrote @folders folder(s).', array('@folders' => count($folders))), 'log');
    }
    if (function_exists('drush_log')) {
      drush_log('  ' . dt('Wrote @folders folder(s).', array('@folders' => count($folders))));
    }
    $folders = [];
  }
}


/**
 * Convert file metadata returned from S3 into a metadata cache array.
 *
 * @param string $uri
 *   The uri of the resource.
 * @param array  $s3_metadata
 *   An array containing the collective metadata for the object in S3.
 *   The caller may send an empty array here to indicate that the returned
 *   metadata should represent a directory.
 *
 * @return array
 *   A file metadata cache array.
 */
function _s3fs_convert_metadata($uri, $s3_metadata) {
  // Need to fill in a default value for everything, so that DB calls
  // won't complain about missing fields.
  $metadata = [
    'uri'       => $uri,
    'filesize'  => 0,
    'timestamp' => REQUEST_TIME,
    'dir'       => 0,
    'version'   => '',
  ];

  if (empty($s3_metadata)) {
    // The caller wants directory metadata.
    $metadata['dir'] = 1;
  }
  else {
    // The filesize value can come from either the Size or ContentLength
    // attribute, depending on which AWS API call built $s3_metadata.
    if (isset($s3_metadata['ContentLength'])) {
      $metadata['filesize'] = $s3_metadata['ContentLength'];
    }
    elseif (isset($s3_metadata['Size'])) {
      $metadata['filesize'] = $s3_metadata['Size'];
    }

    if (isset($s3_metadata['LastModified'])) {
      $metadata['timestamp'] = date(
        'U',
        strtotime($s3_metadata['LastModified'])
      );
    }

    if (isset($s3_metadata['VersionId'])
      && $s3_metadata['VersionId'] != 'null'
    ) {
      $metadata['version'] = $s3_metadata['VersionId'];
    }
  }
  return $metadata;
}

/**
 * Sets up the S3Client object.
 *
 * For performance reasons, only one S3Client object will ever be created
 * within a single request.
 *
 * @param array $config
 *   Array of configuration settings from which to configure the client.
 *
 * @return Aws\S3\S3Client
 *   The fully-configured S3Client object.
 */
function _s3fs_get_amazons3_client() {
  static $s3;
  static $static_config;
  $config = config('s3fs.settings');
  // If the client hasn't been set up yet, or the config given to this call is
  // different from the previous call, (re)build the client.
  if (!isset($s3) || $static_config != $config) {
    // Check if the keys managed by the Key module exist.
    if (module_exists('key')) {
      // Load the keys from the Key module configuration.
      $key_config = config('key.module');

      // Get the key values using key_get_key_value().
      $access_key = key_get_key_value($config->get('keymodule_access_key_name'));
      $secret_key = key_get_key_value($config->get('keymodule_secret_key_name'));

      if ($access_key === null || $secret_key === null) {
        // If the keys are not set in the Key module configuration, fall back to S3FS configuration.
        $access_key = $config->get('s3fs_awssdk_access_key');
        $secret_key = $config->get('s3fs_awssdk_secret_key');
      }

    } else {
      // Get the key values using key_get_key_value().
      $access_key = $config->get('s3fs_awssdk_access_key') ?: $config->get('keymodule_access_key_name');
      $secret_key = $config->get('s3fs_awssdk_secret_key') ?: $config->get('keymodule_secret_key_name');
    }

    $use_instance_profile = $config->get('s3fs_use_instance_profile');
    $credentials_file = $config->get('s3fs_credentials_file');

    // Check if AWS SDK is loaded and accessible.
    $library = _s3fs_load_awssdk_library();
    if (!$library['loaded']) {
      throw new S3fsException(
        t(
          'Unable to load the AWS SDK. Please ensure that the awssdk2 library is installed correctly.'
        )
      );
    }
    elseif (!class_exists('Aws\S3\S3Client')) {
      throw new S3fsException(
        t(
          'Cannot load Aws\S3\S3Client class. Please ensure that the AWS SDK library is installed correctly.'
        )
      );
    }

    // Create the Aws\S3\S3Client object.
    $client_config = [];
    // If we have configured credentials locally use them, otherwise let
    // the SDK find them per API docs.
    // @see https://docs.aws.amazon.com/aws-sdk-php/v3/guide/guide/configuration.html
    if ($use_instance_profile) {
      // If defined path use that otherwise SDK will check home directory.
      if ($credentials_file) {
        $provider = CredentialProvider::ini(NULL, $credentials_file);
      }
      else {
        // Assume an instance profile provider if no path.
        $provider = CredentialProvider::instanceProfile();
      }
      // Cache the results in a memoize function to avoid loading and parsing
      // the ini file on every API operation.
      // @see https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials_provider.html
      $provider = CredentialProvider::memoize($provider);
      $client_config['credentials'] = $provider;
    }
    // Use defined keys.
    elseif (!empty($access_key) && !empty($secret_key)) {
      $client_config['credentials'] = [
        'key'    => $access_key,
        'secret' => $secret_key,
      ];
    }
    if (!empty($config->get('s3fs_region'))) {
      $client_config['region'] = $config->get('s3fs_region');
      // Signature v4 is only required in the Beijing and Frankfurt regions.
      // Also, setting it will throw an exception if a region hasn't been set.
      $client_config['signature'] = 'v4';
    }
    if (!empty($config->get('s3fs_use_customhost')) && !empty
      ($config->get('s3fs_hostname'))) {
      $client_config['use_path_style_endpoint'] = false;
      $client_config['endpoint'] = $config->get('s3fs_hostname');
    }
    // Create the Aws\S3\S3Client object with the specified configuration.
    // S3 Service only supports 2006-03-01 API version currently. V3 requires
    // an explicit version declaration, and use of 'latest' is discouraged.
    $client_config['version'] = '2006-03-01';
    $s3 = new S3Client($client_config);
  }
  $static_config = $config;
  return $s3;
}

/**
 * Returns the current s3fs configuration settings.
 *
 * The functions in S3 File System which utilize variables always accept a
 * config array instead of calling variable_get() themselves. This allows for
 * their callers to override these configuration settings when necessary (like
 * when attempting to validate new settings).
 *
 * @param $reset bool
 *               This function uses a static cache for performance reasons.
 *               Passing TRUE will reset that cache.
 *
 * @return array
 *   An associative array of all the s3fs_* config settings, with the "s3fs_"
 *   prefix removed from their names. Also includes any awssdk_ prefixed vars,
 *   with their prefix left intact.
 */
function _s3fs_get_config($reset = FALSE) {
  $config = &backdrop_static('s3fs_settings');
  if ($config === NULL || $reset) {
    // The global $conf array contains all the variables, including overrides
    // from settings.php.
    global $conf;
    $config = [];
    foreach ($conf as $key => $value) {
      // Retrieve the s3fs_ prefixed vars, and strip the prefix.
      if (substr($key, 0, 5) == 's3fs_') {
        $config[substr($key, 5)] = $value;
      }
    }
    foreach ($conf as $key => $value) {
      // Retrieve the awssdk_ prefixed vars, but don't strip the prefix.
      // These will override any s3fs_awssdk_ prefixed vars.
      if (substr($key, 0, 7) == 'awssdk_') {
        $config[$key] = $value;
      }
    }
    // Remove any leading or trailing slashes from these settings, in case the user added them.
    if (!empty($config['s3fs_root_folder'])) {
      $config['s3fs_root_folder'] = trim($config['s3fs_root_folder'], '\/');
    }
    if (!empty($config['s3fs_public_folder'])) {
      $config['s3fs_public_folder'] = trim($config['s3fs_public_folder'], '\/');
    }
    if (!empty($config['s3fs_private_folder'])) {
      $config['s3fs_private_folder'] = trim($config['s3fs_private_folder'], '\/');
    }
  }
  return $config;
}

/**
 * Internal function to retrieve the value of a specific setting, taking
 * overrides in settings.php into account.
 *
 * This function is most useful on the config form and for retrieving the AWS
 * SDK settings.
 * _s3fs_get_config() should be used in most other cases.
 *
 * @param string $setting
 *   The short name of the setting. e.g. the "s3fs_use_cname" variable's short
 *   name is "use_cname".
 */
function _s3fs_get_setting($setting, $default = '') {
  $config = _s3fs_get_config();
  // Get the value from _s3fs_get_config(), if it's set. This will include any overrides from settings.php, including
  // the awssdk_ prefixed vars.
  return !empty($config[$setting]) ? $config[$setting] : $default;
}

/**
 * Loads the AWS SDK library.
 *
 * This function is a replacement for calling libraries_load('awssdk'). It's
 * needed because libraries_load() caches failures to load the library, meaning
 * that temporarily having a bad setup (e.g. nonexistent or unreadable files
 * in the awssdk folder) can lead to the library being permanently unable to
 * be loaded, even after the bad setup is repaired. This can only be remedied
 * by clearing the full site cache.
 *
 * This is especially disastrous when upgrading the AWS SDK library on a
 * system that is currently using it, because if the upgrade results in a bad
 * setup, the site cache may become impossible to clear. If some other module's
 * data has been cached in S3 (e.g. ctools css cache), the cache clearing
 * process itself will attempt to use S3FS. But if the Libraries cache has not
 * yet been cleared by this time, it will continue to insist that AWS SDK is not
 * installed, and the cache clear will crash because s3fs can't function
 * without the AWS SDK library. This leaves the site in an unrecoverable broken
 * state.
 *
 * @return array
 *   The array returned by libraries_load('awssdk'), as if it used no cache.
 */
function _s3fs_load_awssdk_library() {
  $loaded = include_once(backdrop_get_path('module', 's3fs')
    . '/vendor/autoload.php');
  return ['loaded' => $loaded];
}

/**
 * Copies all the local files from the specified file system into S3.
 */
function _s3fs_copy_file_system_to_s3($scheme) {
  $config = config('s3fs.settings');
  if ($scheme == 'public') {
    $source_folder = realpath(
      config_get('system.core', 'file_public_path')
    );
  }
  elseif ($scheme == 'private') {
    $source_folder = config_get('system.core', 'file_private_path');
    $source_folder_real = realpath($source_folder);
    if (empty($source_folder) || empty($source_folder_real)) {
      bee_message (bt(
        'Private file system base path is unknown. Unable to perform S3 copy.',
        'error'
      ));
      return;
    }
  }

  $file_paths = _s3fs_recursive_dir_scan($source_folder);
  foreach ($file_paths as $path) {
    $relative_path = str_replace($source_folder . '/', '', $path);
    print "Copying $scheme://$relative_path into S3...\n";
    // Finally get to make use of S3fsStreamWrapper's "S3 is actually a local
    // file system. No really!" functionality.
    copy($path, "$scheme://$relative_path");
  }
  if (function_exists('bee_message')) {
    bee_message(bt('Copied all local %scheme files to S3.', ['%scheme' => $scheme]),
      'status'
    );
  }
}

function _s3fs_recursive_dir_scan($dir) {
  $output = [];
  $files = scandir($dir);
  foreach ($files as $file) {
    $path = "$dir/$file";

    if ($file != '.' && $file != '..') {
      // In case they put their private root folder inside their public one,
      // skip it. When listing the private file system contents, $path will
      // never trigger this.
      if ($path == realpath(
          config_get('system.core', 'file_private_path')
        )
      ) {
        continue;
      }

      if (is_dir($path)) {
        $output = array_merge($output, _s3fs_recursive_dir_scan($path));
      }
      else {
        $output[] = $path;
      }
    }
  }
  return $output;
}

/*
 * Confirmation form displaying system images to be deleted from S3 if the process is continued.
 */
/**
 * Implements hook_config_info().
 */
function s3fs_config_info() {
  $prefixes['s3fs.settings'] = [
    'label' => t('S3 File System settings'),
    'group' => t('Configuration'),
  ];
  return $prefixes;
}

/**
 * @see  http://drupal.org/node/1354
 * @todo Please document this function.
 */
function s3fs_copy_system_images_confirm_form($form, &$form_state) {
  $config = config('s3fs.settings');
  if (!$config->get('s3fs_no_rewrite_cssjs')) {
    backdrop_goto('admin/config/media/s3fs/actions');
  }

  // Active stream wrapper.
  $wrapper = config_get('system.core', 'file_default_scheme') . '://';
  $directories = s3fs_copy_system_images_directories();

  if (!empty($directories['root_dirs'])) {
    $description = t(
      '<h2><strong><span class="warning">Warning: Unrelated files may be deleted. Proceed with caution.</span></strong></h2>'
    );
    $description .= t(
      '
      <p>All files located in the S3 directories shown below will be deleted as part of this copy process. If you have stored files in
      one of these locations that is not related to S3 system images, <strong>cancel this process immediately and resolve the conflicts
      before proceeding.</strong> If a large number of modules are installed, this process can take several minutes to complete.</p>'
    );
    $description .= t(
      '<h3>!count images to be copied to S3:</h3>',
      ['!count' => $directories['total']]
    );
    $description .= t('<h4>Summary</h4>');
    $description .= '<ul>';
    foreach ($directories['root_dirs'] as $root_dir => $root_count) {
      $description .= '<li>' . $wrapper . $root_dir . ' (' . $root_count
        . ')</li>';
    }
    $description .= '</ul><br>';
    $description .= t('<h3>Details</h3>');
    $description .= '<ul>';
    foreach ($directories['dirs'] as $dir => $count) {
      $description .= '<li>' . $wrapper . $dir . ' (' . $count . ')</li>';
    }
    $description .= '</ul><br>';
  }
  else {
    $description = t(
      '<p>Copy system images from modules, themes, and libraries to the S3 File System.</p>
      <p>If a large number of modules are installed, this process can take several minutes to complete.</p>'
    );
  }

  $form = confirm_form(
    $form,
    t('Copy System Images to S3'),
    ['path' => 'admin/config/media/s3fs/actions'],
    $description
  );

  // Pass data to form submit through build_info.
  $form_state['build_info']['system_stream_wrapper'] = $wrapper;
  $form_state['build_info']['system_image_directories'] = $directories;
  // Set submit function.
  $form['actions']['submit']['#submit']
    = ['s3fs_copy_system_images_confirm_form_submit'];

  return $form;
}

/*
 * Form submit to remove previously saved system images from S3 and copy new ones.
 */
function s3fs_copy_system_images_confirm_form_submit($form, &$form_state) {
  $wrapper = $form_state['build_info']['system_stream_wrapper'];
  $directories = $form_state['build_info']['system_image_directories'];
  s3fs_copy_system_images_batch_set($wrapper, $directories);
  if (function_exists('bee_message')) {
    bee_message(bt(
      'Copying of system images from modules, themes, and libraries to S3 is complete.'));
  }
  $form_state['redirect'] = array('path' => 'admin/config/media/s3fs/actions');
}

/**
 * Set batch process to perform copy of system images.
 *
 * @param string $wrapper
 *   System stream wrapper used with S3.
 * @param array $directories
 *   Directories where system images are stored.
 */
function s3fs_copy_system_images_batch_set($wrapper, $directories) {
  $operations = array();
  // Use root directories to recursively delete all images within them.
  foreach ($directories['s3fs_root_dirs'] as $directory => $count) {
    $operations[] = array('s3fs_copy_system_images_delete_batch', array($wrapper, $directory));
  }
  // Use individual directories for identifying where images need to be saved.
  foreach ($directories['dirs'] as $directory => $count) {
    $operations[] = array('s3fs_copy_system_images_save_batch', array($wrapper, $directory));
  }

  $batch = array(
    'title' => t('Copying system images to S3'),
    'operations' => $operations,
    'init_message' => t('Starting process of copying system files to S3.'),
    'error_message' => t('Copying system files to S3 has encountered an error.'),
  );
  batch_set($batch);
}

/**
 * Build array of modules, themes, and libraries
 * that may contain system images.
 *
 * @return array
 *   Associative array containing all active sub-directories and
 *   root-level directories where system images exist, counts for
 *   each directory, and an overall total of files to be copied.
 */
function s3fs_copy_system_images_directories() {
  // Add Backdrop's default misc and modules directories.
  $dirs = array('core/misc', 'modules');
  // Add directories of enabled modules.
  foreach (module_list() as $module_name => $module) {
    $dirs[] = backdrop_get_path('module', $module_name);
  }
  // Add directories of available themes.
  foreach (list_themes() as $theme_name => $theme) {
    $dirs[] = backdrop_get_path('theme', $theme_name);
  }
  // Remove duplicate directories.
  $dirs = array_values(array_unique($dirs));
  // Remove directories without images, fonts, txt, or css files, and add counts for those that have them.
  $file_extensions = 'png|gif|jpe?g|svg|bmp|eot|ttf|woff|woff2|txt|css'; //
  // Include image, font, txt, and css extensions here
  $file_pattern = '/^.*\.(' . $file_extensions . ')$/';
  $image_dirs = array();
  foreach ($dirs as $dir) {
    if ($files = file_scan_directory($dir, $file_pattern)) {
      $image_dirs[$dir] = count($files);
    }
  }

  // Create an array of root-level directories where files reside, and counts for each directory.
  $root_image_dirs = array();
  foreach ($image_dirs as $image_dir => $image_count) {
    $dir_split = explode('/', $image_dir);
    if (array_key_exists($dir_split[0], $root_image_dirs)) {
      $root_image_dirs[$dir_split[0]] = $root_image_dirs[$dir_split[0]] + $image_count;
    }
    else {
      $root_image_dirs[$dir_split[0]] = $image_count;
    }
  }

  // Sort arrays alphabetically for display.
  ksort($image_dirs);
  ksort($root_image_dirs);

  return array(
    'total' => array_sum($image_dirs),
    'dirs' => $image_dirs,
    'root_dirs' => $root_image_dirs,
  );
}


/**
 * Delete previously saved system images from S3. This is done to ensure files
 * have been removed from modules or themes that are no longer in use.
 *
 * @param string $wrapper
 *   Default file system stream wrapper.
 * @param array $directory
 *   A root-level directory of a module, theme, or
 *   library where system images may exist on S3.
 */
function s3fs_copy_system_images_delete_batch($wrapper, $directory) {
  file_unmanaged_delete_recursive($wrapper . $directory);
}


/**
 * Save system images to S3 for a list of directories.
 *
 * @param string $wrapper
 *   Default file system stream wrapper.
 * @param array  $directory
 *   A directory of a module, theme, or library
 *   where system images are to be saved.
 */
function s3fs_copy_system_images_save_batch($wrapper, $directory) {

  // Scan for images in the relevant directory.
  $local_files = file_scan_directory($directory, '/^.*\.(png|gif|jpe?g|svg|bmp|eot|ttf|woff|woff2|txt|css)$/');

  // Loop through each local file. "local_file_uri" is path + filename (ex: 'misc/farbtastic/marker.png')
  foreach ($local_files as $local_file_uri => $local_file) {

    // Determine file path for each selected file.
    $remote_file_path = str_replace('/' . $local_file->filename, '', $local_file_uri);


    // Use file path to make directories. Only create if it doesn't exist.
    if (!is_dir($wrapper . $remote_file_path)) {
      backdrop_mkdir($wrapper . $remote_file_path, NULL, TRUE, NULL);
    }

    // Copy file from the original file path to its location in the stream wrapper.
    file_unmanaged_copy($local_file_uri, $wrapper . $local_file_uri, FILE_EXISTS_REPLACE);
  }
}


/**
 * Implements hook_field_default_field_bases_alter().
 *
 * Allows a variable to override all exported field bases to use 'Amazon S3' as
 * the Upload destination. For example this can be added to environment-specific
 * Backdrop settings files, to allow certain environments to upload to S3 while
 * other environments upload to the exported (public or private) URI scheme:
 * @code
 * $conf['s3fs_file_uri_scheme_override'] = 's3';
 * @endcode
 */
function s3fs_field_default_field_bases_alter(&$fields) {
  if ($uri_scheme = config_get(
    's3fs.settings',
    's3fs_file_uri_scheme_override'
  )
  ) {
    foreach ($fields as $key => $item) {
      if (isset($item['settings']['uri_scheme'])) {
        $fields[$key]['settings']['uri_scheme'] = $uri_scheme;
      }
    }
  }
}

/**
 * Implements hook_autoload_info().
 */
function s3fs_autoload_info() {
  return [
    'S3fsStreamWrapper' => 'includes/S3fsStreamWrapper.inc',
    'S3fsException'     => 's3fs.module',
  ];
}