From a74e9ba6ead2d396975ed8964be02590d97d2941 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Tue, 18 Aug 2020 13:43:15 +0300 Subject: [PATCH 01/13] Init of migration server for shared content --- ..._plus.migration_group.virtual_y_shared.yml | 7 ++ ...migration.virtual_y_shared_media_video.yml | 85 +++++++++++++++++++ ....migration.virtual_y_shared_node_video.yml | 80 +++++++++++++++++ .../src/SourceMigrationDeriver.php | 84 ++++++++++++++++++ 4 files changed, 256 insertions(+) create mode 100644 modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml create mode 100644 modules/openy_gc_shared_content/src/SourceMigrationDeriver.php diff --git a/modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml b/modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml new file mode 100644 index 000000000..642e3f388 --- /dev/null +++ b/modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml @@ -0,0 +1,7 @@ +langcode: en +status: true +dependencies: { } +id: virtual_y_shared +label: 'Virtual Y Shared Content Server sync' +description: 'Imports shared items from Virtual Y sources to the main Demo Server.' +module: openy_gc_shared_content diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml new file mode 100644 index 000000000..de47cec68 --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml @@ -0,0 +1,85 @@ +dependencies: + module: + - openy_media_video + enforced: + module: + - openy_gc_demo +id: virtual_y_shared_media_video +migration_tags: { } +migration_group: virtual_y_shared +label: Import Virtual Y videos +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +source: + json_includes: + - field_gc_video_media + plugin: url + track_changes: true + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: name + label: 'Name' + selector: /attributes/name + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: type + label: 'Type' + selector: /type + - name: field_media_in_library + label: 'Media in library' + selector: /attributes/field_media_in_library + - name: field_media_source + label: 'Media source' + selector: /attributes/field_media_source + - name: field_media_video_embed_field + label: 'Video Embed' + selector: /attributes/field_media_video_embed_field + - name: field_media_video_id + label: 'Video ID' + selector: /attributes/field_media_video_id + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + name: + - + plugin: get + source: name + bundle: + - + plugin: default_value + default_value: video + uid: + - + plugin: default_value + default_value: 1 + field_media_video_embed_field: + - + plugin: get + source: field_media_video_embed_field + +destination: + plugin: 'entity:media' +migration_dependencies: + required: { } + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml new file mode 100644 index 000000000..846b3547b --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -0,0 +1,80 @@ +langcode: en +status: true +dependencies: + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_node_video +migration_tags: + - openy_complete_installation +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: 'Sync shared videos to Open Y shared demo server.' +source: + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: data/ + fields: + - name: nid + label: 'nid' + selector: /attributes/drupal_internal__nid + - name: title + label: 'Title' + selector: /attributes/title + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: field_gc_video_media_id + label: 'Video' + selector: /relationships/field_gc_video_media/data/id + - name: field_gc_video_description + label: 'Video description' + selector: /attributes/field_gc_video_description/value + - name: field_gc_video_duration + label: 'Duration' + selector: /attributes/field_gc_video_duration + ids: + nid: + type: integer +process: + langcode: + plugin: default_value + source: language + default_value: en + status: status + created: created + changed: changed + uid: + plugin: default_value + default_value: 1 + title: title + field_gc_video_description/value: field_gc_video_description + field_gc_video_description/format: + plugin: default_value + default_value: full_html + field_gc_video_duration: field_gc_video_duration + field_gc_video_media: + plugin: migration + migration: virtual_y_shared_media_video:REPLACE_ME + source: field_gc_video_media_id +destination: + plugin: 'entity:node' + default_bundle: gc_video + +migration_dependencies: + required: + - virtual_y_shared_media_video:REPLACE_ME + optional: { } diff --git a/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php new file mode 100644 index 000000000..493211cd1 --- /dev/null +++ b/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php @@ -0,0 +1,84 @@ + implode(',', $this->getRemoteRelationshipsList($base_plugin_definition)), + 'sort[sortByDate][path]' => 'created', + 'sort[sortByDate][direction]' => 'DESC', + 'filter[status]' => 1, + //@TODO add shared_content_filter once it will be at test servers. + //'limit' => 1000 + ]; + + $jsonapi_uri = '/jsonapi/node/gc_video?' . http_build_query($params); + + foreach ($urls as $url) { + $url_long = $url . $jsonapi_uri; + $derivative = $this->getDerivativeValues($base_plugin_definition, $url_long, $url); + //dump($url_long); + $this->derivatives[$this->getKey($url)] = $derivative; + } + + return $this->derivatives; + } + + /** + * Private method that dynamically updates migration source. + * + * @param array $base_plugin_definition + * Plugin settings. + * @param $url + * Dynamic url for every Virtual Y content source. + * + * @return arrayx + */ + private function getDerivativeValues(array $base_plugin_definition, $url_long, $url) { + + $base_plugin_definition['source']['urls'] = $url_long; + if (!empty($base_plugin_definition['process']['field_gc_video_media'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_gc_video_media']['migration'] + ); + $base_plugin_definition['process']['field_gc_video_media']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'] = [$migration]; + } + + return $base_plugin_definition; + } + + private function getRemoteRelationshipsList($base_plugin_definition) { + return !empty($base_plugin_definition['source']['json_includes']) ? $base_plugin_definition['source']['json_includes'] : []; + } + + private function getKey($url) { + $url_key = str_replace('http://', '', $url); + $url_key = str_replace('.', '_', $url_key); + return $url_key; + } + +} \ No newline at end of file From c1be686b6e2ebf2e0c5ad5df80bddd8cda3fedf0 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 01:05:39 +0300 Subject: [PATCH 02/13] Updated deriver for all shared migrations --- .../src/SourceMigrationDeriver.php | 64 +++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php index 493211cd1..6f63dee9a 100644 --- a/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php @@ -30,15 +30,14 @@ public function getDerivativeDefinitions($base_plugin_definition) { 'sort[sortByDate][direction]' => 'DESC', 'filter[status]' => 1, //@TODO add shared_content_filter once it will be at test servers. - //'limit' => 1000 + //'limit' => 100 ]; - $jsonapi_uri = '/jsonapi/node/gc_video?' . http_build_query($params); + $jsonapi_uri = '/jsonapi/node/' . $base_plugin_definition['source']['entity_type'] . '?' . http_build_query($params); foreach ($urls as $url) { $url_long = $url . $jsonapi_uri; $derivative = $this->getDerivativeValues($base_plugin_definition, $url_long, $url); - //dump($url_long); $this->derivatives[$this->getKey($url)] = $derivative; } @@ -50,14 +49,17 @@ public function getDerivativeDefinitions($base_plugin_definition) { * * @param array $base_plugin_definition * Plugin settings. + * @param $url_long + * Url with request part * @param $url * Dynamic url for every Virtual Y content source. * - * @return arrayx + * @return array */ private function getDerivativeValues(array $base_plugin_definition, $url_long, $url) { $base_plugin_definition['source']['urls'] = $url_long; + if (!empty($base_plugin_definition['process']['field_gc_video_media'])) { $migration = str_replace( 'REPLACE_ME', @@ -68,17 +70,69 @@ private function getDerivativeValues(array $base_plugin_definition, $url_long, $ $base_plugin_definition['migration_dependencies']['required'] = [$migration]; } + if (!empty($base_plugin_definition['process']['field_vy_blog_image'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_vy_blog_image']['migration'] + ); + $base_plugin_definition['process']['field_vy_blog_image']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'] = [$migration]; + } + + if (!empty($base_plugin_definition['process']['field_media_image'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_media_image']['migration'] + ); + $base_plugin_definition['process']['field_media_image']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'] = [$migration]; + } + + if (!empty($base_plugin_definition['source']['constants']['DOMAIN'])) { + $base_plugin_definition['source']['constants']['DOMAIN'] = $url; + } + + if (!empty($base_plugin_definition['process']['field_gc_video_image'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_gc_video_image']['migration'] + ); + $base_plugin_definition['process']['field_gc_video_image']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'][] = $migration; + } + return $base_plugin_definition; } + /** + * Helper function that checks if we need includes to jsonapi request. + * + * @param $base_plugin_definition + * Migration array. + * + * @return array + * Includes array for JSON:API + */ private function getRemoteRelationshipsList($base_plugin_definition) { return !empty($base_plugin_definition['source']['json_includes']) ? $base_plugin_definition['source']['json_includes'] : []; } + /** + * Helper function that prepare entity key from url. + * + * @param $url + * Url to source. + * + * @return mixed + * Machine name url for key. + */ private function getKey($url) { $url_key = str_replace('http://', '', $url); $url_key = str_replace('.', '_', $url_key); return $url_key; } -} \ No newline at end of file +} From 1db809411994f391eec4258f3733e0d12be8320e Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 01:06:10 +0300 Subject: [PATCH 03/13] Migrations for shared content --- ...s.migration.virtual_y_shared_file_blog.yml | 109 ++++++++++++++++++ ....migration.virtual_y_shared_file_video.yml | 109 ++++++++++++++++++ ...tion.virtual_y_shared_media_blog_image.yml | 77 +++++++++++++ ...migration.virtual_y_shared_media_video.yml | 1 + ...ion.virtual_y_shared_media_video_image.yml | 77 +++++++++++++ ...s.migration.virtual_y_shared_node_blog.yml | 77 +++++++++++++ ....migration.virtual_y_shared_node_video.yml | 12 +- 7 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml create mode 100644 modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml new file mode 100644 index 000000000..0eea896db --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml @@ -0,0 +1,109 @@ +dependencies: + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_file_blog +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: 'Import Shared VirtualY files' +migration_group: virtual_y_shared +source: + entity_type: vy_blog_post + json_includes: + - field_vy_blog_image.field_media_image + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: type + label: 'Include type' + selector: /type + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: filename + label: 'FileName' + selector: /attributes/filename + - name: filemime + label: 'FileMIME' + selector: /attributes/filemime + - name: url + label: 'File url' + selector: /attributes/uri/url + ids: + id: + type: string + constants: + DOMAIN: replace_me + DRUPAL_FILE_DIRECTORY: 'public://virtual-y-shared-images/' +process: + type: + plugin: skip_on_value + method: row + source: type + value: media--image + langcode: + plugin: default_value + source: language + default_value: en + created: created + changed: changed + pseudo_source_full_path: + - + plugin: concat + delimiter: / + source: + - constants/DOMAIN + - url + - + plugin: urlencode + pseudo_destination_path: + - plugin: concat + source: + - constants/DRUPAL_FILE_DIRECTORY + - filename + uri_file: + - + plugin: concat + delimiter: / + source: + - constants/uri_file + - name + - + plugin: urlencode + filename: filename + uri: + plugin: file_copy + source: + - '@pseudo_source_full_path' + - '@pseudo_destination_path' + file_exists: rename + move: false + status: + - + plugin: default_value + default_value: 1 + uid: + - + plugin: default_value + default_value: 1 +destination: + plugin: 'entity:file' +migration_dependencies: + required: { } + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml new file mode 100644 index 000000000..3bdae7a5c --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml @@ -0,0 +1,109 @@ +dependencies: + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_file_video +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: 'Import Shared VirtualY files' +migration_group: virtual_y_shared +source: + entity_type: gc_video + json_includes: + - field_gc_video_image.field_media_image + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: type + label: 'Include type' + selector: /type + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: filename + label: 'FileName' + selector: /attributes/filename + - name: filemime + label: 'FileMIME' + selector: /attributes/filemime + - name: url + label: 'File url' + selector: /attributes/uri/url + ids: + id: + type: string + constants: + DOMAIN: replace_me + DRUPAL_FILE_DIRECTORY: 'public://virtual-y-shared-images/' +process: + type: + plugin: skip_on_value + method: row + source: type + value: media--image + langcode: + plugin: default_value + source: language + default_value: en + created: created + changed: changed + pseudo_source_full_path: + - + plugin: concat + delimiter: / + source: + - constants/DOMAIN + - url + - + plugin: urlencode + pseudo_destination_path: + - plugin: concat + source: + - constants/DRUPAL_FILE_DIRECTORY + - filename + uri_file: + - + plugin: concat + delimiter: / + source: + - constants/uri_file + - name + - + plugin: urlencode + filename: filename + uri: + plugin: file_copy + source: + - '@pseudo_source_full_path' + - '@pseudo_destination_path' + file_exists: rename + move: false + status: + - + plugin: default_value + default_value: 1 + uid: + - + plugin: default_value + default_value: 1 +destination: + plugin: 'entity:file' +migration_dependencies: + required: { } + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml new file mode 100644 index 000000000..a57b9c24b --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml @@ -0,0 +1,77 @@ +dependencies: + module: + - openy_media_image + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_media_blog_image +migration_tags: { } +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: Fetch Image +source: + entity_type: vy_blog_post + json_includes: + - field_vy_blog_image.field_media_image + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: type + label: 'Include type' + selector: /type + - name: name + label: 'Media Name' + selector: /attributes/name + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: field_media_image + label: 'File' + selector: /relationships/field_media_image/data/id + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + created: created + changed: changed + name: name + type: + plugin: skip_on_value + method: row + source: type + value: file--file + bundle: + plugin: default_value + default_value: image + uid: + plugin: default_value + default_value: 1 + field_media_image: + plugin: migration + migration: virtual_y_shared_file_blog:REPLACE_ME + source: field_media_image +destination: + plugin: 'entity:media' +migration_dependencies: + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml index de47cec68..423cff24d 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml @@ -10,6 +10,7 @@ migration_group: virtual_y_shared label: Import Virtual Y videos deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver source: + entity_type: gc_video json_includes: - field_gc_video_media plugin: url diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml new file mode 100644 index 000000000..504d7c77c --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml @@ -0,0 +1,77 @@ +dependencies: + module: + - openy_media_image + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_media_video_image +migration_tags: { } +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: Fetch Image +source: + entity_type: gc_video + json_includes: + - field_gc_video_image.field_media_image + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: type + label: 'Include type' + selector: /type + - name: name + label: 'Media Name' + selector: /attributes/name + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: field_media_image + label: 'File' + selector: /relationships/field_media_image/data/id + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + created: created + changed: changed + name: name + type: + plugin: skip_on_value + method: row + source: type + value: file--file + bundle: + plugin: default_value + default_value: image + uid: + plugin: default_value + default_value: 1 + field_media_image: + plugin: migration + migration: virtual_y_shared_file_video:REPLACE_ME + source: field_media_image +destination: + plugin: 'entity:media' +migration_dependencies: + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml new file mode 100644 index 000000000..e4c2895e0 --- /dev/null +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml @@ -0,0 +1,77 @@ +langcode: en +status: true +dependencies: + enforced: + module: + - openy_gc_shared_content +id: virtual_y_shared_node_blog +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +label: 'Sync shared blog posts to Open Y shared demo server.' +source: + entity_type: vy_blog_post + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + item_selector: data/ + fields: + - name: nid + label: 'nid' + selector: /attributes/drupal_internal__nid + - name: title + label: 'Title' + selector: /attributes/title + - name: created + label: 'Created' + selector: /attributes/created + - name: changed + label: 'Changed' + selector: /attributes/changed + - name: status + label: 'Status' + selector: /attributes/status + - name: field_vy_blog_image_id + label: 'Image' + selector: /relationships/field_vy_blog_image/data/id + - name: field_vy_blog_description + label: 'Blog description' + selector: /attributes/field_vy_blog_description/value + ids: + nid: + type: integer +process: + langcode: + plugin: default_value + source: language + default_value: en + created: created + changed: changed + status: + plugin: default_value + default_value: 1 + uid: + plugin: default_value + default_value: 1 + title: title + field_vy_blog_description/value: field_vy_blog_description + field_vy_blog_description/format: + plugin: default_value + default_value: full_html + field_vy_blog_image: + plugin: migration + migration: virtual_y_shared_media_blog_image:REPLACE_ME + source: field_vy_blog_image_id +destination: + plugin: 'entity:node' + default_bundle: vy_blog_post + +migration_dependencies: + required: + - virtual_y_shared_media_blog_image:REPLACE_ME + optional: { } diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml index 846b3547b..0255f1be6 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -5,12 +5,11 @@ dependencies: module: - openy_gc_shared_content id: virtual_y_shared_node_video -migration_tags: - - openy_complete_installation migration_group: virtual_y_shared deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver label: 'Sync shared videos to Open Y shared demo server.' source: + entity_type: gc_video plugin: url data_fetcher_plugin: http # Specifies the JSON parser plugin. @@ -46,6 +45,9 @@ source: - name: field_gc_video_duration label: 'Duration' selector: /attributes/field_gc_video_duration + - name: field_gc_video_image_id + label: 'Teaser image' + selector: /relationships/field_gc_video_image/data/id ids: nid: type: integer @@ -70,6 +72,11 @@ process: plugin: migration migration: virtual_y_shared_media_video:REPLACE_ME source: field_gc_video_media_id + field_gc_video_image: + plugin: migration + migration: virtual_y_shared_media_video_image:REPLACE_ME + source: field_gc_video_image_id + destination: plugin: 'entity:node' default_bundle: gc_video @@ -77,4 +84,5 @@ destination: migration_dependencies: required: - virtual_y_shared_media_video:REPLACE_ME + - virtual_y_shared_media_video_image:REPLACE_ME optional: { } From e15c2e9887fa05ce4fc89230a167b0a419a7d2f0 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 01:14:27 +0300 Subject: [PATCH 04/13] Added instructor to videos migration --- .../migrate_plus.migration.virtual_y_shared_node_video.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml index 0255f1be6..1149ce799 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -36,6 +36,9 @@ source: - name: status label: 'Status' selector: /attributes/status + - name: field_gc_video_instructor + label: 'Instructor Name' + selector: /attributes/field_gc_video_instructor - name: field_gc_video_media_id label: 'Video' selector: /relationships/field_gc_video_media/data/id @@ -68,6 +71,7 @@ process: plugin: default_value default_value: full_html field_gc_video_duration: field_gc_video_duration + field_gc_video_instructor: field_gc_video_instructor field_gc_video_media: plugin: migration migration: virtual_y_shared_media_video:REPLACE_ME From 7ef98d83a52858087c77ba482a66b3a563b8aace Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 12:49:20 +0300 Subject: [PATCH 05/13] Added origin propagation for shared content migrations --- .../migrate_plus.migration.virtual_y_shared_file_blog.yml | 8 ++++---- .../migrate_plus.migration.virtual_y_shared_node_blog.yml | 3 +++ ...migrate_plus.migration.virtual_y_shared_node_video.yml | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml index 0eea896db..d9ec97b42 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml @@ -73,10 +73,10 @@ process: - plugin: urlencode pseudo_destination_path: - - plugin: concat - source: - - constants/DRUPAL_FILE_DIRECTORY - - filename + plugin: concat + source: + - constants/DRUPAL_FILE_DIRECTORY + - filename uri_file: - plugin: concat diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml index e4c2895e0..794c0cb88 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml @@ -19,6 +19,8 @@ source: Accept: 'application/json; charset=utf-8' Content-Type: 'application/json' urls: replace_me + constants: + DOMAIN: replace_me item_selector: data/ fields: - name: nid @@ -59,6 +61,7 @@ process: plugin: default_value default_value: 1 title: title + field_gc_origin: constants/DOMAIN field_vy_blog_description/value: field_vy_blog_description field_vy_blog_description/format: plugin: default_value diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml index 1149ce799..3092723c7 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml +++ b/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -19,6 +19,8 @@ source: Accept: 'application/json; charset=utf-8' Content-Type: 'application/json' urls: replace_me + constants: + DOMAIN: replace_me item_selector: data/ fields: - name: nid @@ -66,6 +68,7 @@ process: plugin: default_value default_value: 1 title: title + field_gc_origin: constants/DOMAIN field_gc_video_description/value: field_gc_video_description field_gc_video_description/format: plugin: default_value From af36f7170c76764d112e52a1e7178cb99b820aae Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 14:15:14 +0300 Subject: [PATCH 06/13] Moved shared content migration to the server module --- .../migrate_plus.migration_group.virtual_y_shared.yml | 2 +- .../migrate_plus.migration.virtual_y_shared_file_blog.yml | 4 ++-- .../migrate_plus.migration.virtual_y_shared_file_video.yml | 4 ++-- ...te_plus.migration.virtual_y_shared_media_blog_image.yml | 6 ++---- ...migrate_plus.migration.virtual_y_shared_media_video.yml | 7 ++----- ...e_plus.migration.virtual_y_shared_media_video_image.yml | 6 ++---- .../migrate_plus.migration.virtual_y_shared_node_blog.yml | 4 ++-- .../migrate_plus.migration.virtual_y_shared_node_video.yml | 4 ++-- .../src/SourceMigrationDeriver.php | 2 +- 9 files changed, 16 insertions(+), 23 deletions(-) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/config/install/migrate_plus.migration_group.virtual_y_shared.yml (83%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml (95%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml (95%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml (92%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml (93%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml (92%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml (94%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml (95%) rename modules/{openy_gc_shared_content => openy_gc_shared_content_server}/src/SourceMigrationDeriver.php (98%) diff --git a/modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml b/modules/openy_gc_shared_content_server/config/install/migrate_plus.migration_group.virtual_y_shared.yml similarity index 83% rename from modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml rename to modules/openy_gc_shared_content_server/config/install/migrate_plus.migration_group.virtual_y_shared.yml index 642e3f388..30a5beac2 100644 --- a/modules/openy_gc_shared_content/config/install/migrate_plus.migration_group.virtual_y_shared.yml +++ b/modules/openy_gc_shared_content_server/config/install/migrate_plus.migration_group.virtual_y_shared.yml @@ -4,4 +4,4 @@ dependencies: { } id: virtual_y_shared label: 'Virtual Y Shared Content Server sync' description: 'Imports shared items from Virtual Y sources to the main Demo Server.' -module: openy_gc_shared_content +module: openy_gc_shared_content_server diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml similarity index 95% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml index d9ec97b42..400daed51 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml @@ -1,9 +1,9 @@ dependencies: enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_file_blog -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: 'Import Shared VirtualY files' migration_group: virtual_y_shared source: diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml similarity index 95% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml index 3bdae7a5c..f36b938fe 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml @@ -1,9 +1,9 @@ dependencies: enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_file_video -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: 'Import Shared VirtualY files' migration_group: virtual_y_shared source: diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml similarity index 92% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml index a57b9c24b..bcbdb88ea 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml @@ -1,13 +1,11 @@ dependencies: - module: - - openy_media_image enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_media_blog_image migration_tags: { } migration_group: virtual_y_shared -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: Fetch Image source: entity_type: vy_blog_post diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml similarity index 93% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml index 423cff24d..61c06dac1 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video.yml @@ -1,14 +1,11 @@ dependencies: - module: - - openy_media_video enforced: module: - - openy_gc_demo + - openy_gc_shared_content_server id: virtual_y_shared_media_video -migration_tags: { } migration_group: virtual_y_shared label: Import Virtual Y videos -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver source: entity_type: gc_video json_includes: diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml similarity index 92% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml index 504d7c77c..5f814ed2b 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml @@ -1,13 +1,11 @@ dependencies: - module: - - openy_media_image enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_media_video_image migration_tags: { } migration_group: virtual_y_shared -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: Fetch Image source: entity_type: gc_video diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml similarity index 94% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml index 794c0cb88..8adcc7cfa 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml @@ -3,10 +3,10 @@ status: true dependencies: enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_node_blog migration_group: virtual_y_shared -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: 'Sync shared blog posts to Open Y shared demo server.' source: entity_type: vy_blog_post diff --git a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml similarity index 95% rename from modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml rename to modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml index 3092723c7..6354d45fa 100644 --- a/modules/openy_gc_shared_content/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -3,10 +3,10 @@ status: true dependencies: enforced: module: - - openy_gc_shared_content + - openy_gc_shared_content_server id: virtual_y_shared_node_video migration_group: virtual_y_shared -deriver: Drupal\openy_gc_shared_content\SourceMigrationDeriver +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver label: 'Sync shared videos to Open Y shared demo server.' source: entity_type: gc_video diff --git a/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php similarity index 98% rename from modules/openy_gc_shared_content/src/SourceMigrationDeriver.php rename to modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index 6f63dee9a..e89650e73 100644 --- a/modules/openy_gc_shared_content/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -1,6 +1,6 @@ Date: Thu, 20 Aug 2020 15:28:22 +0300 Subject: [PATCH 07/13] Replaced hard-coded sources with entities --- .../src/SourceMigrationDeriver.php | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index e89650e73..3288558ef 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -4,6 +4,7 @@ use Drupal\Component\Plugin\Derivative\DeriverBase; use Drupal\Component\Plugin\Derivative\DeriverInterface; +use Drupal\openy_gc_shared_content_server\Entity\SharedContentSource; /** * Class SourceMigrationDeriver @@ -17,12 +18,18 @@ class SourceMigrationDeriver extends DeriverBase implements DeriverInterface { */ public function getDerivativeDefinitions($base_plugin_definition) { - //@TODO rewrite it to source entities - $urls = [ - 'http://rose.demo.openy.ci.fivejars.com', - 'http://lily.demo.openy.ci.fivejars.com', - 'http://carnation.demo.openy.ci.fivejars.com' - ]; + $ids = \Drupal::service('entity.query') + ->get('shared_content_source')->execute(); + + if (empty($ids)) { + return []; + } + + $sources = SharedContentSource::loadMultiple($ids); + $urls = []; + foreach ($sources as $source) { + $urls[] = $source->getUrl(); + } $params = [ 'include' => implode(',', $this->getRemoteRelationshipsList($base_plugin_definition)), @@ -30,7 +37,6 @@ public function getDerivativeDefinitions($base_plugin_definition) { 'sort[sortByDate][direction]' => 'DESC', 'filter[status]' => 1, //@TODO add shared_content_filter once it will be at test servers. - //'limit' => 100 ]; $jsonapi_uri = '/jsonapi/node/' . $base_plugin_definition['source']['entity_type'] . '?' . http_build_query($params); From f80f41c812be204a127044103060d82e3692a0b2 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 16:40:27 +0300 Subject: [PATCH 08/13] Added terms migration for video --- ...migration.virtual_y_shared_gc_category.yml | 51 +++++++++++++++++++ ...igration.virtual_y_shared_gc_equipment.yml | 50 ++++++++++++++++++ ...us.migration.virtual_y_shared_gc_level.yml | 50 ++++++++++++++++++ ....migration.virtual_y_shared_node_video.yml | 31 ++++++++++- .../src/SourceMigrationDeriver.php | 32 +++++++++++- 5 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_category.yml create mode 100644 modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_equipment.yml create mode 100644 modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_level.yml diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_category.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_category.yml new file mode 100644 index 000000000..e551108de --- /dev/null +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_category.yml @@ -0,0 +1,51 @@ +langcode: en +status: true +dependencies: + enforced: + module: + - openy_gc_shared_content_server +id: virtual_y_shared_gc_category +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver +label: 'GC Category' +source: + entity_type: gc_video + json_includes: + - field_gc_video_category + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + constants: + DOMAIN: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: title + label: 'title' + selector: /attributes/name + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + status: + plugin: default_value + default_value: 1 + name: title +destination: + plugin: entity:taxonomy_term + default_bundle: gc_category + +migration_dependencies: + optional: { } diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_equipment.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_equipment.yml new file mode 100644 index 000000000..c1a5dbed1 --- /dev/null +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_equipment.yml @@ -0,0 +1,50 @@ +langcode: en +status: true +dependencies: + enforced: + module: + - openy_gc_shared_content_server +id: virtual_y_shared_gc_equipment +migration_tags: + - openy_complete_installation +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver +label: 'GC Equipment' +source: + entity_type: gc_video + json_includes: + - field_gc_video_equipment + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + constants: + DOMAIN: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: title + label: 'title' + selector: /attributes/name + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + status: + plugin: default_value + default_value: 1 + name: title +destination: + plugin: entity:taxonomy_term + default_bundle: gc_equipment diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_level.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_level.yml new file mode 100644 index 000000000..7d24bf48b --- /dev/null +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_gc_level.yml @@ -0,0 +1,50 @@ +langcode: en +status: true +dependencies: + enforced: + module: + - openy_gc_shared_content_server +id: virtual_y_shared_gc_level +migration_tags: + - openy_complete_installation +migration_group: virtual_y_shared +deriver: Drupal\openy_gc_shared_content_server\SourceMigrationDeriver +label: 'GC Level' +source: + entity_type: gc_video + json_includes: + - field_gc_video_level + plugin: url + data_fetcher_plugin: http + # Specifies the JSON parser plugin. + data_parser_plugin: json + track_changes: true + headers: + Accept: 'application/json; charset=utf-8' + Content-Type: 'application/json' + urls: replace_me + constants: + DOMAIN: replace_me + item_selector: included/ + fields: + - name: id + label: 'id' + selector: /id + - name: title + label: 'title' + selector: /attributes/name + ids: + id: + type: string +process: + langcode: + plugin: default_value + source: language + default_value: en + status: + plugin: default_value + default_value: 1 + name: title +destination: + plugin: entity:taxonomy_term + default_bundle: gc_level diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml index 6354d45fa..dc7eac5d1 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_video.yml @@ -53,6 +53,15 @@ source: - name: field_gc_video_image_id label: 'Teaser image' selector: /relationships/field_gc_video_image/data/id + - name: field_gc_video_category_id + label: 'Category' + selector: /relationships/field_gc_video_category/data/id + - name: field_gc_video_equipment_id + label: 'Equipment' + selector: /relationships/field_gc_video_equipment/data/id + - name: field_gc_video_level_id + label: 'Level' + selector: /relationships/field_gc_video_level/data/id ids: nid: type: integer @@ -62,8 +71,14 @@ process: source: language default_value: en status: status - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime uid: plugin: default_value default_value: 1 @@ -83,6 +98,18 @@ process: plugin: migration migration: virtual_y_shared_media_video_image:REPLACE_ME source: field_gc_video_image_id + field_gc_video_category: + plugin: migration + migration: virtual_y_shared_gc_category:REPLACE_ME + source: field_gc_video_category_id + field_gc_video_equipment: + plugin: migration + migration: virtual_y_shared_gc_equipment:REPLACE_ME + source: field_gc_video_equipment_id + field_gc_video_level: + plugin: migration + migration: virtual_y_shared_gc_level:REPLACE_ME + source: field_gc_video_level_id destination: plugin: 'entity:node' diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index 3288558ef..059da238d 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -7,7 +7,7 @@ use Drupal\openy_gc_shared_content_server\Entity\SharedContentSource; /** - * Class SourceMigrationDeriver + * Class SourceMigrationDeriver. * * @package Drupal\openy_gc_shared_content */ @@ -110,6 +110,36 @@ private function getDerivativeValues(array $base_plugin_definition, $url_long, $ $base_plugin_definition['migration_dependencies']['required'][] = $migration; } + if (!empty($base_plugin_definition['process']['field_gc_video_category'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_gc_video_category']['migration'] + ); + $base_plugin_definition['process']['field_gc_video_category']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'][] = $migration; + } + + if (!empty($base_plugin_definition['process']['field_gc_video_equipment'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_gc_video_equipment']['migration'] + ); + $base_plugin_definition['process']['field_gc_video_equipment']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'][] = $migration; + } + + if (!empty($base_plugin_definition['process']['field_gc_video_level'])) { + $migration = str_replace( + 'REPLACE_ME', + $this->getKey($url), + $base_plugin_definition['process']['field_gc_video_level']['migration'] + ); + $base_plugin_definition['process']['field_gc_video_level']['migration'] = $migration; + $base_plugin_definition['migration_dependencies']['required'][] = $migration; + } + return $base_plugin_definition; } From 44f210b95adf58399cbe6803a7aadcf15c443f8a Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 16:40:55 +0300 Subject: [PATCH 09/13] Fixed date translation bug for migrations --- ...grate_plus.migration.virtual_y_shared_file_blog.yml | 10 ++++++++-- ...rate_plus.migration.virtual_y_shared_file_video.yml | 10 ++++++++-- ...lus.migration.virtual_y_shared_media_blog_image.yml | 10 ++++++++-- ...us.migration.virtual_y_shared_media_video_image.yml | 10 ++++++++-- ...grate_plus.migration.virtual_y_shared_node_blog.yml | 10 ++++++++-- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml index 400daed51..2cd145dbc 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_blog.yml @@ -61,8 +61,14 @@ process: plugin: default_value source: language default_value: en - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime pseudo_source_full_path: - plugin: concat diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml index f36b938fe..f888dc49e 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_file_video.yml @@ -61,8 +61,14 @@ process: plugin: default_value source: language default_value: en - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime pseudo_source_full_path: - plugin: concat diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml index bcbdb88ea..a3e8abc3c 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_blog_image.yml @@ -51,8 +51,14 @@ process: plugin: default_value source: language default_value: en - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime name: name type: plugin: skip_on_value diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml index 5f814ed2b..20710e03f 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_media_video_image.yml @@ -51,8 +51,14 @@ process: plugin: default_value source: language default_value: en - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime name: name type: plugin: skip_on_value diff --git a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml index 8adcc7cfa..0528b99aa 100644 --- a/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml +++ b/modules/openy_gc_shared_content_server/migrations/migrate_plus.migration.virtual_y_shared_node_blog.yml @@ -52,8 +52,14 @@ process: plugin: default_value source: language default_value: en - created: created - changed: changed + created: + plugin: callback + source: created + callable: strtotime + changed: + plugin: callback + source: changed + callable: strtotime status: plugin: default_value default_value: 1 From 980b9f01bc8c87137df1c84837b52068a80f0435 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 16:49:01 +0300 Subject: [PATCH 10/13] CS fix --- .../src/SourceMigrationDeriver.php | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index 059da238d..b9beac7e4 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -4,21 +4,47 @@ use Drupal\Component\Plugin\Derivative\DeriverBase; use Drupal\Component\Plugin\Derivative\DeriverInterface; +use Drupal\Core\Entity\Query\QueryFactoryInterface; +use Symfony\Component\DependencyInjection\ContainerInterface; use Drupal\openy_gc_shared_content_server\Entity\SharedContentSource; + /** * Class SourceMigrationDeriver. * * @package Drupal\openy_gc_shared_content */ -class SourceMigrationDeriver extends DeriverBase implements DeriverInterface { +class SourceMigrationDeriver extends DeriverBase implements DeriverInterface, ContainerDeriverInterface { + + /** + * @var \Drupal\Core\Entity\Query\QueryFactoryInterface + */ + protected $entity_query; + + /** + * SourceMigrationDeriver constructor. + * + * @param \Drupal\Core\Entity\Query\QueryFactoryInterface $entityQuery + */ + public function __construct(QueryFactoryInterface $entityQuery) { + $this->entity_query= $entityQuery; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, $base_plugin_id) { + return new static( + $container->get('entity.query') + ); + } /** * {@inheritdoc} */ public function getDerivativeDefinitions($base_plugin_definition) { - $ids = \Drupal::service('entity.query') + $ids = $this->entity_query ->get('shared_content_source')->execute(); if (empty($ids)) { From f80317cc021290835bafae059dc183659ee3b8e1 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 16:56:23 +0300 Subject: [PATCH 11/13] cs fix --- .../src/SourceMigrationDeriver.php | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index b9beac7e4..17b18c337 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -8,7 +8,6 @@ use Symfony\Component\DependencyInjection\ContainerInterface; use Drupal\openy_gc_shared_content_server\Entity\SharedContentSource; - /** * Class SourceMigrationDeriver. * @@ -17,17 +16,20 @@ class SourceMigrationDeriver extends DeriverBase implements DeriverInterface, ContainerDeriverInterface { /** + * EntityQuery service instance. + * * @var \Drupal\Core\Entity\Query\QueryFactoryInterface */ - protected $entity_query; + protected $entityQuery; /** * SourceMigrationDeriver constructor. * * @param \Drupal\Core\Entity\Query\QueryFactoryInterface $entityQuery + * EntityQuery instance. */ public function __construct(QueryFactoryInterface $entityQuery) { - $this->entity_query= $entityQuery; + $this->entityQuery = $entityQuery; } /** @@ -44,8 +46,7 @@ public static function create(ContainerInterface $container, $base_plugin_id) { */ public function getDerivativeDefinitions($base_plugin_definition) { - $ids = $this->entity_query - ->get('shared_content_source')->execute(); + $ids = $this->entityQuery->get('shared_content_source')->execute(); if (empty($ids)) { return []; @@ -62,7 +63,7 @@ public function getDerivativeDefinitions($base_plugin_definition) { 'sort[sortByDate][path]' => 'created', 'sort[sortByDate][direction]' => 'DESC', 'filter[status]' => 1, - //@TODO add shared_content_filter once it will be at test servers. + // @TODO add shared_content_filter once it will be at test servers. ]; $jsonapi_uri = '/jsonapi/node/' . $base_plugin_definition['source']['entity_type'] . '?' . http_build_query($params); @@ -81,12 +82,13 @@ public function getDerivativeDefinitions($base_plugin_definition) { * * @param array $base_plugin_definition * Plugin settings. - * @param $url_long - * Url with request part - * @param $url + * @param string $url_long + * Url with request part. + * @param string $url * Dynamic url for every Virtual Y content source. * * @return array + * Updated plugin data. */ private function getDerivativeValues(array $base_plugin_definition, $url_long, $url) { @@ -172,20 +174,20 @@ private function getDerivativeValues(array $base_plugin_definition, $url_long, $ /** * Helper function that checks if we need includes to jsonapi request. * - * @param $base_plugin_definition + * @param array $base_plugin_definition * Migration array. * * @return array * Includes array for JSON:API */ - private function getRemoteRelationshipsList($base_plugin_definition) { + private function getRemoteRelationshipsList(array $base_plugin_definition) { return !empty($base_plugin_definition['source']['json_includes']) ? $base_plugin_definition['source']['json_includes'] : []; } /** * Helper function that prepare entity key from url. * - * @param $url + * @param string $url * Url to source. * * @return mixed From b144cc7e9d0b6d491688b36b0dd25a3fe71b2e84 Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Thu, 20 Aug 2020 17:00:22 +0300 Subject: [PATCH 12/13] Fixed errors after cs --- .../src/SourceMigrationDeriver.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index 17b18c337..8ca5dcfe8 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -4,9 +4,10 @@ use Drupal\Component\Plugin\Derivative\DeriverBase; use Drupal\Component\Plugin\Derivative\DeriverInterface; -use Drupal\Core\Entity\Query\QueryFactoryInterface; +use Drupal\Core\Entity\Query\QueryFactory; use Symfony\Component\DependencyInjection\ContainerInterface; use Drupal\openy_gc_shared_content_server\Entity\SharedContentSource; +use Drupal\Core\Plugin\Discovery\ContainerDeriverInterface; /** * Class SourceMigrationDeriver. @@ -18,17 +19,17 @@ class SourceMigrationDeriver extends DeriverBase implements DeriverInterface, Co /** * EntityQuery service instance. * - * @var \Drupal\Core\Entity\Query\QueryFactoryInterface + * @var \Drupal\Core\Entity\Query\QueryFactory */ protected $entityQuery; /** * SourceMigrationDeriver constructor. * - * @param \Drupal\Core\Entity\Query\QueryFactoryInterface $entityQuery + * @param \Drupal\Core\Entity\Query\QueryFactory $entityQuery * EntityQuery instance. */ - public function __construct(QueryFactoryInterface $entityQuery) { + public function __construct(QueryFactory $entityQuery) { $this->entityQuery = $entityQuery; } From 40c22eb3cbc170e20ba165f4ab623657eeb94e1b Mon Sep 17 00:00:00 2001 From: Anatoliy Polyakov Date: Fri, 21 Aug 2020 12:48:35 +0300 Subject: [PATCH 13/13] Added real shared content filter --- .../src/SourceMigrationDeriver.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php index 8ca5dcfe8..aa3d23d4d 100644 --- a/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php +++ b/modules/openy_gc_shared_content_server/src/SourceMigrationDeriver.php @@ -64,7 +64,7 @@ public function getDerivativeDefinitions($base_plugin_definition) { 'sort[sortByDate][path]' => 'created', 'sort[sortByDate][direction]' => 'DESC', 'filter[status]' => 1, - // @TODO add shared_content_filter once it will be at test servers. + 'filter[field_gc_share]' => 1, ]; $jsonapi_uri = '/jsonapi/node/' . $base_plugin_definition['source']['entity_type'] . '?' . http_build_query($params);