-
Notifications
You must be signed in to change notification settings - Fork 63
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
123 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
# frozen_string_literal: true | ||
|
||
require "fileutils" | ||
|
||
# TODO: This class needs tests | ||
module Cloning | ||
# Outputs data to CSV ZIP bundle. | ||
class Exporter | ||
attr_accessor :relations, :options | ||
|
||
def initialize(relations, **options) | ||
self.relations = relations | ||
self.options = options | ||
end | ||
|
||
def export | ||
expander = RelationExpander.new(relations, dont_implicitly_expand: options[:dont_implicitly_expand]) | ||
buffer = Zip::OutputStream.write_buffer do |out| | ||
expander.expanded.each do |klass, relations| | ||
# TODO: Improve this logic a bit, make it more structured and check table name | ||
col_names = klass.column_names - %w[standard_copy last_mission_id] | ||
relations.each_with_index do |relation, idx| | ||
out.put_next_entry("#{klass.name.tr(':', '_')}-#{idx}.csv") | ||
relation = relation.select(col_names.join(", ")) unless col_names == klass.column_names | ||
relation.copy_to { |line| out.write(line) } | ||
end | ||
end | ||
end | ||
FileUtils.mkdir_p(export_dir) | ||
File.open(zipfile_path, "wb") { |f| f.write(buffer.string) } | ||
end | ||
|
||
private | ||
|
||
def export_dir | ||
@export_dir ||= Rails.root.join("tmp/exports") | ||
end | ||
|
||
def zipfile_path | ||
@zipfile_path ||= export_dir.join("#{Time.zone.now.to_s(:filename_datetime)}.zip") | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
# frozen_string_literal: true | ||
|
||
# TODO: This class needs tests | ||
module Cloning | ||
# Imports from CSV ZIP bundle. | ||
class Importer | ||
attr_accessor :zip_file | ||
|
||
def initialize(zip_file) | ||
self.zip_file = zip_file | ||
end | ||
|
||
def import | ||
ApplicationRecord.transaction do | ||
# Defer constraints so that constraints are not checked until all data is loaded. | ||
SqlRunner.instance.run("SET CONSTRAINTS ALL DEFERRED") | ||
Zip::InputStream.open(zip_file) do |io| | ||
index = 0 | ||
while (entry = io.get_next_entry) | ||
class_name = entry.name.match(/\A(\w+)-\d+\.csv\z/)[1] | ||
klass = class_name.sub(/.csv$/, "").tr("_", ":").constantize | ||
tmp_table = "tmp_table_#{index}" | ||
SqlRunner.instance.run("CREATE TEMP TABLE #{tmp_table} | ||
ON COMMIT DROP AS SELECT * FROM #{klass.table_name} WITH NO DATA") | ||
klass.copy_from(io, table: tmp_table) | ||
col_names = klass.column_names - %w[standard_copy last_mission_id] | ||
select = col_names == klass.column_names ? "*" : col_names.join(", ") | ||
insert_cols = col_names == klass.column_names ? "" : "(#{col_names.join(', ')})" | ||
SqlRunner.instance.run("INSERT INTO #{klass.table_name}#{insert_cols} | ||
SELECT #{select} FROM #{tmp_table} ON CONFLICT DO NOTHING") | ||
index += 1 | ||
end | ||
end | ||
end | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
# frozen_string_literal: true | ||
|
||
module Cloning | ||
# Expands a given set of relations to include all necessary related objects. | ||
class RelationExpander | ||
attr_accessor :initial_relations, :relations_by_class, :options | ||
|
||
def initialize(relations, **options) | ||
self.options = options | ||
options[:dont_implicitly_expand] ||= [] | ||
self.initial_relations = relations | ||
self.relations_by_class = relations.group_by(&:klass) | ||
end | ||
|
||
# Returns a hash of form {ModelClass => [Relation, Relation, ...], ...}, mapping model classes | ||
# to arrays of Relations. | ||
def expanded | ||
initial_relations.each { |r| expand(r) } | ||
relations_by_class | ||
end | ||
|
||
private | ||
|
||
def expand(relation) | ||
(relation.klass.clone_options[:follow] || []).each do |assn_name| | ||
assn = relation.klass.reflect_on_association(assn_name) | ||
|
||
# dont_implicitly_expand is provided if the caller wants to indicate that one of the initial_relations | ||
# should cover all relevant rows and therefore implicit expansion is not necessary. This improves | ||
# performance by simplifying the eventual SQL queries. | ||
next if options[:dont_implicitly_expand].include?(assn.klass) | ||
|
||
new_rel = if assn.belongs_to? | ||
assn.klass.where("id IN (#{relation.select(assn.foreign_key).to_sql})") | ||
else | ||
assn.klass.where("#{assn.foreign_key} IN (#{relation.select(:id).to_sql})") | ||
end | ||
(relations_by_class[assn.klass] ||= []) << new_rel | ||
expand(new_rel) | ||
end | ||
end | ||
end | ||
end |