From 2e194cf8c3e4e061ff233ba6f48c85f1a71fb402 Mon Sep 17 00:00:00 2001 From: kitsuy Date: Tue, 21 Nov 2017 19:37:39 +0900 Subject: [PATCH 1/2] Add supporting SQL Server --- Dockerfile | 15 ++++++++++++++- Gemfile | 2 ++ Gemfile.lock | 7 ++++++- app/models/data_source.rb | 8 ++++++++ app/models/data_source_table.rb | 12 +++++++++--- app/models/table_memo_raw_dataset_column.rb | 21 ++++++++++++++++----- app/views/data_sources/_form.html.haml | 2 +- config/database.yml | 1 + 8 files changed, 57 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a277dd..80ace58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,18 @@ FROM ruby:2.3.1-slim -RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libmysqlclient-dev nodejs nodejs-legacy npm git +RUN apt-get update -qq && \ + apt-get install -y \ + build-essential libpq-dev libmysqlclient-dev nodejs nodejs-legacy npm git \ + freetds-dev freetds-bin unixodbc unixodbc-dev tdsodbc libc6-dev wget + +RUN mkdir /tmp/freetds && \ + cd /tmp/freetds && \ + wget ftp://ftp.freetds.org/pub/freetds/stable/freetds-1.00.21.tar.gz && \ + tar -xzf freetds-1.00.21.tar.gz && \ + cd freetds-1.00.21 && \ + ./configure --prefix=/usr/local --with-tdsver=7.3 && \ + make && \ + make install + RUN mkdir /app ADD Gemfile /tmp/Gemfile diff --git a/Gemfile b/Gemfile index 6dfce39..52d1973 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,8 @@ gem 'rails', '~> 4.2.6' gem 'pg', '~> 0.15' gem 'mysql2' gem 'activerecord4-redshift-adapter', '~> 0.2.0' +gem 'activerecord-sqlserver-adapter', '~> 4.2.0' +gem 'tiny_tds' gem 'sass-rails', '~> 5.0' gem 'uglifier', '>= 1.3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 0dbd5e0..97b1a09 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,6 +32,8 @@ GEM activemodel (= 4.2.7.1) activesupport (= 4.2.7.1) arel (~> 6.0) + activerecord-sqlserver-adapter (4.2.18) + activerecord (~> 4.2.1) activerecord4-redshift-adapter (0.2.1) activerecord (~> 4.2.0) pg @@ -301,6 +303,7 @@ GEM thread_safe (0.3.5) tilt (2.0.5) tins (1.6.0) + tiny_tds (2.1.0) tzinfo (1.2.2) thread_safe (~> 0.1) uglifier (3.0.3) @@ -315,6 +318,7 @@ PLATFORMS DEPENDENCIES action_args active_decorator + activerecord-sqlserver-adapter (~> 4.2.0) activerecord4-redshift-adapter (~> 0.2.0) addressable byebug @@ -354,7 +358,8 @@ DEPENDENCIES ruby-prof sass-rails (~> 5.0) simplecov + tiny_tds uglifier (>= 1.3.0) BUNDLED WITH - 1.12.5 + 1.16.0 diff --git a/app/models/data_source.rb b/app/models/data_source.rb index d799343..e0cb950 100644 --- a/app/models/data_source.rb +++ b/app/models/data_source.rb @@ -75,6 +75,14 @@ def source_table_names ) tables ORDER BY schemaname, tablename; SQL + when ActiveRecord::ConnectionAdapters::SQLServerAdapter + schemas_and_tables = source_base_class.connection.select_rows(<<-SQL, 'SCHEMA') + SELECT table_schema + , table_name + FROM information_schema.tables + WHERE table_type = 'BASE TABLE'; + SQL + schemas_and_tables.map {|table_schema, table_name| [table_schema, table_name]} else source_base_class.connection.tables.map {|table_name| [dbname, table_name] } end diff --git a/app/models/data_source_table.rb b/app/models/data_source_table.rb index 392f196..b3d84e2 100644 --- a/app/models/data_source_table.rb +++ b/app/models/data_source_table.rb @@ -17,9 +17,15 @@ def fetch_rows(limit=20) data_source.access_logging do adapter = connection.pool.connections.first column_names = columns.map {|column| adapter.quote_column_name(column.name) }.join(", ") - rows = connection.select_rows(<<-SQL, "#{table_name.classify} Load") - SELECT #{column_names} FROM #{adapter.quote_table_name(full_table_name)} LIMIT #{limit}; - SQL + if data_source[:adapter] == 'sqlserver' + rows = connection.select_rows(<<-SQL, "#{table_name.classify} Load") + SELECT TOP #{limit} #{column_names} FROM #{adapter.quote_table_name(full_table_name)}; + SQL + else + rows = connection.select_rows(<<-SQL, "#{table_name.classify} Load") + SELECT #{column_names} FROM #{adapter.quote_table_name(full_table_name)} LIMIT #{limit}; + SQL + end rows.map {|row| columns.zip(row).map {|column, value| column.type_cast_from_database(value) } } diff --git a/app/models/table_memo_raw_dataset_column.rb b/app/models/table_memo_raw_dataset_column.rb index f82e6b2..50c6eb9 100644 --- a/app/models/table_memo_raw_dataset_column.rb +++ b/app/models/table_memo_raw_dataset_column.rb @@ -2,12 +2,23 @@ class TableMemoRawDatasetColumn < ActiveRecord::Base belongs_to :table_memo_raw_dataset, class_name: "TableMemoRawDataset" def format_value(value) - case sql_type - when 'timestamp without time zone' - d, t, z = value.to_s.split - "#{d} #{t}" + datasource_encoding = table_memo_raw_dataset.table_memo.database_memo.data_source.encoding + datasource_adapter = table_memo_raw_dataset.table_memo.database_memo.data_source.adapter + if datasource_adapter == 'sqlserver' + case sql_type + when 'timestamp' + '0x' + value.unpack('H*')[0] + else + value.to_s.encode(Encoding.default_internal, datasource_encoding).gsub(/\u0000/, '') + end else - value.to_s + case sql_type + when 'timestamp without time zone' + d, t, z = value.to_s.split + "#{d} #{t}" + else + value.to_s + end end end end diff --git a/app/views/data_sources/_form.html.haml b/app/views/data_sources/_form.html.haml index aeedc2c..c169a49 100644 --- a/app/views/data_sources/_form.html.haml +++ b/app/views/data_sources/_form.html.haml @@ -8,7 +8,7 @@ .col-sm-10= f.text_field :description, class: "form-control", placeholder: "localhost database" .form-group = f.label :adapter, class: "col-sm-2 control-label" - .col-sm-10= f.select :adapter, %w(redshift postgresql mysql2), {}, class: "form-control" + .col-sm-10= f.select :adapter, %w(redshift postgresql mysql2 sqlserver), {}, class: "form-control" .form-group = f.label :host, class: "col-sm-2 control-label" .col-sm-10= f.text_field :host, class: "form-control", placeholder: "localhost" diff --git a/config/database.yml b/config/database.yml index e57e352..11f0fc7 100644 --- a/config/database.yml +++ b/config/database.yml @@ -13,4 +13,5 @@ test: database: dmemo_test production: + <<: *default url: <%= ENV["DATABASE_URL"] %> From 567bceb01ffb2579e1a2706427142227fc86e688 Mon Sep 17 00:00:00 2001 From: kitsuy Date: Fri, 24 Nov 2017 20:28:21 +0900 Subject: [PATCH 2/2] Addapt default encoding in SQL Server --- app/models/data_source_table.rb | 4 +++- app/models/table_memo_raw_dataset_column.rb | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/data_source_table.rb b/app/models/data_source_table.rb index b3d84e2..ea976fb 100644 --- a/app/models/data_source_table.rb +++ b/app/models/data_source_table.rb @@ -16,7 +16,9 @@ def initialize(data_source, schema_name, table_name) def fetch_rows(limit=20) data_source.access_logging do adapter = connection.pool.connections.first - column_names = columns.map {|column| adapter.quote_column_name(column.name) }.join(", ") + decode_with = data_source.encoding or 'UTF-8' + encode_to = Encoding.default_internal or 'UTF-8' + column_names = columns.map {|column| adapter.quote_column_name(column.name.encode(encode_to, decode_with)) }.join(", ") if data_source[:adapter] == 'sqlserver' rows = connection.select_rows(<<-SQL, "#{table_name.classify} Load") SELECT TOP #{limit} #{column_names} FROM #{adapter.quote_table_name(full_table_name)}; diff --git a/app/models/table_memo_raw_dataset_column.rb b/app/models/table_memo_raw_dataset_column.rb index 50c6eb9..a763946 100644 --- a/app/models/table_memo_raw_dataset_column.rb +++ b/app/models/table_memo_raw_dataset_column.rb @@ -9,7 +9,8 @@ def format_value(value) when 'timestamp' '0x' + value.unpack('H*')[0] else - value.to_s.encode(Encoding.default_internal, datasource_encoding).gsub(/\u0000/, '') + dst_enc = Encoding.default_internal or 'UTF-8' + value.to_s.encode(dst_enc, datasource_encoding).gsub(/\u0000/, '') end else case sql_type