diff --git a/.github/clickhouse/clickhouse_keeper.xml b/.github/clickhouse/clickhouse_keeper.xml new file mode 100644 index 0000000..6331870 --- /dev/null +++ b/.github/clickhouse/clickhouse_keeper.xml @@ -0,0 +1,42 @@ + + + + + true + + localhost + 9000 + + + + + + + 9181 + 0 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + + + + + 1 + localhost + 9234 + + + + + + + localhost + 9181 + + + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39145c3..a307874 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,25 +17,52 @@ jobs: matrix: ruby: ["3.1", "3.2", "3.3", "3.4"] - services: - postgres: - image: postgres - env: - POSTGRES_USER: root - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 env: - PGHOST: localhost - PGUSER: root - + PGHOST: pg + PGUSER: user + PGPASSWORD: pass steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Create docker network + run: docker network create dbnet + + - name: Start PostgreSQL + run: | + docker run -d \ + --name pg \ + --network dbnet \ + -e POSTGRES_PASSWORD=pass \ + -e POSTGRES_USER=user \ + -e POSTGRES_DB=umbrellio_utils_test \ + -p 5432:5432 \ + postgres:14 + + - name: Start ClickHouse + run: | + docker run -d \ + --name ch \ + --network dbnet \ + -e CLICKHOUSE_SKIP_USER_SETUP=1 -e CLICKHOUSE_DB=umbrellio_utils_test \ + -p 9000:9000 -p 8123:8123 \ + -v ${{ github.workspace }}/.github/clickhouse/clickhouse_keeper.xml:/etc/clickhouse-server/config.d/keeper.xml \ + clickhouse/clickhouse-server:25.3.6.56-alpine + + - name: Wait for Postgres + run: | + for i in {1..30}; do + if docker exec pg pg_isready -U user; then exit 0; fi + sleep 1 + done + exit 1 + + - name: Wait for ClickHouse + run: | + for i in {1..30}; do + if docker exec ch clickhouse-client --query "SELECT 1"; then exit 0; fi + sleep 1 + done + exit 1 - uses: ruby/setup-ruby@v1 with: @@ -43,8 +70,6 @@ jobs: rubygems: latest bundler-cache: true - - run: psql -c 'CREATE DATABASE umbrellio_utils_test' - - name: Run Linter run: bundle exec ci-helper RubocopLint diff --git a/.rubocop.yml b/.rubocop.yml index 1367e0a..783ce52 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -12,6 +12,9 @@ Naming/MethodParameterName: RSpec/EmptyLineAfterHook: Enabled: false +Metrics/ModuleLength: + Enabled: false + Naming/FileName: Exclude: - lib/umbrellio-utils.rb diff --git a/Gemfile b/Gemfile index 797fd4c..59ad9ef 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,8 @@ gemspec gem "activesupport" gem "bundler" gem "ci-helper" +gem "click_house", github: "umbrellio/click_house", branch: "master" +gem "csv" gem "http" gem "net-pop" gem "nokogiri" diff --git a/Gemfile.lock b/Gemfile.lock index cbcddd6..6574b16 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,16 @@ +GIT + remote: https://github.com/umbrellio/click_house.git + revision: 1bbf8cb909a248b401d0ba9a9f6f1de2e2c068db + branch: master + specs: + click_house (2.1.2) + activesupport + faraday (>= 1.7, < 3) + PATH remote: . specs: - umbrellio-utils (1.9.0) + umbrellio-utils (1.10.0) memery (~> 1) GEM @@ -100,6 +109,7 @@ GEM concurrent-ruby (1.3.5) connection_pool (2.5.4) crass (1.0.6) + csv (3.3.5) date (3.4.1) diff-lcs (1.6.2) docile (1.4.1) @@ -109,6 +119,12 @@ GEM erb (4.0.4) cgi (>= 0.3.3) erubi (1.13.1) + faraday (2.14.0) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.2) + net-http (~> 0.5) ffi (1.17.2-aarch64-linux-gnu) ffi (1.17.2-aarch64-linux-musl) ffi (1.17.2-arm-linux-gnu) @@ -163,6 +179,8 @@ GEM method_source (1.1.0) mini_mime (1.1.5) minitest (5.25.5) + net-http (0.8.0) + uri (>= 0.11.1) net-imap (0.5.10) date net-protocol @@ -364,6 +382,7 @@ GEM unicode-display_width (3.2.0) unicode-emoji (~> 4.1) unicode-emoji (4.1.0) + uri (1.1.1) useragent (0.16.11) websocket-driver (0.8.0) base64 @@ -388,6 +407,8 @@ DEPENDENCIES activesupport bundler ci-helper + click_house! + csv http net-pop nokogiri diff --git a/bin/clickhouse-server b/bin/clickhouse-server new file mode 100755 index 0000000..221c58b --- /dev/null +++ b/bin/clickhouse-server @@ -0,0 +1,20 @@ +#!/bin/bash +set -eu + +docker stop clickhouse-server || true +docker rm clickhouse-server || true + +docker run \ + --detach \ + --network host \ + --name clickhouse-server \ + --ulimit nofile=262144:262144 \ + $CLICKHOUSE_IMAGE_TAG + +# Wait for ClickHouse server to become available +until docker exec clickhouse-server clickhouse-client --query "SELECT 1" &>/dev/null; do + echo "Waiting for ClickHouse to be ready..." + sleep 1 +done + +rails ch:create ch:migrate diff --git a/lib/umbrellio-utils.rb b/lib/umbrellio-utils.rb index 37910e1..e56a39a 100644 --- a/lib/umbrellio-utils.rb +++ b/lib/umbrellio-utils.rb @@ -1,3 +1,7 @@ # frozen_string_literal: true require_relative "umbrellio_utils" + +if defined?(Rake) + Dir[File.join(__dir__, "umbrellio_utils/tasks/**/*.rake")].each { |f| load f } +end diff --git a/lib/umbrellio_utils.rb b/lib/umbrellio_utils.rb index d7da4ce..8b922a3 100644 --- a/lib/umbrellio_utils.rb +++ b/lib/umbrellio_utils.rb @@ -48,18 +48,21 @@ def synchronize(&) require_relative "umbrellio_utils/cards" require_relative "umbrellio_utils/checks" +require_relative "umbrellio_utils/click_house" require_relative "umbrellio_utils/constants" require_relative "umbrellio_utils/control" require_relative "umbrellio_utils/database" require_relative "umbrellio_utils/formatting" require_relative "umbrellio_utils/http_client" require_relative "umbrellio_utils/jobs" +require_relative "umbrellio_utils/migrations" require_relative "umbrellio_utils/misc" require_relative "umbrellio_utils/parsing" require_relative "umbrellio_utils/passwords" require_relative "umbrellio_utils/random" require_relative "umbrellio_utils/request_wrapper" require_relative "umbrellio_utils/rounding" +require_relative "umbrellio_utils/sql" require_relative "umbrellio_utils/semantic_logger/tiny_json_formatter" require_relative "umbrellio_utils/store" require_relative "umbrellio_utils/vault" diff --git a/lib/umbrellio_utils/click_house.rb b/lib/umbrellio_utils/click_house.rb new file mode 100644 index 0000000..00692fc --- /dev/null +++ b/lib/umbrellio_utils/click_house.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +module UmbrellioUtils + module ClickHouse + include Memery + + extend self + + delegate :create_database, :drop_database, :tables, :config, to: :client + + def insert(table_name, db_name: self.db_name, rows: []) + client.insert(full_table_name(table_name, db_name), rows, format: "JSONEachRow") + end + + def from(source, db_name: self.db_name) + ds = + case source + when Symbol + DB.from(db_name == self.db_name ? SQL[source] : SQL[db_name][source]) + when nil + DB.dataset + else + DB.from(source) + end + + ds.clone(ch: true) + end + + def execute(sql, host: nil, **opts) + log_errors(sql) do + client(host).execute(sql, params: opts) + end + end + + def query(dataset, host: nil, **opts) + sql = sql_for(dataset) + + log_errors(sql) do + select_all(sql, host:, **opts).map { |x| Misc::StrictHash[x.symbolize_keys] } + end + end + + def query_value(dataset, host: nil, **opts) + sql = sql_for(dataset) + + log_errors(sql) do + select_value(sql, host:, **opts) + end + end + + def count(dataset) + query_value(dataset.select(SQL.ch_count)) + end + + def optimize_table!(table_name, db_name: self.db_name) + execute("OPTIMIZE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster FINAL") + end + + def truncate_table!(table_name, db_name: self.db_name) + execute("TRUNCATE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC") + end + + def drop_table!(table_name, db_name: self.db_name) + execute("DROP TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC") + end + + def describe_table(table_name, db_name: self.db_name) + sql = "DESCRIBE TABLE #{full_table_name(table_name, db_name)} FORMAT JSON" + + log_errors(sql) do + select_all(sql).map { |x| Misc::StrictHash[x.symbolize_keys] } + end + end + + def db_name + client.config.database.to_sym + end + + def parse_value(value, type:) + case type + when /String/ + value&.to_s + when /DateTime/ + Time.zone.parse(value) if value + else + value + end + end + + def server_version + select_value("SELECT version()").to_f + end + + def pg_table_connection(table) + host = ENV["PGHOST"] || DB.opts[:host].presence || "localhost" + port = DB.opts[:port] || 5432 + database = DB.opts[:database] + username = DB.opts[:user] + password = DB.opts[:password] + + Sequel.function(:postgresql, "#{host}:#{port}", database, table, username, password) + end + + def with_temp_table( + dataset, temp_table_name:, primary_key: [:id], primary_key_types: [:integer], **opts, & + ) + unless DB.table_exists?(temp_table_name) + UmbrellioUtils::Database.create_temp_table( + nil, primary_key:, primary_key_types:, temp_table_name:, & + ) + populate_temp_table!(temp_table_name, dataset) + end + UmbrellioUtils::Database.with_temp_table(nil, primary_key:, temp_table_name:, **opts, &) + end + + private + + def client(host = nil) + cfg = ::ClickHouse.config + cfg.host = resolve(host) if host + ::ClickHouse::Connection.new(cfg) + end + memoize :client, ttl: 1.minute + + def resolve(host) + IPSocket.getaddress(host) + rescue => e + Exceptions.notify!(e, raise_errors: false) + config.host + end + + def logger + client.config.logger + end + + def log_errors(sql) + yield + rescue ::ClickHouse::Error => e + logger.error("ClickHouse error: #{e.inspect}\nSQL: #{sql}") + raise e + end + + def sql_for(dataset) + unless ch_dataset?(dataset) + raise "Non-ClickHouse dataset: #{dataset.inspect}. " \ + "You should use `CH.from` instead of `DB`" + end + + dataset.sql + end + + def ch_dataset?(dataset) + case dataset + when Sequel::Dataset + dataset.opts[:ch] && Array(dataset.opts[:from]).all? { |x| ch_dataset?(x) } + when Sequel::SQL::AliasedExpression + ch_dataset?(dataset.expression) + when Sequel::SQL::Identifier, Sequel::SQL::QualifiedIdentifier + true + else + raise "Unknown dataset type: #{dataset.inspect}" + end + end + + def full_table_name(table_name, db_name) + table_name = table_name.value if table_name.is_a?(Sequel::SQL::Identifier) + "#{db_name}.#{table_name}" + end + + def select_all(sql, host: nil, **opts) + response = client(host).get(body: sql, query: { default_format: "JSON", **opts }) + ::ClickHouse::Response::Factory.response(response, client(host).config) + end + + def select_value(...) + select_all(...).first.to_a.dig(0, -1) + end + + def populate_temp_table!(temp_table_name, dataset) + execute(<<~SQL.squish) + INSERT INTO TABLE FUNCTION #{DB.literal(pg_table_connection(temp_table_name))} + #{dataset.sql} + SQL + end + end +end diff --git a/lib/umbrellio_utils/database.rb b/lib/umbrellio_utils/database.rb index 9a35dc6..2d1c1d7 100644 --- a/lib/umbrellio_utils/database.rb +++ b/lib/umbrellio_utils/database.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# rubocop:disable Metrics/ModuleLength module UmbrellioUtils module Database extend self @@ -79,26 +78,28 @@ def with_temp_table( end # rubocop:enable Metrics/ParameterLists - def create_temp_table(dataset, primary_key: nil, temp_table_name: nil) + def create_temp_table(dataset, primary_key: nil, primary_key_types: nil, temp_table_name: nil) time = Time.current - model = dataset.model + query_table_name = dataset&.model&.table_name - temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}" + temp_table_name ||= :"temp_#{query_table_name}_#{time.to_i}_#{time.nsec}" return temp_table_name if DB.table_exists?(temp_table_name) primary_key = primary_key_from(dataset, primary_key:) + primary_key_types ||= primary_key.map { |x| dataset.model.db_schema[x][:db_type] } DB.create_table(temp_table_name, unlogged: true) do - primary_key.each do |field| - type = model.db_schema[field][:db_type] - column(field, type) + primary_key.each.with_index do |field, i| + column(field, primary_key_types[i]) end primary_key(primary_key) end - insert_ds = dataset.select(*qualified_pk(model.table_name, primary_key)) - DB[temp_table_name].disable_insert_returning.insert(insert_ds) + unless dataset.nil? + insert_ds = dataset.select(*qualified_pk(query_table_name, primary_key)) + DB[temp_table_name].disable_insert_returning.insert(insert_ds) + end temp_table_name end @@ -154,4 +155,3 @@ def pop_next_pk_batch(temp_table_name, primary_key, batch_size) end end end -# rubocop:enable Metrics/ModuleLength diff --git a/lib/umbrellio_utils/migrations.rb b/lib/umbrellio_utils/migrations.rb new file mode 100644 index 0000000..0b8fa2f --- /dev/null +++ b/lib/umbrellio_utils/migrations.rb @@ -0,0 +1,271 @@ +# frozen_string_literal: true + +module UmbrellioUtils + module Migrations + extend self + + def create_new_id_bigint_column(table_name) + DB.run(<<~SQL.squish) + LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE; + + CREATE OR REPLACE FUNCTION id_trigger() + RETURNS trigger + AS + $BODY$ + DECLARE + BEGIN + NEW.id_bigint := NEW.id; + RETURN NEW; + END; + $BODY$ LANGUAGE plpgsql; + + ALTER TABLE #{table_name} ADD id_bigint BIGINT; + + CREATE TRIGGER #{table_name}_bigint + BEFORE INSERT OR UPDATE + ON #{table_name} + FOR EACH ROW + EXECUTE FUNCTION id_trigger(); + SQL + end + + def drop_old_id_column(table_name, associations = {}, skip_fk_create: false) # rubocop:disable Metrics/MethodLength + query_start = <<~SQL.squish + LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE; + DROP TRIGGER #{table_name}_bigint ON #{table_name}; + ALTER TABLE #{table_name} RENAME id TO id_integer; + ALTER TABLE #{table_name} RENAME id_bigint TO id; + + CREATE SEQUENCE IF NOT EXISTS new_#{table_name}_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + SELECT setval( + 'new_#{table_name}_id_seq', + COALESCE((SELECT MAX(id) + 1 FROM #{table_name}), 1), + false + ); + ALTER TABLE #{table_name} + ALTER COLUMN id SET DEFAULT nextval('new_#{table_name}_id_seq'); + SQL + + fkey_query = "" + associations.map do |assoc_table, assoc_name| + constraint_name = "#{assoc_table}_#{assoc_name}_fkey" + + fkey_query += <<~SQL.squish + ALTER TABLE #{assoc_table} + DROP CONSTRAINT IF EXISTS #{constraint_name} + SQL + if skip_fk_create + fkey_query += ";" + next + end + + fkey_query += <<~SQL.squish + , ADD CONSTRAINT #{constraint_name} + FOREIGN KEY (#{assoc_name}) REFERENCES #{table_name}(id) NOT VALID; + SQL + end + + query_end = <<~SQL.squish + ALTER TABLE #{table_name} DROP id_integer; + ALTER TABLE #{table_name} ADD CONSTRAINT #{table_name}_pkey PRIMARY KEY + USING INDEX #{table_name}_id_bigint_index; + SQL + + query = query_start + fkey_query + query_end + DB.run(query) + end + + def drop_foreign_keys(_table_name, associations) + associations.map do |assoc_table, assoc_name| + constraint_name = "#{assoc_table}_#{assoc_name}_fkey" + fkey_query = <<~SQL.squish + ALTER TABLE #{assoc_table} DROP CONSTRAINT IF EXISTS #{constraint_name}; + SQL + DB.run(fkey_query) + end + end + + def create_foreign_keys(table_name, associations) + associations.map do |assoc_table, assoc_name| + constraint_name = "#{assoc_table}_#{assoc_name}_fkey" + fkey_query = <<~SQL.squish + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = '#{constraint_name}' + ) THEN + ALTER TABLE #{assoc_table} ADD CONSTRAINT #{constraint_name} + FOREIGN KEY (#{assoc_name}) REFERENCES #{table_name}(id) NOT VALID; + END IF; + END$$; + SQL + DB.run(fkey_query) + end + end + + def create_new_foreign_key_column(table_name, column_name) + DB.run(<<~SQL.squish) + LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE; + + CREATE OR REPLACE FUNCTION #{column_name}_bigint_trigger() + RETURNS trigger + AS + $BODY$ + DECLARE + BEGIN + NEW.#{column_name}_bigint := NEW.#{column_name}; + RETURN NEW; + END; + $BODY$ LANGUAGE plpgsql; + + ALTER TABLE #{table_name} ADD #{column_name}_bigint BIGINT; + + CREATE TRIGGER #{table_name}_#{column_name}_bigint + BEFORE INSERT OR UPDATE + ON #{table_name} + FOR EACH ROW + EXECUTE FUNCTION #{column_name}_bigint_trigger(); + SQL + end + + def check_id_consistency(table_name, col_name = "id") + res = DB[table_name].where( + Sequel[col_name.to_sym] !~ SQL.coalesce(Sequel[:"#{col_name}_bigint"], 0), + ).count + raise "Inconsistent ids in #{table_name}: #{res} records" if res.positive? + true + end + + # rubocop:disable Metrics/MethodLength + def drop_old_foreign_key_column(table_name, column_name, skip_constraint: false, + primary_key: [], uniq_constr: false) + query_start = <<~SQL.squish + LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE; + DROP TRIGGER #{table_name}_#{column_name}_bigint ON #{table_name}; + ALTER TABLE #{table_name} RENAME #{column_name} TO #{column_name}_integer; + ALTER TABLE #{table_name} RENAME #{column_name}_bigint TO #{column_name}; + SQL + + fkey_query = "" + unless skip_constraint + constraint_name = "#{table_name}_#{column_name}_fkey" + ref_table_name = column_name.to_s.delete_suffix("_id").pluralize + fkey_query = <<~SQL.squish + ALTER TABLE #{table_name} + DROP CONSTRAINT IF EXISTS #{constraint_name}, + ADD CONSTRAINT #{constraint_name} + FOREIGN KEY (#{column_name}) REFERENCES #{ref_table_name}(id) NOT VALID; + SQL + end + + drop_query = <<~SQL.squish + ALTER TABLE #{table_name} DROP #{column_name}_integer; + SQL + + constr_query = "" + if uniq_constr + constr_query = <<~SQL.squish + ALTER TABLE #{table_name} + ADD CONSTRAINT #{table_name}_#{column_name}_key UNIQUE (#{column_name}); + SQL + end + + pkey_query = "" + if primary_key.present? + pkey_query = <<~SQL.squish + ALTER TABLE #{table_name} ADD CONSTRAINT #{table_name}_pkey PRIMARY KEY + USING INDEX #{table_name}_#{primary_key.join("_")}_index; + SQL + end + + query = query_start + fkey_query + drop_query + constr_query + pkey_query + DB.run(query) + end + # rubocop:enable Metrics/MethodLength + + def check_associations(model, method, reverse_method) + model.dataset.limit(10).all.each do |record| + res = record.public_send(method).public_send(reverse_method) + raise StandardError if res.blank? + end + true + end + + def create_distributed_table!(table_name, sharding_key, db_name: UmbrellioUtils::ClickHouse.db_name) + UmbrellioUtils::ClickHouse.execute(<<~SQL.squish) + DROP TABLE IF EXISTS #{db_name}.#{table_name}_distributed + ON CLUSTER click_cluster + SQL + + UmbrellioUtils::ClickHouse.execute(<<~SQL.squish) + CREATE TABLE #{db_name}.#{table_name}_distributed + ON CLUSTER click_cluster + AS #{db_name}.#{table_name} + ENGINE = Distributed(click_cluster, #{db_name}, #{table_name}, #{sharding_key}) + SQL + end + + # @example + # add_columns_to_view( + # "orders_clickhouse_view", + # Sequel[:orders][:data].pg_jsonb.get_text("some_data_column").as(:some_column), + # Sequel[:orders][:column].as(:some_other_column), + # ) + def add_columns_to_view(view_name, *sequel_columns) + sequel_columns.each do |column| + unless column.is_a?(Sequel::SQL::AliasedExpression) + raise ArgumentError.new("not Sequel::SQL::AliasedExpression") + end + end + + DB.transaction do + DB.run("LOCK TABLE #{view_name}") + definition = view_definition(view_name) + sql = sequel_columns.map { |x| DB.literal(x) }.join(", ") + new_definition = definition.sub("FROM", ", #{sql} FROM") + DB.run("CREATE OR REPLACE VIEW #{view_name} AS #{new_definition}") + end + end + + # @example + # drop_columns_from_view("orders_clickhouse_view", "id", "guid") + def drop_columns_from_view(view_name, *columns) + DB.transaction do + DB.run("LOCK TABLE #{view_name}") + definition = view_definition(view_name) + parsed_columns = parse_columns(definition) + parsed_columns.reject! { |name, _| name.in?(columns) } + sql = parsed_columns.map { |_, sql| sql }.join(", ") + new_definition = definition.sub(/SELECT(.*?)FROM/i, "SELECT #{sql} FROM") + DB.run("DROP VIEW #{view_name}") + DB.run("CREATE VIEW #{view_name} AS #{new_definition}") + end + end + + private + + def parse_columns(definition) + fields_sql = definition[/SELECT(.*?)FROM/i, 1].strip + fields = fields_sql.scan(/(?:[^,(]+|\([^)]*\))+/).map(&:strip) + field_names = fields.map do |field| + field[/as (.*)/i, 1] || field[/\.(.*)\z/, 1] + end + field_names.zip(fields) + end + + def view_definition(view) + DB[:pg_views] + .where(viewname: view.to_s) + .select(:definition).first[:definition] + .squish + end + end +end diff --git a/lib/umbrellio_utils/misc.rb b/lib/umbrellio_utils/misc.rb index 62e157b..d66713e 100644 --- a/lib/umbrellio_utils/misc.rb +++ b/lib/umbrellio_utils/misc.rb @@ -4,6 +4,11 @@ module UmbrellioUtils module Misc extend self + class StrictHash < Hash + alias get [] + alias [] fetch + end + def table_sync(scope, delay: 1, routing_key: nil) scope.in_batches do |batch| batch_for_sync = batch.all.reject { |model| model.try(:skip_table_sync?) } diff --git a/lib/umbrellio_utils/sql.rb b/lib/umbrellio_utils/sql.rb new file mode 100644 index 0000000..c54081c --- /dev/null +++ b/lib/umbrellio_utils/sql.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +module UmbrellioUtils + module SQL + extend self + + UniqueConstraintViolation = Sequel::UniqueConstraintViolation + + def [](*args) + Sequel[*args] + end + + def func(...) + Sequel.function(...) + end + + def cast(...) + Sequel.cast(...) + end + + def case(...) + Sequel.case(...) + end + + def pg_jsonb(...) + Sequel.pg_jsonb(...) + end + + def to_utc(date) + func(:timezone, "UTC", date) + end + + def to_timezone(zone, date) + utc_date = to_utc(date) + func(:timezone, zone, cast(utc_date, :timestamptz)) + end + + def and(*conditions) + Sequel.&(*Array(conditions.flatten.presence || true)) + end + + def not(...) + Sequel.~(...) + end + + def or(*conditions) + Sequel.|(*Array(conditions.flatten.presence || true)) + end + + def pg_range(from_value, to_value, **opts) + Sequel::Postgres::PGRange.new(from_value, to_value, **opts) + end + + def pg_range_by_range(range) + Sequel::Postgres::PGRange.from_range(range) + end + + def max(expr) + func(:max, expr) + end + + def min(expr) + func(:min, expr) + end + + def sum(expr) + func(:sum, expr) + end + + def count(expr = nil) + expr ? func(:count, expr) : func(:count).* + end + + def ch_count(*args) + Sequel.function(:count, *args) + end + + def avg(expr) + func(:avg, expr) + end + + def pg_percentile(expr, percentile) + func(:percentile_cont, percentile).within_group(expr) + end + + def pg_median(expr) + pg_percentile(expr, 0.5) + end + + def ch_median(expr) + func(:median, expr) + end + + def abs(expr) + func(:abs, expr) + end + + def coalesce(*exprs) + func(:coalesce, *exprs) + end + + def coalesce0(*args) + coalesce(*args, 0) + end + + def nullif(main_expr, checking_expr) + func(:nullif, main_expr, checking_expr) + end + + def distinct(expr) + func(:distinct, expr) + end + + def least(*exprs) + func(:least, *exprs) + end + + def greatest(*exprs) + func(:greatest, *exprs) + end + + def date_trunc(truncate, expr) + func(:date_trunc, truncate.to_s, expr) + end + + def ch_timestamp(time) + time&.strftime("%F %T.%6N") + end + + def ch_timestamp_expr(time) + time = Time.zone.parse(time) if time.is_a?(String) + func(:toDateTime64, Sequel[ch_timestamp(time)], 6) + end + + def ch_time_range(range) + Range.new(ch_timestamp(range.begin), ch_timestamp(range.end), range.exclude_end?) + end + + def jsonb_dig(jsonb, path) + path.reduce(jsonb) { |acc, cur| acc[cur] } + end + + def jsonb_typeof(jsonb) + func(:jsonb_typeof, jsonb) + end + + def empty_jsonb + Sequel.pg_jsonb({}) + end + + def round(value, precision = 0) + func(:round, value, precision) + end + + def row(*values) + func(:row, *values) + end + + def map_to_expr(hash) + hash.map { |aliaz, expr| expr.as(aliaz) } + end + + def intersect(left_expr, right_expr) + Sequel.lit("SELECT ? INTERSECT SELECT ?", left_expr, right_expr) + end + + # can rewrite scalar values + def jsonb_unsafe_set(jsonb, path, value) + parent_path = path.slice(..-2) + raw_parent = jsonb_dig(jsonb, parent_path) + parent = jsonb_rewrite_scalar(raw_parent) + last_path = path.slice(-1..-1) + updated_parent = parent.set(last_path, value) + result = self.case({ { value => nil } => parent }, updated_parent) + jsonb.set(parent_path, result) + end + + def true + Sequel.lit("true") + end + + def false + Sequel.lit("false") + end + + private + + def jsonb_rewrite_scalar(jsonb) + self.case({ { jsonb_typeof(jsonb) => %w[object array] } => jsonb }, empty_jsonb).pg_jsonb + end + end +end diff --git a/lib/umbrellio_utils/tasks/clickhouse_connect.rake b/lib/umbrellio_utils/tasks/clickhouse_connect.rake new file mode 100644 index 0000000..dbdc5ff --- /dev/null +++ b/lib/umbrellio_utils/tasks/clickhouse_connect.rake @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +namespace :ch do + desc "run clickhouse client" + task connect: :environment do + params = { + host: ENV.fetch("CLICKHOUSE_HOST", UmbrellioUtils::ClickHouse.config.host), + user: ENV.fetch("CLICKHOUSE_USER", UmbrellioUtils::ClickHouse.config.username), + password: ENV.fetch("CLICKHOUSE_PASSWORD", UmbrellioUtils::ClickHouse.config.password), + database: ENV.fetch("CLICKHOUSE_DATABASE", UmbrellioUtils::ClickHouse.config.database), + **UmbrellioUtils::ClickHouse.config.global_params, + }.compact_blank + + cmd = Shellwords.join(["clickhouse", "client", *params.map { |k, v| "--#{k}=#{v}" }]) + exec(cmd) + end +end diff --git a/lib/umbrellio_utils/testing/sequel_patches.rb b/lib/umbrellio_utils/testing/sequel_patches.rb new file mode 100644 index 0000000..82346ca --- /dev/null +++ b/lib/umbrellio_utils/testing/sequel_patches.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.before(:suite) do + # Make Postgres return rows truly randomly in specs unless order is properly specified + class Sequel::Postgres::Dataset # rubocop:disable Lint/ConstantDefinitionInBlock + def select_sql + return super if @opts[:_skip_order_patch] || @opts[:append_sql] + return super if @opts[:ch] && @opts[:order].present? + order = @opts[:order].dup || [] + fn = @opts.key?(:ch) ? :rand : :random + order << Sequel.function(fn) + clone(order:, _skip_order_patch: true).select_sql + end + end + end +end diff --git a/lib/umbrellio_utils/version.rb b/lib/umbrellio_utils/version.rb index 9613d8d..495981b 100644 --- a/lib/umbrellio_utils/version.rb +++ b/lib/umbrellio_utils/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module UmbrellioUtils - VERSION = "1.9.0" + VERSION = "1.10.0" end diff --git a/spec/support/clickhouse.rb b/spec/support/clickhouse.rb new file mode 100644 index 0000000..404dfb9 --- /dev/null +++ b/spec/support/clickhouse.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "logger" +require "csv" +require "click_house" + +config = ClickHouse.config do |config| + config.assign(host: "localhost", database: "umbrellio_utils_test") + config.logger = Logger.new("log/ch.log") +end + +client = ClickHouse::Connection.new(config) + +client.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS test (id Int32) + ENGINE = MergeTree() + ORDER BY id; +SQL diff --git a/spec/support/database.rb b/spec/support/database.rb index 36125a1..fc4a558 100644 --- a/spec/support/database.rb +++ b/spec/support/database.rb @@ -4,7 +4,13 @@ begin db_name = "umbrellio_utils_test" - DB = Sequel.connect(ENV.fetch("DB_URL", "postgres:///#{db_name}")) + DB = Sequel.postgres( + "umbrellio_utils_test", + user: ENV.fetch("PGUSER"), + password: ENV.fetch("PGPASSWORD"), + host: "localhost", + port: 5432, + ) rescue Sequel::DatabaseConnectionError => error puts error abort "You probably need to create a test database. " \ @@ -16,6 +22,10 @@ Sequel::Model.db = DB DB.extension :batches +DB.extension :pg_json +DB.extension :pg_range + +Sequel.extension :pg_json_ops DB.drop_table? :users, cascade: true DB.create_table :users do @@ -43,6 +53,18 @@ foreign_key :user_id, :users end +DB.drop_table? :test_migrations, cascade: true +DB.create_table :test_migrations do + primary_key :id + column :test, :text +end + +DB.drop_table? :test_migration_references +DB.create_table :test_migration_references do + primary_key :id + foreign_key :test_migration_id, :test_migrations +end + class User < Sequel::Model(:users) def skip_table_sync? false @@ -68,3 +90,12 @@ def skip_table_sync? false end end + +class TestMigration < Sequel::Model(:test_migrations) + one_to_many :test_migration_references, + class_name: "TestMigrationReference", key: :test_migration_id, primary_key: :id +end + +class TestMigrationReference < Sequel::Model(:test_migration_references) + many_to_one :test_migration, class_name: "TestMigration", key: :test_migration_id +end diff --git a/spec/support/sequel_patches.rb b/spec/support/sequel_patches.rb index 45a90e3..7df39a4 100644 --- a/spec/support/sequel_patches.rb +++ b/spec/support/sequel_patches.rb @@ -1,15 +1,3 @@ # frozen_string_literal: true -RSpec.configure do |config| - config.before(:suite) do - # Make Postgres return rows truly randomly in specs unless order is properly specified - class Sequel::Postgres::Dataset # rubocop:disable Lint/ConstantDefinitionInBlock - def select_sql - return super if @opts[:_skip_order_patch] - order = @opts[:order].dup || [] - order << Sequel.function(:random) - clone(order:, _skip_order_patch: true).select_sql - end - end - end -end +require "umbrellio_utils/testing/sequel_patches" diff --git a/spec/umbrellio_utils/clickhouse_spec.rb b/spec/umbrellio_utils/clickhouse_spec.rb new file mode 100644 index 0000000..6698aab --- /dev/null +++ b/spec/umbrellio_utils/clickhouse_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +describe UmbrellioUtils::ClickHouse do + let(:ch) { described_class } + + before do + ch.truncate_table!("test") + ch.insert("test", rows: [{ id: 1 }, { id: 2 }, { id: 3 }]) + ch.optimize_table!("test") # just for coverage + end + + describe "#from" do + context "with another db" do + specify do + expect(ch.from(:test, db_name: :test).sql).to eq( + 'SELECT * FROM "test"."test" ORDER BY rand()', + ) + end + end + + context "with nil" do + specify do + expect(ch.from(nil).sql).to eq("SELECT * ORDER BY rand()") + end + end + + context "with another source" do + specify do + expect(ch.from(ch.from(:test)).sql).to eq( + 'SELECT * FROM (SELECT * FROM "test") AS "t1" ORDER BY rand()'.b, + ) + end + end + end + + describe "#query" do + specify do + query = ch.from(:test).order(:id).select(:id) + expect(ch.query(query)).to eq([{ id: 1 }, { id: 2 }, { id: 3 }]) + end + end + + describe "#query_value" do + specify do + query = ch.from(:test).order(:id).select(:id) + expect(ch.query_value(query)).to eq(1) + end + end + + describe "#count" do + specify do + query = ch.from(:test) + expect(ch.count(query)).to eq(3) + end + end + + describe "#describe_table" do + specify do + expect(ch.describe_table("test")).to eq( + [ + codec_expression: "", + comment: "", + default_expression: "", + default_type: "", + name: "id", + ttl_expression: "", + type: "Int32", + ], + ) + end + end + + describe "#db_name" do + specify do + expect(ch.db_name).to eq(:umbrellio_utils_test) + end + end + + describe "#server_version" do + specify do + expect(ch.server_version).to match(Numeric) + end + end + + describe "#with_temp_table" do + let(:result) { [] } + let(:dataset) { ch.from(:test).order(:id) } + + context "when no temp table" do + before { DB.drop_table?(:some_test_table) } + + specify do + ch.with_temp_table(dataset, temp_table_name: "some_test_table", page_size: 1) do |batch| + result << batch + end + expect(result).to eq([[3], [2], [1]]) + end + end + + context "when table already exist" do + before { DB.create_table("some_test_table") { primary_key :id } } + before { DB[:some_test_table].multi_insert([{ id: 4 }, { id: 5 }, { id: 6 }]) } + + it "takes from existing" do + ch.with_temp_table(dataset, temp_table_name: "some_test_table", page_size: 1) do |batch| + result << batch + end + expect(result).to eq([[6], [5], [4]]) + end + end + end + + describe "#execute" do + after { ch.drop_table!("test2") } + + specify do + ch.execute("CREATE TABLE test2 (id Int64) ENGINE=MergeTree() ORDER BY id;") + end + end + + describe "#parse_value" do + it "parses string" do + expect(ch.parse_value("123", type: "String")).to eq("123") + end + + it "parses nil as string" do + expect(ch.parse_value(nil, type: "String")).to be_nil + end + + it "parses time" do + expect(ch.parse_value("2020-01-01", type: "DateTime")).to eq(Time.zone.parse("2020-01-01")) + end + + it "parses integer" do + expect(ch.parse_value(123, type: "Int32")).to eq(123) + end + end +end diff --git a/spec/umbrellio_utils/migrations_spec.rb b/spec/umbrellio_utils/migrations_spec.rb new file mode 100644 index 0000000..c42d117 --- /dev/null +++ b/spec/umbrellio_utils/migrations_spec.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +describe UmbrellioUtils::Migrations do + before do + DB.drop_table? :test_migrations, cascade: true + DB.create_table :test_migrations do + primary_key :id + column :test, :text + end + + DB.run("CREATE OR REPLACE VIEW test_migrations_view AS SELECT id from users") + + DB.drop_table? :test_migration_references + DB.create_table :test_migration_references do + primary_key :id + foreign_key :test_migration_id, :test_migrations + end + end + + before do + DB[:test_migrations].multi_insert(test_data) + DB[:test_migration_references].multi_insert(test_reference_data) + end + + let(:test_data) do + Array.new(10) { |index| Hash[id: index + 1, test: index.to_s] } + end + + let(:test_reference_data) do + Array.new(10) { |index| Hash[id: index + 1, test_migration_id: index + 1] } + end + + context "with migrate to bigint" do + def expected_foreign_key + DB.foreign_key_list(:test_migration_references).first + end + + def check_contains_fk! + expect(expected_foreign_key).to include( + columns: [:test_migration_id], table: :test_migrations, + ) + end + + def check_contains_no_fk! + expect(expected_foreign_key).to be_nil + end + + let(:associations) { Hash[test_migration_references: :test_migration_id] } + + it "migrates to bigint column" do + described_class.create_new_id_bigint_column(:test_migrations) + expect(DB[:test_migrations].columns).to eq(%i[id test id_bigint]) # creates id_bigint column + + # contains trigger which copy from id column + DB[:test_migrations].insert(id: 11, test: 11) + expect(DB[:test_migrations].first(id: 11)).to eq(id: 11, test: "11", id_bigint: 11) + + DB[:test_migrations].update(id_bigint: :id) + DB.alter_table(:test_migrations) { add_index :id_bigint, unique: true } + described_class.check_id_consistency(:test_migrations) + + described_class.drop_old_id_column(:test_migrations, associations) + DB[:test_migrations].send(:clear_columns_cache) + expect(DB[:test_migrations].columns).to eq(%i[test id]) + expect(DB.schema(:test_migrations).to_h[:id]).to include(db_type: "bigint", primary_key: true) + check_contains_fk! + end + + it "updates foreign key to bigint" do + described_class.create_new_foreign_key_column(:test_migration_references, :test_migration_id) + expect(DB[:test_migration_references].columns).to eq( # creates bigint column + %i[id test_migration_id test_migration_id_bigint], + ) + + # contains trigger which copy from test_migration_id column + DB[:test_migrations].insert(id: 11, test: 11) + DB[:test_migration_references].insert(id: 11, test_migration_id: 11) + expect(DB[:test_migration_references].first(id: 11)).to eq( + id: 11, test_migration_id: 11, test_migration_id_bigint: 11, + ) + + DB[:test_migration_references].update(test_migration_id: :id) + + described_class.drop_old_foreign_key_column(:test_migration_references, :test_migration_id) + DB[:test_migration_references].send(:clear_columns_cache) + expect(DB[:test_migration_references].columns).to eq(%i[id test_migration_id]) + expect(DB[:test_migration_references].first(id: 11)).to eq(id: 11, test_migration_id: 11) + type = DB.schema(:test_migration_references).to_h.dig(:test_migration_id, :db_type) + expect(type).to eq("bigint") + check_contains_fk! + end + + context "with skip create fk" do + it "doesn't create fk" do + described_class.create_new_id_bigint_column(:test_migrations) + DB[:test_migrations].update(id_bigint: :id) + DB.alter_table(:test_migrations) { add_index :id_bigint, unique: true } + described_class.drop_old_id_column(:test_migrations, associations, skip_fk_create: true) + + check_contains_no_fk! + end + end + + context "with drop and create foreign keys" do + specify do + described_class.drop_foreign_keys(:test_migration_references, associations) + check_contains_no_fk! + described_class.create_foreign_keys(:test_migrations, associations) + check_contains_fk! + end + end + end + + describe "#check_id_consistency" do + specify do + described_class.create_new_id_bigint_column(:test_migrations) + expect { described_class.check_id_consistency(:test_migrations) }.to raise_error( + RuntimeError, /Inconsistent ids in test_migrations: 10 records/ + ) + end + end + + describe "#check_associations" do + specify do + result = described_class.check_associations( + TestMigrationReference, :test_migration, :test_migration_references + ) + expect(result).to be_truthy + end + + context "with invalid" do + it "raises error" do + stub_const("TestMigrationReference", Class.new(Sequel::Model(:test_migrations)) do + def test_migration + Struct.new(:test_migration_references).new([]) + end + end) + + expect do + described_class.check_associations( + TestMigrationReference, :test_migration, :test_migration_references + ) + end.to raise_error(StandardError) + end + end + end + + describe "#create_distributed_table" do + specify do + described_class.create_distributed_table!("test", "id") + expect(UmbrellioUtils::ClickHouse.describe_table("test_distributed")).to be_present + end + end + + context "with view" do + specify do + described_class.add_columns_to_view("test_migrations_view", Sequel[:email].as(:test)) + expect(DB[:test_migrations_view].columns).to eq(%i[id test]) + + described_class.drop_columns_from_view("test_migrations_view", "test") + DB[:test_migrations_view].send(:clear_columns_cache) + expect(DB[:test_migrations_view].columns).to eq(%i[id]) + end + end +end diff --git a/spec/umbrellio_utils/sql_spec.rb b/spec/umbrellio_utils/sql_spec.rb new file mode 100644 index 0000000..f42126b --- /dev/null +++ b/spec/umbrellio_utils/sql_spec.rb @@ -0,0 +1,250 @@ +# frozen_string_literal: true + +describe UmbrellioUtils::SQL do + let(:sql) { described_class } + + subject(:result) { DB.literal(expr) } + + describe "#[]" do + let(:expr) { sql[:test] } + + specify { expect(result).to eq('"test"') } + end + + describe "#func" do + let(:expr) { sql.func(:some_function, :test) } + + specify { expect(result).to eq('some_function("test")') } + end + + describe "#cast" do + let(:expr) { sql.cast(:test, :integer) } + + specify { expect(result).to eq('CAST("test" AS integer)') } + end + + describe "#case" do + let(:expr) { sql.case({ sql[:test] =~ sql[:test2] => 1 }, 2) } + + specify { expect(result).to eq('(CASE WHEN ("test" = "test2") THEN 1 ELSE 2 END)') } + end + + describe "#pg_jsonb" do + let(:expr) { sql.pg_jsonb(test: 123) } + + specify { expect(result).to eq(%('{"test":123}'::jsonb)) } + end + + describe "#and" do + let(:expr) { sql.and({ test: 123 }, { test2: 321 }) } + + specify { expect(result).to eq('(("test" = 123) AND ("test2" = 321))') } + end + + describe "#or" do + let(:expr) { sql.or({ test: 123 }, { test2: 321 }) } + + specify { expect(result).to eq('(("test" = 123) OR ("test2" = 321))') } + end + + describe "#to_utc" do + let(:expr) { sql.to_utc("2020-01-01 00:00:00.000000") } + + specify { expect(result).to eq("timezone('UTC', '2020-01-01 00:00:00.000000')") } + end + + describe "#to_timezone" do + let(:expr) { sql.to_timezone("UTC+6", "2020-01-01 00:00:00.000000") } + + specify do + expect(result).to eq( + "timezone('UTC+6', CAST(timezone('UTC', '2020-01-01 00:00:00.000000') AS timestamptz))", + ) + end + end + + describe "#pg_range" do + let(:expr) { sql.pg_range(Time.zone.parse("2014-01-01"), Time.zone.parse("2015-01-01")) } + + specify do + expect(result).to eq("'[2014-01-01 00:00:00.000000+0000,2015-01-01 00:00:00.000000+0000]'") + end + end + + describe "#pg_range_by_range" do + let(:expr) do + sql.pg_range_by_range(Time.zone.parse("2014-01-01")..Time.zone.parse("2015-01-01")) + end + + specify do + expect(result).to eq("'[2014-01-01 00:00:00.000000+0000,2015-01-01 00:00:00.000000+0000]'") + end + end + + describe "#coalesce0" do + let(:expr) { sql.coalesce0("test") } + + specify { expect(result).to eq("coalesce('test', 0)") } + end + + describe "#nullif" do + let(:expr) { sql.nullif(1, 1) } + + specify { expect(result).to eq("nullif(1, 1)") } + end + + %w[max min sum avg abs coalesce least greatest distinct jsonb_typeof row].each do |function| + describe "##{function}" do + let(:expr) { sql.public_send(function, :test) } + + specify { expect(result).to eq(%(#{function}("test"))) } + end + end + + describe "#count" do + context "with expression" do + let(:expr) { sql.count(:test) } + + specify { expect(result).to eq('count("test")') } + end + + context "without expression" do + let(:expr) { sql.count } + + specify { expect(result).to eq("count(*)") } + end + end + + describe "#ch_count" do + let(:expr) { sql.ch_count(:test) } + + specify { expect(result).to eq('count("test")') } + end + + describe "#pg_percentile" do + let(:expr) { sql.pg_percentile(:test, 0.1) } + + specify { expect(result).to eq('percentile_cont(0.1) WITHIN GROUP (ORDER BY "test")') } + end + + describe "#pg_median" do + let(:expr) { sql.pg_median(:test) } + + specify { expect(result).to eq('percentile_cont(0.5) WITHIN GROUP (ORDER BY "test")') } + end + + describe "#ch_median" do + let(:expr) { sql.ch_median(:test) } + + specify { expect(result).to eq('median("test")') } + end + + describe "#date_trunc" do + let(:expr) { sql.date_trunc("hour", :test) } + + specify { expect(result).to eq(%(date_trunc('hour', "test"))) } + end + + describe "#ch_timestamp" do + context "with time" do + let(:expr) { sql.ch_timestamp(Time.zone.parse("2020-01-01")) } + + specify { expect(result).to eq("'2020-01-01 00:00:00.000000'") } + end + + context "with nil" do + let(:expr) { sql.ch_timestamp(nil) } + + specify { expect(result).to eq("NULL") } + end + end + + describe "#ch_timestamp_expr" do + context "with time" do + let(:expr) { sql.ch_timestamp_expr(Time.zone.parse("2020-01-01")) } + + specify { expect(result).to eq("toDateTime64('2020-01-01 00:00:00.000000', 6)") } + end + + context "with string" do + let(:expr) { sql.ch_timestamp_expr("2020-01-01") } + + specify { expect(result).to eq("toDateTime64('2020-01-01 00:00:00.000000', 6)") } + end + end + + describe "#ch_time_range" do + let(:expr) { sql.ch_time_range(Time.zone.parse("2020-01-01")..Time.zone.parse("2020-01-02")) } + + specify { expect(result).to eq("'[2020-01-01 00:00:00.000000,2020-01-02 00:00:00.000000]'") } + end + + describe "#jsonb_dig" do + let(:expr) { sql.jsonb_dig(sql[:test].pg_jsonb, %w[test test2]) } + + specify { expect(result).to eq(%("test"['test']['test2'])) } + end + + describe "#empty_jsonb" do + let(:expr) { sql.empty_jsonb } + + specify { expect(result).to eq("'{}'::jsonb") } + end + + describe "#round" do + let(:expr) { sql.round(:test, 5) } + + specify { expect(result).to eq('round("test", 5)') } + end + + describe "#map_to_expr" do + let(:expr) { sql.map_to_expr({ test: Sequel[:test1], test2: Sequel[:some] }) } + + specify { expect(result).to eq('("test1" AS "test", "some" AS "test2")') } + end + + describe "#intersect" do + let(:expr) { sql.intersect(:test, :test2) } + + specify { expect(result).to eq('SELECT "test" INTERSECT SELECT "test2"') } + end + + describe "#jsonb_unsafe_set" do + let(:expr) { sql.jsonb_unsafe_set(sql[:test].pg_jsonb, %w[test test2], 123) } + + specify do + expect(result).to eq( + <<~SQL.squish.gsub("( ", "(").gsub(" )", ")"), + jsonb_set( + "test", ('test'), + (CASE WHEN (123 IS NULL) THEN ( + CASE WHEN ( + jsonb_typeof("test"['test']) IN ('object', 'array') + ) THEN "test"['test'] ELSE '{}'::jsonb END + ) ELSE jsonb_set( + (CASE WHEN ( + jsonb_typeof("test"['test']) IN ('object', 'array') + ) THEN "test"['test'] ELSE '{}'::jsonb END), + ('test2'), + 123, + true + ) END + ), + true) + SQL + ) + end + end + + describe "#true" do + let(:expr) { sql.true } + + specify { expect(result).to eq("true") } + end + + describe "#false" do + let(:expr) { sql.false } + + specify { expect(result).to eq("false") } + end +end diff --git a/umbrellio_utils.gemspec b/umbrellio_utils.gemspec index f9566d6..d46ce19 100644 --- a/umbrellio_utils.gemspec +++ b/umbrellio_utils.gemspec @@ -23,8 +23,8 @@ Gem::Specification.new do |spec| spec.files = Dir.chdir(File.expand_path(__dir__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.bindir = "bin" + spec.executables = ["clickhouse-server"] spec.require_paths = ["lib"] spec.add_dependency "memery", "~> 1"