Skip to content

Commit

Permalink
resolves #3
Browse files Browse the repository at this point in the history
  • Loading branch information
Romil Punetha committed Nov 24, 2022
1 parent e384d38 commit 666faa9
Show file tree
Hide file tree
Showing 15 changed files with 825 additions and 12 deletions.
17 changes: 16 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,16 @@
/target
**/*target
/.idea
# Generated by Cargo
# will have compiled files and executables
debug/
target/

# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock

# These are backup files generated by rustfmt
**/*.rs.bk

# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
13 changes: 5 additions & 8 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
[package]
name = "eventscope"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
[workspace]
resolver = "1"
members = [
"eventscope",
]
370 changes: 370 additions & 0 deletions docker/clickhouse-kafka/config/clickhouse_config.xml
View file

Large diffs are not rendered by default.

59 changes: 59 additions & 0 deletions docker/clickhouse-kafka/config/clickhouse_metrika.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
<yandex>
<clickhouse_remote_servers>
<cluster_1>
<shard>
<weight>1</weight>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-01</host>
<port>9000</port>
</replica>
<!-- <replica>-->
<!-- <host>clickhouse-06</host>-->
<!-- <port>9000</port>-->
<!-- </replica>-->
</shard>
<!-- <shard>-->
<!-- <weight>1</weight>-->
<!-- <internal_replication>true</internal_replication>-->
<!-- <replica>-->
<!-- <host>clickhouse-02</host>-->
<!-- <port>9000</port>-->
<!-- </replica>-->
<!-- <replica>-->
<!-- <host>clickhouse-03</host>-->
<!-- <port>9000</port>-->
<!-- </replica>-->
<!-- </shard>-->
<!-- <shard>-->
<!-- <weight>1</weight>-->
<!-- <internal_replication>true</internal_replication>-->

<!-- <replica>-->
<!-- <host>clickhouse-04</host>-->
<!-- <port>9000</port>-->
<!-- </replica>-->
<!-- <replica>-->
<!-- <host>clickhouse-05</host>-->
<!-- <port>9000</port>-->
<!-- </replica>-->
<!-- </shard>-->
</cluster_1>
</clickhouse_remote_servers>
<zookeeper-servers>
<node index="1">
<host>clickhouse-zookeeper</host>
<port>2181</port>
</node>
</zookeeper-servers>
<networks>
<ip>::/0</ip>
</networks>
<clickhouse_compression>
<case>
<min_part_size>10000000000</min_part_size>
<min_part_size_ratio>0.01</min_part_size_ratio>
<method>lz4</method>
</case>
</clickhouse_compression>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-01.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-01</replica>
<shard>01</shard>
<layer>01</layer>
</macros>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-02.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-02</replica>
<shard>02</shard>
<layer>01</layer>
</macros>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-03.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-03</replica>
<shard>02</shard>
<layer>01</layer>
</macros>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-04.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-04</replica>
<shard>03</shard>
<layer>01</layer>
</macros>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-05.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-05</replica>
<shard>03</shard>
<layer>01</layer>
</macros>
</yandex>
7 changes: 7 additions & 0 deletions docker/clickhouse-kafka/config/macros/macros-06.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<yandex>
<macros>
<replica>clickhouse-06</replica>
<shard>01</shard>
<layer>01</layer>
</macros>
</yandex>
125 changes: 125 additions & 0 deletions docker/clickhouse-kafka/config/users.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
<?xml version="1.0"?>
<yandex>
<!-- Profiles of settings. -->
<profiles>
<!-- Default settings. -->
<default>
<!-- Maximum memory usage for processing single query, in bytes. -->
<max_memory_usage>10000000000</max_memory_usage>
<allow_experimental_object_type>1</allow_experimental_object_type>
<allow_experimental_window_functions>1</allow_experimental_window_functions>
<allow_nondeterministic_mutations>1</allow_nondeterministic_mutations>
<date_time_input_format>best_effort</date_time_input_format>
<date_time_output_format>iso</date_time_output_format>
<input_format_avro_allow_missing_fields>1</input_format_avro_allow_missing_fields>
<stream_like_engine_allow_direct_select>1</stream_like_engine_allow_direct_select>
<input_format_skip_unknown_fields>1</input_format_skip_unknown_fields>
<input_format_import_nested_json>1</input_format_import_nested_json>
<!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. -->
<use_uncompressed_cache>0</use_uncompressed_cache>

<!-- How to choose between replicas during distributed query processing.
random - choose random replica from set of replicas with minimum number of errors
nearest_hostname - from set of replicas with minimum number of errors, choose replica
with minumum number of different symbols between replica's hostname and local hostname
(Hamming distance).
in_order - first live replica is choosen in specified order.
-->
<load_balancing>random</load_balancing>
</default>

<!-- Profile that allows only read queries. -->
<readonly>
<readonly>1</readonly>
</readonly>
</profiles>

<!-- Users and ACL. -->
<users>
<user1>
<password>123456</password>
<networks incl="networks" replace="replace">
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
</user1>
<!-- If user name was not specified, 'default' user is used. -->
<default>
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
If you want to specify password in plaintext (not recommended), place it in 'password' element.
Example: <password>qwerty</password>.
Password could be empty.
If you want to specify SHA256, place it in 'password_sha256_hex' element.
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
How to generate decent password:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
In first line will be password and in second - corresponding SHA256.
-->
<password></password>

<!-- List of networks with open access.
To open access from everywhere, specify:
<ip>::/0</ip>
To open access only from localhost, specify:
<ip>::1</ip>
<ip>127.0.0.1</ip>
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
To check access, DNS PTR query is performed for peer address and then regexp is applied.
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
Strongly recommended that regexp is ends with $
All results of DNS requests are cached till server restart.
-->
<networks incl="networks" replace="replace">
<ip>::/0</ip>
</networks>

<!-- Settings profile for user. -->
<profile>default</profile>

<!-- Quota for user. -->
<quota>default</quota>
</default>

<!-- Example of user with readonly access. -->
<readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly>
</users>

<!-- Quotas. -->
<quotas>
<!-- Name of quota. -->
<default>
<!-- Limits for time interval. You could specify many intervals with different limits. -->
<interval>
<!-- Length of interval. -->
<duration>3600</duration>

<!-- No limits. Just calculate resource usage for time interval. -->
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</yandex>
60 changes: 60 additions & 0 deletions docker/clickhouse-kafka/docker-compose.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
version: '3'
services:
kafka:
image: docker.io/bitnami/kafka:3.2
ports:
- "9093:9093"
environment:
- KAFKA_CFG_ZOOKEEPER_CONNECT=clickhouse-zookeeper:2181
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://0.0.0.0:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://clickhouse-kafka:9092,EXTERNAL://localhost:9093
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
container_name: clickhouse-kafka
hostname: clickhouse-kafka
depends_on:
- clickhouse-zookeeper
clickhouse-zookeeper:
image: zookeeper
ports:
- "2181:2181"
- "2182:2182"
container_name: clickhouse-zookeeper
hostname: clickhouse-zookeeper

kafka-ui:
image: provectuslabs/kafka-ui:latest
ports:
- "8080:8080"
environment:
- KAFKA_CLUSTERS_0_NAME=local
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092
container_name: kafka-ui
hostname: kafka-ui
depends_on:
- kafka

clickhouse-01:
image: clickhouse/clickhouse-server:22.8.5.29
hostname: clickhouse-01
container_name: clickhouse-01
ports:
- 9001:9000
- 8123:8123
volumes:
- ./config/clickhouse_config.xml:/etc/clickhouse-server/config.xml
- ./config/clickhouse_metrika.xml:/etc/clickhouse-server/metrika.xml
- ./config/macros/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml
- ./config/users.xml:/etc/clickhouse-server/users.xml
ulimits:
nofile:
soft: 262144
hard: 262144
depends_on:
- "clickhouse-zookeeper"

networks:
default:
external:
name: clickhouse-net
16 changes: 16 additions & 0 deletions eventscope/Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[package]
name = "eventscope"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
actix-web = "4.2.1"
serde = { version = "1.0.147", features = ["derive"] }
serde_json = "1.0.88"
clickhouse-rs = "1.0.0-alpha.1"
time = { version = "0.3.17", features = ["serde"] }
uuid = { version = "1.1.2", features = ["v4", "serde"] }
big_s = "1.0.2"
flatten-serde-json = "0.1.0"
Loading

0 comments on commit 666faa9

Please sign in to comment.