Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[enhancement][docker]update kafka docker file for branch-2.0" #43180

Merged
merged 12 commits into from
Nov 5, 2024
Merged

[enhancement][docker]update kafka docker file for branch-2.0" #43180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
204d419
update run-thirdparties-dorcker.sh for p0 testing
Oct 16, 2023
9e8aee9
Merge branch 'apache:branch-2.0' into branch-2.0
XuJianxu Oct 17, 2023
40f0ef9
Merge branch 'apache:branch-2.0' into branch-2.0
XuJianxu Jan 30, 2024
b38a5c4
[enhancement][docker]update kafka docker file for branch-2.0
XuJianxu Nov 4, 2024
d724939
add kafka dependency
XuJianxu Nov 4, 2024
c071de7
remove workload group for routine load
XuJianxu Nov 4, 2024
2e284f6
add kafka data for routine load test cases
XuJianxu Nov 4, 2024
c4dc3e5
update routine load cases
XuJianxu Nov 4, 2024
9d8d38f
update routine load cases
XuJianxu Nov 4, 2024
22e160a
update routine load case
XuJianxu Nov 5, 2024
b130340
update test routine laod cases
XuJianxu Nov 5, 2024
013f881
update routine load cases
XuJianxu Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions docker/thirdparties/run-thirdparties-docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,33 +261,25 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
sed -i "s/localhost/${IP_HOST}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env down
start_kafka_producers() {
local container_id="$1"
local ip_host="$2"

declare -a topics=("basic_data" "basic_array_data" "basic_data_with_errors" "basic_array_data_with_errors" "basic_data_timezone" "basic_array_data_timezone" "multi_table_csv" "multi_table_csv1")
create_kafka_topics() {
local container_id="$1"
local ip_host="$2"
local backup_dir=/home/work/pipline/backup_center

declare -a topics=("basic_data" "basic_array_data" "basic_data_with_errors" "basic_array_data_with_errors" "basic_data_timezone" "basic_array_data_timezone")

for topic in "${topics[@]}"; do
while IFS= read -r line; do
docker exec "${container_id}" bash -c "echo '$line' | /opt/kafka/bin/kafka-console-producer.sh --broker-list '${ip_host}:19193' --topic '${topic}'"
done < "${ROOT}/docker-compose/kafka/scripts/${topic}.csv"
echo "docker exec "${container_id}" bash -c echo '/opt/kafka/bin/kafka-topics.sh --create --broker-list '${ip_host}:19193' --partitions 10' --topic '${topic}'"
docker exec "${container_id}" bash -c "/opt/kafka/bin/kafka-topics.sh --create --broker-list '${ip_host}:19193' --partitions 10' --topic '${topic}'"
done

declare -a json_topics=("basic_data_json" "basic_array_data_json" "basic_array_data_json_by_line" "basic_data_json_by_line" "multi_table_json" "multi_table_json1")

for json_topic in "${json_topics[@]}"; do
echo ${json_topics}
while IFS= read -r json_line; do
docker exec "${container_id}" bash -c "echo '$json_line' | /opt/kafka/bin/kafka-console-producer.sh --broker-list '${ip_host}:19193' --topic '${json_topic}'"
echo "echo '$json_line' | /opt/kafka/bin/kafka-console-producer.sh --broker-list '${ip_host}:19193' --topic '${json_topic}'"
done < "${ROOT}/docker-compose/kafka/scripts/${json_topic}.json"
done
}

if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env up --build --remove-orphans -d
sleep 30s
start_kafka_producers "${KAFKA_CONTAINER_ID}" "${IP_HOST}"
sleep 10s
create_kafka_topics "${KAFKA_CONTAINER_ID}" "${IP_HOST}"
fi
fi

Expand Down
5 changes: 5 additions & 0 deletions regression-test/framework/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ under the License.
<version>${groovy.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
Expand Down
20 changes: 20 additions & 0 deletions regression-test/suites/load_p0/routine_load/data/basic_array_data.csv
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions regression-test/suites/load_p0/routine_load/data/basic_array_data_json.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"k00": "2", "k01": "[0, 0, 0, 0, 0, 0]", "k02": "[117, 117, 117, 117, 117, 117]", "k03": "[-4744, -4744, -4744, -4744, -4744, -4744]", "k04": "[-1593211961, -1593211961, -1593211961, -1593211961, -1593211961, -1593211961]", "k05": "[-3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780]", "k06": "[8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567]", "k07": "[-30948.857, -30948.857, -30948.857, -30948.857, -30948.857]", "k08": "[804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905]", "k09": "[-74019648, -74019648, -74019648, -74019648, -74019648, -74019648]", "k10": "[13024168, 13024168, 13024168, 13024168, 13024168, 13024168]", "k11": "[2023-08-22, 2023-08-22, 2023-08-22, 2023-08-22, 2023-08-22, 2023-08-22]", "k12": "[2022-09-30 07:47:12, 2022-09-30 07:47:12, 2022-09-30 07:47:12, 2022-09-30 07:47:12, 2022-09-30 07:47:12, 2022-09-30 07:47:12]", "k13": "[2023-04-21, 2023-04-21, 2023-04-21, 2023-04-21, 2023-04-21, 2023-04-21]", "k14": "[2022-11-24 15:07:56, 2022-11-24 15:07:56, 2022-11-24 15:07:56, 2022-11-24 15:07:56, 2022-11-24 15:07:56, 2022-11-24 15:07:56]", "k15": "['g', 'g', 'g', 'g', 'g', 'g']", "k16": "['a', 'a', 'a', 'a', 'a', 'a']", "k17": "['S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl', 'S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl', 'S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl', 'S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl', 'S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl', 'S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl']"}]
Loading
Loading