Skip to content

Commit

Permalink
[test](hive) run suite cases both in hive2 and hive3 (apache#33874)
Browse files Browse the repository at this point in the history
Followup: apache#33115
There are hive2 and hive3 dockers in regress-test, so the the original test cases need to be tested in hive2 and hive3 environments.
  • Loading branch information
suxiaogang223 authored and morningman committed Apr 26, 2024
1 parent 9aa08d8 commit e7aec24
Show file tree
Hide file tree
Showing 92 changed files with 137,305 additions and 1,711 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
HIVE_SITE_CONF_hive_server2_thrift_port=10000
HIVE_SITE_CONF_hive_compactor_initiator_on=true
HIVE_SITE_CONF_hive_compactor_worker_threads=2
HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader

CORE_CONF_fs_defaultFS=hdfs://${IP_HOST}:${FS_PORT}
Expand Down
2 changes: 2 additions & 0 deletions docker/thirdparties/docker-compose/hive/hadoop-hive.env
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
HIVE_SITE_CONF_hive_server2_thrift_port=10000
HIVE_SITE_CONF_hive_compactor_initiator_on=true
HIVE_SITE_CONF_hive_compactor_worker_threads=2
HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader

CORE_CONF_fs_defaultFS=hdfs://namenode:8020
Expand Down
8 changes: 4 additions & 4 deletions docker/thirdparties/docker-compose/hive/hive-2x_settings.env
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# Do not use "_" or other sepcial characters, only number and alphabeta.
# NOTICE: change this uid will modify hive-*.yaml

export FS_PORT=8220 #should be same in regression-conf.groovy
export HMS_PORT=9283 #should be same in regression-conf.groovy
export HS_PORT=12000 #should be same in regression-conf.groovy
export PG_PORT=5632 #should be same in regression-conf.groovy
export FS_PORT=8020 #should be same as hive2HdfsPort in regression-conf.groovy
export HMS_PORT=9083 #should be same as hive2HmsPort in regression-conf.groovy
export HS_PORT=10000 #should be same as hive2ServerPort in regression-conf.groovy
export PG_PORT=5432 #should be same as hive2PgPort in regression-conf.groovy
8 changes: 4 additions & 4 deletions docker/thirdparties/docker-compose/hive/hive-3x_settings.env
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# Do not use "_" or other sepcial characters, only number and alphabeta.
# NOTICE: change this uid will modify hive-*.yaml

export FS_PORT=8020 #should be same in regression-conf.groovy
export HMS_PORT=9083 #should be same in regression-conf.groovy
export HS_PORT=10000 #should be same in regression-conf.groovy
export PG_PORT=5432 #should be same in regression-conf.groovy
export FS_PORT=8320 #should be same as hive3HdfsPort in regression-conf.groovy
export HMS_PORT=9383 #should be same as hive3HmsPort in regression-conf.groovy
export HS_PORT=13000 #should be same as hive3ServerPort in regression-conf.groovy
export PG_PORT=5732 #should be same as hive3PgPort in regression-conf.groovy
16 changes: 8 additions & 8 deletions docker/thirdparties/run-thirdparties-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ Usage: $0 <options>
[no option] start all components
--help,-h show this usage
-c mysql start MySQL
-c mysql,hive start MySQL and Hive
-c mysql,hive3 start MySQL and Hive3
--stop stop the specified components
All valid components:
mysql,pg,oracle,sqlserver,clickhouse,es,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2
mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2
"
exit 1
}
Expand All @@ -60,7 +60,7 @@ STOP=0

if [[ "$#" == 1 ]]; then
# default
COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
else
while true; do
case "$1" in
Expand Down Expand Up @@ -92,7 +92,7 @@ else
done
if [[ "${COMPONENTS}"x == ""x ]]; then
if [[ "${STOP}" -eq 1 ]]; then
COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
fi
fi
fi
Expand Down Expand Up @@ -125,7 +125,7 @@ RUN_PG=0
RUN_ORACLE=0
RUN_SQLSERVER=0
RUN_CLICKHOUSE=0
RUN_HIVE=0
RUN_HIVE2=0
RUN_HIVE3=0;
RUN_ES=0
RUN_ICEBERG=0
Expand All @@ -149,8 +149,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
RUN_CLICKHOUSE=1
elif [[ "${element}"x == "es"x ]]; then
RUN_ES=1
elif [[ "${element}"x == "hive"x ]]; then
RUN_HIVE=1
elif [[ "${element}"x == "hive2"x ]]; then
RUN_HIVE2=1
elif [[ "${element}"x == "hive3"x ]]; then
RUN_HIVE3=1
elif [[ "${element}"x == "kafka"x ]]; then
Expand Down Expand Up @@ -301,7 +301,7 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
fi
fi

if [[ "${RUN_HIVE}" -eq 1 ]]; then
if [[ "${RUN_HIVE2}" -eq 1 ]]; then
# hive2
# If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine
#default value
Expand Down
16 changes: 12 additions & 4 deletions regression-test/conf/regression-conf.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,18 @@ db2_11_port=50000
// See `docker/thirdparties/start-thirdparties-docker.sh`
enableHiveTest=false
enablePaimonTest=false
hms_port=9083
hdfs_port=8020
hiveServerPort=10000
hive_pg_port=5432

// port of hive2 docker
hive2HmsPort=9083
hive2HdfsPort=8020
hive2ServerPort=10000
hive2PgPort=5432

// port of hive3 docker
hive3HmsPort=9383
hive3HdfsPort=8320
hive3ServerPort=13000
hive3PgPort=5732

// kafka test config
// to enable kafka test, you need firstly to start kafka container
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,48 @@
2 9999-12-31 9999-12-31 9999-12-31 23:59:59.0 9999-12-31 23:59:59.0 2023-04-20 00:00:00.12 2023-04-20 00:00:00.3344 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
3 2023-04-21 2023-04-21 2023-04-20 12:34:56.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.123 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678

-- !select_base1 --
1 doris1 18
2 doris2 40
3 \N \N
4 doris4 -2147483648
5 doris5 2147483647
6 \N -2147483648
7 \N 0
8 nereids \N

-- !select_tvf1 --
1 doris1 18
2 doris2 40
3 \N \N
4 doris4 -2147483648
5 doris5 2147483647
6 \N -2147483648
7 \N 0
8 nereids \N

-- !hive_docker_01 --
1 doris1 18
2 doris2 40
3 \N \N
4 doris4 -2147483648
5 doris5 2147483647
6 \N -2147483648
7 \N 0
8 nereids \N

-- !select_base2 --
1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678

-- !select_tvf2 --
1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678

-- !hive_docker_02 --
1 2023-04-20 2023-04-20 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000
2 9999-12-31 9999-12-31 9999-12-31 23:59:59.0 9999-12-31 23:59:59.0 2023-04-20 00:00:00.12 2023-04-20 00:00:00.3344 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999
3 2023-04-21 2023-04-21 2023-04-20 12:34:56.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.123 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678

Loading

0 comments on commit e7aec24

Please sign in to comment.