Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[BugFix] Disable date_trunc equivalent replace if binary type is LE (backport #53229) #53275

Merged
merged 4 commits into from
Nov 28, 2024
Merged

[BugFix] Disable date_trunc equivalent replace if binary type is LE (backport #53229) #53275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f69fd2d
[BugFix] Disable date_trunc equivalent replace if binary type is LE (…
LiShuMing Nov 27, 2024
66d0aa3
fix bugs
LiShuMing Nov 28, 2024
02767f2
fix bugs
LiShuMing Nov 28, 2024
e08c9ba
fix test bugs
LiShuMing Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...a/com/starrocks/sql/optimizer/rule/transformation/materialization/PredicateExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ private static RangePredicate rewriteByEquivalent(BinaryPredicateOperator predic
}

TreeRangeSet<ConstantOperator> range = range(predicate.getBinaryType(), op2);
if (DateTruncEquivalent.INSTANCE.isEquivalent(op1, op2)) {
if (DateTruncEquivalent.isSupportedBinaryType(predicate.getBinaryType()) &&
DateTruncEquivalent.INSTANCE.isEquivalent(op1, op2)) {
TreeRangeSet<ConstantOperator> rangeSet = TreeRangeSet.create();
rangeSet.addAll(range);
return new ColumnRangePredicate(op1.getChild(1).cast(), rangeSet);
Expand Down
54 changes: 42 additions & 12 deletions ...cks/sql/optimizer/rule/transformation/materialization/equivalent/DateTruncEquivalent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
// limitations under the License.
package com.starrocks.sql.optimizer.rule.transformation.materialization.equivalent;

import com.google.common.collect.ImmutableSet;
import com.starrocks.analysis.BinaryType;
import com.starrocks.catalog.FunctionSet;
import com.starrocks.catalog.PrimitiveType;
import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
Expand All @@ -22,11 +24,35 @@
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import com.starrocks.sql.optimizer.rewrite.ScalarOperatorFunctions;

import java.util.Set;

public class DateTruncEquivalent extends IPredicateRewriteEquivalent {
public static final DateTruncEquivalent INSTANCE = new DateTruncEquivalent();

public DateTruncEquivalent() {}

/**
* TODO: we can support this later.
* Change date_trunc('month', dt) to col = '2023-12-01' will get a wrong result.
* MV : select date_trunc('day', dt) as dt from t
* Query1 : select date_trunc('month, dt) from t dt = '2023-11-01'
* -- cannot be rewritten, rewrite result will be wrong
* Rewritten: select date_trunc('month, dt) from t where date_trunc('month', dt) = '2023-11-01'
*
* Query2 : select date_trunc('month, dt) from t where dt between '2023-11-01' and '2023-12-01'
* -- cannot be rewritten, dt='2023-12-01' doesn't match with date_trunc('month', dt)= '2023-11-01'
* Rewritten : select date_trunc('month, dt) from t where date_trunc('month', dt) between '2023-11-01' and '2023-12-01'
*/
private static Set<BinaryType> SUPPORTED_BINARY_TYPES = ImmutableSet.of(
BinaryType.GE,
BinaryType.GT,
BinaryType.LT
);

public static boolean isSupportedBinaryType(BinaryType binaryType) {
return SUPPORTED_BINARY_TYPES.contains(binaryType);
}

@Override
public boolean isEquivalent(ScalarOperator op1, ConstantOperator op2) {
if (!(op1 instanceof CallOperator)) {
Expand Down Expand Up @@ -68,20 +94,24 @@ public ScalarOperator rewrite(RewriteEquivalentContext eqContext,
EquivalentShuttleContext shuttleContext,
ColumnRefOperator replace,
ScalarOperator newInput) {
if (!(newInput instanceof BinaryPredicateOperator)) {
return null;
}
ScalarOperator left = newInput.getChild(0);
ScalarOperator right = newInput.getChild(1);
if (newInput instanceof BinaryPredicateOperator) {
ScalarOperator left = newInput.getChild(0);
ScalarOperator right = newInput.getChild(1);

if (!right.isConstantRef() || !left.equals(eqContext.getEquivalent())) {
return null;
}
if (!isEquivalent(eqContext.getInput(), (ConstantOperator) right)) {
if (!right.isConstantRef() || !left.equals(eqContext.getEquivalent())) {
return null;
}
if (!isEquivalent(eqContext.getInput(), (ConstantOperator) right)) {
return null;
}
BinaryPredicateOperator predicate = (BinaryPredicateOperator) newInput.clone();
if (!isSupportedBinaryType(predicate.getBinaryType())) {
return null;
}
predicate.setChild(0, replace);
return predicate;
} else {
return null;
}
BinaryPredicateOperator predicate = (BinaryPredicateOperator) newInput.clone();
predicate.setChild(0, replace);
return predicate;
}
}
181 changes: 173 additions & 8 deletions fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,13 +248,9 @@ public void testDateTruncPartitionColumnExpr() throws Exception {
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"COMMENT \"MATERIALIZED_VIEW\"\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY HASH(`order_num`) BUCKETS 10\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\"" +
")\n" +
"AS\n" +
"SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds\n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
Expand All @@ -269,6 +265,62 @@ public void testDateTruncPartitionColumnExpr() throws Exception {
"group by ds")
.match("test_partition_expr_mv1");
}
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}
starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr2() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL DEFAULT \"-1\" COMMENT \"\",\n" +
" `dt` datetime NOT NULL DEFAULT \"1996-01-01 00:00:00\" COMMENT \"\",\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY RANGE(`dt`)\n" +
"(\n" +
"PARTITION p2023041017 VALUES [(\"2023-04-10 17:00:00\"), (\"2023-04-10 18:00:00\")),\n" +
"PARTITION p2023041021 VALUES [(\"2023-04-10 21:00:00\"), (\"2023-04-10 22:00:00\"))\n" +
")\n" +
"DISTRIBUTED BY HASH(`order_id`)";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds\n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('month', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('month', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
Expand All @@ -277,7 +329,120 @@ public void testDateTruncPartitionColumnExpr() throws Exception {
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.match("test_partition_expr_mv1");
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr3() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL,\n" +
" `dt` datetime NOT NULL,\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY RANGE(`dt`)\n" +
"(\n" +
"PARTITION p2023041017 VALUES [(\"2023-04-10 17:00:00\"), (\"2023-04-10 18:00:00\")),\n" +
"PARTITION p2023041021 VALUES [(\"2023-04-10 21:00:00\"), (\"2023-04-10 22:00:00\"))\n" +
")\n" +
"DISTRIBUTED BY HASH(`order_id`)";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num`, \n" +
"date_trunc('hour', `dt`) AS ds\n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr4() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL,\n" +
" `dt` datetime NOT NULL,\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY date_trunc('hour', `dt`)\n" + // with date_trunc partition expression
"DISTRIBUTED BY HASH(`order_id`)";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num`, \n" +
"date_trunc('hour', `dt`) AS ds\n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
Expand Down
8 changes: 4 additions & 4 deletions ...ocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr1() throws Exception {

{
String query = "select date_trunc('minute', `k1`) AS ds, sum(v1) " +
" FROM base_tbl1 where `k1` = '2020-02-11' group by ds";
" FROM base_tbl1 where date_trunc('minute', `k1`) = '2020-02-11' group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "ds = '2020-02-11 00:00:00'");
}
Expand Down Expand Up @@ -822,7 +822,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {

{
String query = "select date_trunc('minute', `k1`) AS ds, sum(v1) " +
" FROM base_tbl1 where `k1` = '2020-02-11' group by ds";
" FROM base_tbl1 where date_trunc('minute', `k1`) = '2020-02-11' group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "ds = '2020-02-11 00:00:00'");
}
Expand Down Expand Up @@ -857,7 +857,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {
" WHERE date_trunc('minute', `k1`) <= '2020-03-01 00:00:00' " +
" group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "UNION");
PlanTestBase.assertNotContains(plan, "test_mv1", "UNION");
}

{
Expand All @@ -867,7 +867,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {
" date_trunc('minute', `k1`) <= '2020-03-01 00:00:00' " +
" group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "UNION");
PlanTestBase.assertNotContains(plan, "test_mv1", "UNION");
}

dropMv("test", "test_mv1");
Expand Down
Loading