From 834b2b7c6836b98f1d06be5c75a17dd3f6912fec Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 20 Jun 2025 15:50:19 +0800 Subject: [PATCH 1/7] Fix mv_expand inconsistent column order --- .../esql/core/expression/AttributeSet.java | 4 ++ .../src/main/resources/mv_expand.csv-spec | 34 +++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 8 ++- .../rules/physical/ProjectAwayColumns.java | 12 +++++ .../optimizer/PhysicalPlanOptimizerTests.java | 50 +++++++++++++++++++ 5 files changed, 107 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java index d281db4e6bf63..fefaf3098319e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java @@ -261,5 +261,9 @@ public boolean isEmpty() { public AttributeSet build() { return new AttributeSet(mapBuilder.build()); } + + public void clear() { + mapBuilder.keySet().clear(); + } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 20ce3ecc5a396..f357ef5fbebf6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -419,3 +419,37 @@ emp_no:integer | job_positions:keyword 10001 | Accountant 10001 | Senior Python Developer ; + +testMvExpandInconsistentColumnOrder1 +required_capability: fix_mv_expand_inconsistent_column_order +from languages +| eval foo_1 = 1, foo_2 = 1 +| sort language_code +| mv_expand foo_1 +; + +language_code:integer | language_name:keyword | foo_1:integer | foo_2:integer +1 | English | 1 | 1 +2 | French | 1 | 1 +3 | Spanish | 1 | 1 +4 | German | 1 | 1 +; + +testMvExpandInconsistentColumnOrder2 +required_capability: fix_mv_expand_inconsistent_column_order +from message_types,languages_lookup_non_unique_key +| sort type +| eval language_code = 1, `language_name` = false, message = true, foo_3 = 1, foo_2 = null +| eval foo_3 = "1", `foo_3` = -1, foo_1 = 1, `language_code` = null, `foo_2` = "1" +| mv_expand foo_1 +| limit 5 +; + +country:text | country.keyword:keyword | type:keyword | language_name:boolean | message:boolean | foo_3:integer | foo_1:integer | language_code:null | foo_2:keyword +null | null | Development | false | true | -1 | 1 | null | 1 +null | null | Disconnected | false | true | -1 | 1 | null | 1 +null | null | Error | false | true | -1 | 1 | null | 1 +null | null | Production | false | true | -1 | 1 | null | 1 +null | null | Success | false | true | -1 | 1 | null | 1 +; + diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d59ecdaf02e87..c71b9ba64806c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1210,7 +1210,13 @@ public enum Cap { * * https://github.com/elastic/elasticsearch/issues/129322 */ - NO_PLAIN_STRINGS_IN_LITERALS; + NO_PLAIN_STRINGS_IN_LITERALS, + + /** + * Support for the mv_expand target attribute should be retained in its original position. + * see ES|QL: inconsistent column order #129000 + */ + FIX_MV_EXPAND_INCONSISTENT_COLUMN_ORDER; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java index 26cfbf40eb7ff..4ff876f96fcb3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; import org.elasticsearch.xpack.esql.plan.physical.MergeExec; +import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.rule.Rule; @@ -49,6 +50,17 @@ public PhysicalPlan apply(PhysicalPlan plan) { return currentPlanNode; } + // for mv_expand, the target attribute should be retained in its original position + if (currentPlanNode instanceof MvExpandExec mvExpand) { + List updatedAttrs = new ArrayList<>(requiredAttrBuilder.build()); + int idx = updatedAttrs.indexOf(mvExpand.expanded()); + if (idx != -1) { + updatedAttrs.set(idx, (Attribute) mvExpand.target()); + requiredAttrBuilder.clear(); + requiredAttrBuilder.addAll(updatedAttrs); + } + } + // for non-unary execution plans, we apply the rule for each child if (currentPlanNode instanceof MergeExec mergeExec) { keepTraversing.set(FALSE); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 934517f8f2f9b..ade8c07951b6d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -128,6 +128,7 @@ import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec; +import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.ProjectExec; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; @@ -192,6 +193,7 @@ import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsInRelativeOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -3165,6 +3167,54 @@ public void testProjectAwayAllColumnsWhenOnlyTheCountMattersInStats() { assertThat(Expressions.names(esQuery.attrs()), contains("_doc")); } + /** + * LimitExec[1000[INTEGER],336] + * \_MvExpandExec[foo_1{r}#3,foo_1{r}#20] + * \_TopNExec[[Order[emp_no{f}#9,ASC,LAST]],1000[INTEGER],336] + * \_ExchangeExec[[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, gender{f}#11, hire_date{f}#16, job{f}#17, job.raw{f}#18, la + * nguages{f}#12, last_name{f}#13, long_noidx{f}#19, salary{f}#14, foo_1{r}#3, foo_2{r}#5],false] + * \_ProjectExec[[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, gender{f}#11, hire_date{f}#16, job{f}#17, job.raw{f}#18, la + * nguages{f}#12, last_name{f}#13, long_noidx{f}#19, salary{f}#14, foo_1{r}#3, foo_2{r}#5]] + * \_FieldExtractExec[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..]>[],[]< + * \_EvalExec[[1[INTEGER] AS foo_1#3, 1[INTEGER] AS foo_2#5]] + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#35], limit[1000], sort[[FieldSort[field=emp_no{f}#9, direction=ASC, nulls=LAST]]] estimatedRowSize[352] + */ + public void testProjectAwayMvExpandColumnOrder() { + var plan = optimizedPlan(physicalPlan(""" + from test + | eval foo_1 = 1, foo_2 = 1 + | sort emp_no + | mv_expand foo_1 + """)); + var limit = as(plan, LimitExec.class); + var mvExpand = as(limit.child(), MvExpandExec.class); + var topN = as(mvExpand.child(), TopNExec.class); + var exchange = as(topN.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + + assertThat( + Expressions.names(project.projections()), + containsInRelativeOrder( + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "last_name", + "long_noidx", + "salary", + "foo_1", + "foo_2" + ) + ); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var eval = as(fieldExtract.child(), EvalExec.class); + EsQueryExec esQuery = as(eval.child(), EsQueryExec.class); + } + /** * ProjectExec[[a{r}#5]] * \_EvalExec[[__a_SUM@81823521{r}#15 / __a_COUNT@31645621{r}#16 AS a]] From 9cd191e30d4fd69e36ccd79f2554ef71b17b1ab8 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 20 Jun 2025 15:56:33 +0800 Subject: [PATCH 2/7] Update docs/changelog/129745.yaml --- docs/changelog/129745.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/129745.yaml diff --git a/docs/changelog/129745.yaml b/docs/changelog/129745.yaml new file mode 100644 index 0000000000000..35cfd0671bd64 --- /dev/null +++ b/docs/changelog/129745.yaml @@ -0,0 +1,6 @@ +pr: 129745 +summary: "ESQL: Fix `mv_expand` inconsistent column order" +area: ES|QL +type: bug +issues: + - 129000 From dad35c72d1210a988735985b52acea93715fcb15 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Wed, 16 Jul 2025 14:15:14 +0800 Subject: [PATCH 3/7] Respect the order of the attributes --- .../src/main/resources/mv_expand.csv-spec | 29 ++++++++-- .../rules/physical/ProjectAwayColumns.java | 22 +++----- .../optimizer/PhysicalPlanOptimizerTests.java | 55 ++++++++++--------- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index f357ef5fbebf6..3439c8d094407 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -423,20 +423,39 @@ emp_no:integer | job_positions:keyword testMvExpandInconsistentColumnOrder1 required_capability: fix_mv_expand_inconsistent_column_order from languages -| eval foo_1 = 1, foo_2 = 1 +| eval foo_1 = 1, foo_2 = 2 | sort language_code | mv_expand foo_1 ; language_code:integer | language_name:keyword | foo_1:integer | foo_2:integer -1 | English | 1 | 1 -2 | French | 1 | 1 -3 | Spanish | 1 | 1 -4 | German | 1 | 1 +1 | English | 1 | 2 +2 | French | 1 | 2 +3 | Spanish | 1 | 2 +4 | German | 1 | 2 ; testMvExpandInconsistentColumnOrder2 required_capability: fix_mv_expand_inconsistent_column_order +from languages +| eval foo_1 = [1, 3], foo_2 = 2 +| sort language_code +| mv_expand foo_1 +; + +language_code:integer | language_name:keyword | foo_1:integer | foo_2:integer +1 | English | 1 | 2 +1 | English | 3 | 2 +2 | French | 1 | 2 +2 | French | 3 | 2 +3 | Spanish | 1 | 2 +3 | Spanish | 3 | 2 +4 | German | 1 | 2 +4 | German | 3 | 2 +; + +testMvExpandInconsistentColumnOrder3 +required_capability: fix_mv_expand_inconsistent_column_order from message_types,languages_lookup_non_unique_key | sort type | eval language_code = 1, `language_name` = false, message = true, foo_3 = 1, foo_2 = null diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java index 4ff876f96fcb3..a72de9e4a0e6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; import org.elasticsearch.xpack.esql.plan.physical.MergeExec; -import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.rule.Rule; @@ -50,17 +49,6 @@ public PhysicalPlan apply(PhysicalPlan plan) { return currentPlanNode; } - // for mv_expand, the target attribute should be retained in its original position - if (currentPlanNode instanceof MvExpandExec mvExpand) { - List updatedAttrs = new ArrayList<>(requiredAttrBuilder.build()); - int idx = updatedAttrs.indexOf(mvExpand.expanded()); - if (idx != -1) { - updatedAttrs.set(idx, (Attribute) mvExpand.target()); - requiredAttrBuilder.clear(); - requiredAttrBuilder.addAll(updatedAttrs); - } - } - // for non-unary execution plans, we apply the rule for each child if (currentPlanNode instanceof MergeExec mergeExec) { keepTraversing.set(FALSE); @@ -88,7 +76,15 @@ public PhysicalPlan apply(PhysicalPlan plan) { // no need for projection when dealing with aggs if (logicalFragment instanceof Aggregate == false) { - List output = new ArrayList<>(requiredAttrBuilder.build()); + // we should respect the order of the attributes + List output = new ArrayList<>(); + for (Attribute attribute : logicalFragment.outputSet()) { + if (requiredAttrBuilder.contains(attribute)) { + output.add(attribute); + requiredAttrBuilder.remove(attribute); + } + } + output.addAll(requiredAttrBuilder.build()); // if all the fields are filtered out, it's only the count that matters // however until a proper fix (see https://github.com/elastic/elasticsearch/issues/98703) // add a synthetic field (so it doesn't clash with the user defined one) to return a constant diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index f5ee8c2d9c3fd..7c9b4b3cd2e5c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -660,7 +660,7 @@ public void testExtractorForField() { var exchange = asRemoteExchange(topN.child()); var project = as(exchange.child(), ProjectExec.class); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("salary", "emp_no", "last_name")); + assertThat(names(extract.attributesToExtract()), contains("emp_no", "last_name", "salary")); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); assertThat(source.sorts(), is(fieldSorts(topN.order()))); @@ -2496,7 +2496,7 @@ public void testPushDownEvalSwapFilter() { var extract = as(project.child(), FieldExtractExec.class); assertThat( extract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), - contains("last_name", "first_name") + contains("first_name", "last_name") ); // Now verify the correct Lucene push-down of both the filter and the sort @@ -3188,7 +3188,8 @@ public void testProjectAwayAllColumnsWhenOnlyTheCountMattersInStats() { * nguages{f}#12, last_name{f}#13, long_noidx{f}#19, salary{f}#14, foo_1{r}#3, foo_2{r}#5]] * \_FieldExtractExec[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..]>[],[]< * \_EvalExec[[1[INTEGER] AS foo_1#3, 1[INTEGER] AS foo_2#5]] - * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#35], limit[1000], sort[[FieldSort[field=emp_no{f}#9, direction=ASC, nulls=LAST]]] estimatedRowSize[352] + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#35], limit[1000], sort[[FieldSort[field=emp_no{f}#9, + * direction=ASC, nulls=LAST]]] estimatedRowSize[352] */ public void testProjectAwayMvExpandColumnOrder() { var plan = optimizedPlan(physicalPlan(""" @@ -5748,9 +5749,9 @@ public void testPushTopNKeywordToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "location", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "location", "name")); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); assertThat(source.sorts(), is(fieldSorts(topN.order()))); @@ -5790,9 +5791,9 @@ public void testPushTopNAliasedKeywordToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "location", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "location", "name")); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); assertThat(source.sorts(), is(fieldSorts(topN.order()))); @@ -5835,9 +5836,9 @@ public void testPushTopNDistanceToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), is("distance")); @@ -5897,15 +5898,15 @@ public void testPushTopNInlineDistanceToSource() { names(project.projections()), contains( equalTo("abbrev"), - equalTo("name"), - equalTo("location"), - equalTo("country"), equalTo("city"), + equalTo("country"), + equalTo("location"), + equalTo("name"), startsWith("$$order_by$0$") ) ); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), startsWith("$$order_by$0$")); @@ -5972,9 +5973,9 @@ public void testPushTopNDistanceWithFilterToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), is("distance")); @@ -6069,9 +6070,9 @@ public void testPushTopNDistanceWithCompoundFilterToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), is("distance")); @@ -6158,10 +6159,10 @@ public void testPushTopNDistanceAndPushableFieldWithCompoundFilterToSource() { project = as(exchange.child(), ProjectExec.class); assertThat( names(project.projections()), - contains("abbrev", "name", "location", "country", "city", "scalerank", "scale", "distance", "loc") + contains("abbrev", "city", "country", "location", "name", "scalerank", "distance", "scale", "loc") ); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), is("distance")); @@ -6241,9 +6242,9 @@ public void testPushTopNDistanceAndNonPushableEvalWithCompoundFilterToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "scalerank", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var topNChild = as(extract.child(), TopNExec.class); extract = as(topNChild.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("scalerank")); @@ -6317,9 +6318,9 @@ public void testPushTopNDistanceAndNonPushableEvalsWithCompoundFilterToSource() var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "distance", "scalerank")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var topNChild = as(extract.child(), TopNExec.class); var filter = as(topNChild.child(), FilterExec.class); assertThat(filter.condition(), isA(And.class)); @@ -6394,9 +6395,9 @@ public void testPushTopNDistanceWithCompoundFilterToSourceAndDisjunctiveNonPusha var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "scalerank", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name")); var topNChild = as(extract.child(), TopNExec.class); var filter = as(topNChild.child(), FilterExec.class); assertThat(filter.condition(), isA(Or.class)); @@ -6463,9 +6464,9 @@ public void testPushCompoundTopNDistanceWithCompoundFilterToSource() { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + assertThat(names(project.projections()), contains("abbrev", "city", "country", "location", "name", "scalerank", "distance")); var extract = as(project.child(), FieldExtractExec.class); - assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city", "scalerank")); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "city", "country", "name", "scalerank")); var evalExec = as(extract.child(), EvalExec.class); var alias = as(evalExec.fields().get(0), Alias.class); assertThat(alias.name(), is("distance")); From 0906f117e95861388123bb27dfa3f42c67c99377 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 16 Jul 2025 11:39:57 +0200 Subject: [PATCH 4/7] Use simpler indices in csv tests Indices used in enrich policies, as lookup indices, or multiple indices in the FROM pattern are all treated differently by different test suites. --- .../src/main/resources/mv_expand.csv-spec | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 3439c8d094407..c7dbe01ef6f09 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -422,53 +422,62 @@ emp_no:integer | job_positions:keyword testMvExpandInconsistentColumnOrder1 required_capability: fix_mv_expand_inconsistent_column_order -from languages +from message_types | eval foo_1 = 1, foo_2 = 2 -| sort language_code +| sort message | mv_expand foo_1 ; -language_code:integer | language_name:keyword | foo_1:integer | foo_2:integer -1 | English | 1 | 2 -2 | French | 1 | 2 -3 | Spanish | 1 | 2 -4 | German | 1 | 2 +message:keyword | type:keyword | foo_1:integer | foo_2:integer +Connected to 10.1.0.1 | Success | 1 | 2 +Connected to 10.1.0.2 | Success | 1 | 2 +Connected to 10.1.0.3 | Success | 1 | 2 +Connection error | Error | 1 | 2 +Development environment | Development | 1 | 2 +Disconnected | Disconnected | 1 | 2 +Production environment | Production | 1 | 2 ; testMvExpandInconsistentColumnOrder2 required_capability: fix_mv_expand_inconsistent_column_order -from languages -| eval foo_1 = [1, 3], foo_2 = 2 -| sort language_code +from message_types +| eval foo_1 = [1, 3], foo_2 = 2 +| sort message | mv_expand foo_1 ; -language_code:integer | language_name:keyword | foo_1:integer | foo_2:integer -1 | English | 1 | 2 -1 | English | 3 | 2 -2 | French | 1 | 2 -2 | French | 3 | 2 -3 | Spanish | 1 | 2 -3 | Spanish | 3 | 2 -4 | German | 1 | 2 -4 | German | 3 | 2 +message:keyword | type:keyword | foo_1:integer | foo_2:integer +Connected to 10.1.0.1 | Success | 1 | 2 +Connected to 10.1.0.1 | Success | 3 | 2 +Connected to 10.1.0.2 | Success | 1 | 2 +Connected to 10.1.0.2 | Success | 3 | 2 +Connected to 10.1.0.3 | Success | 1 | 2 +Connected to 10.1.0.3 | Success | 3 | 2 +Connection error | Error | 1 | 2 +Connection error | Error | 3 | 2 +Development environment | Development | 1 | 2 +Development environment | Development | 3 | 2 +Disconnected | Disconnected | 1 | 2 +Disconnected | Disconnected | 3 | 2 +Production environment | Production | 1 | 2 +Production environment | Production | 3 | 2 ; testMvExpandInconsistentColumnOrder3 required_capability: fix_mv_expand_inconsistent_column_order -from message_types,languages_lookup_non_unique_key +from message_types | sort type -| eval language_code = 1, `language_name` = false, message = true, foo_3 = 1, foo_2 = null -| eval foo_3 = "1", `foo_3` = -1, foo_1 = 1, `language_code` = null, `foo_2` = "1" -| mv_expand foo_1 +| eval language_code = 1, `language_name` = false, message = true, foo_3 = 1, foo_2 = null +| eval foo_3 = "1", `foo_3` = -1, foo_1 = 1, `language_code` = null, `foo_2` = "1" +| mv_expand foo_1 | limit 5 ; -country:text | country.keyword:keyword | type:keyword | language_name:boolean | message:boolean | foo_3:integer | foo_1:integer | language_code:null | foo_2:keyword -null | null | Development | false | true | -1 | 1 | null | 1 -null | null | Disconnected | false | true | -1 | 1 | null | 1 -null | null | Error | false | true | -1 | 1 | null | 1 -null | null | Production | false | true | -1 | 1 | null | 1 -null | null | Success | false | true | -1 | 1 | null | 1 +type:keyword | language_name:boolean | message:boolean | foo_3:integer | foo_1:integer | language_code:null | foo_2:keyword +Development | false | true | -1 | 1 | null | 1 +Disconnected | false | true | -1 | 1 | null | 1 +Error | false | true | -1 | 1 | null | 1 +Production | false | true | -1 | 1 | null | 1 +Success | false | true | -1 | 1 | null | 1 ; From a49bf29a97f285fea3766e7f8df749ca9297b7be Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Wed, 16 Jul 2025 18:43:16 +0800 Subject: [PATCH 5/7] Update --- .../rules/physical/ProjectAwayColumns.java | 6 +- .../optimizer/PhysicalPlanOptimizerTests.java | 415 ++++++++---------- 2 files changed, 194 insertions(+), 227 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java index a72de9e4a0e6f..118534c408a6b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java @@ -78,13 +78,13 @@ public PhysicalPlan apply(PhysicalPlan plan) { if (logicalFragment instanceof Aggregate == false) { // we should respect the order of the attributes List output = new ArrayList<>(); - for (Attribute attribute : logicalFragment.outputSet()) { + for (Attribute attribute : logicalFragment.output()) { if (requiredAttrBuilder.contains(attribute)) { output.add(attribute); - requiredAttrBuilder.remove(attribute); } } - output.addAll(requiredAttrBuilder.build()); + assert output.size() == requiredAttrBuilder.build().size() + : "output should be the same size with requiredAttrBuilder"; // if all the fields are filtered out, it's only the count that matters // however until a proper fix (see https://github.com/elastic/elasticsearch/issues/98703) // add a synthetic field (so it doesn't clash with the user defined one) to return a constant diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 7c9b4b3cd2e5c..a609a1e494e54 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -627,16 +627,16 @@ public void testTripleExtractorPerField() { } /** - * Expected - * LimitExec[10000[INTEGER]] - * \_AggregateExec[[],[AVG(salary{f}#14) AS x],FINAL] - * \_AggregateExec[[],[AVG(salary{f}#14) AS x],PARTIAL] - * \_FilterExec[ROUND(emp_no{f}#9) > 10[INTEGER]] - * \_TopNExec[[Order[last_name{f}#13,ASC,LAST]],10[INTEGER]] - * \_ExchangeExec[] - * \_ProjectExec[[salary{f}#14, first_name{f}#10, emp_no{f}#9, last_name{f}#13]] -- project away _doc - * \_FieldExtractExec[salary{f}#14, first_name{f}#10, emp_no{f}#9, last_n..] -- local field extraction - * \_EsQueryExec[test], query[][_doc{f}#16], limit[10], sort[[last_name]] + *LimitExec[10000[INTEGER],8] + * \_AggregateExec[[],[SUM(salary{f}#13460,true[BOOLEAN]) AS x#13454],FINAL,[$$x$sum{r}#13466, $$x$seen{r}#13467],8] + * \_AggregateExec[[],[SUM(salary{f}#13460,true[BOOLEAN]) AS x#13454],INITIAL,[$$x$sum{r}#13466, $$x$seen{r}#13467],8] + * \_FilterExec[ROUND(emp_no{f}#13455) > 10[INTEGER]] + * \_TopNExec[[Order[last_name{f}#13459,ASC,LAST]],10[INTEGER],58] + * \_ExchangeExec[[emp_no{f}#13455, last_name{f}#13459, salary{f}#13460],false] + * \_ProjectExec[[emp_no{f}#13455, last_name{f}#13459, salary{f}#13460]] -- project away _doc + * \_FieldExtractExec[emp_no{f}#13455, last_name{f}#13459, salary{f}#1346..] <[],[]> -- local field extraction + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#13482], limit[10], + * sort[[FieldSort[field=last_name{f}#13459, direction=ASC, nulls=LAST]]] estimatedRowSize[74] */ public void testExtractorForField() { var plan = physicalPlan(""" @@ -2221,7 +2221,7 @@ public void testNoPushDownChangeCase() { * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false] * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu * ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]] - * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[],[]> + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[],[]> * \_EsQueryExec[test], indexMode[standard], query[{"esql_single_value":{"field":"first_name","next":{"regexp":{"first_name": * {"value":"foo*","flags_value":65791,"case_insensitive":true,"max_determinized_states":10000,"boost":0.0}}}, * "source":"TO_LOWER(first_name) RLIKE \"foo*\"@2:9"}}][_doc{f}#25], limit[1000], sort[] estimatedRowSize[332] @@ -2342,10 +2342,10 @@ public void testPushDownUpperCaseChangeLike() { * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9],false] * \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9]] - * \_FieldExtractExec[_meta_field{f}#10, gender{f}#6, hire_date{f}#11, jo..]<[],[]> + * \_FieldExtractExec[_meta_field{f}#10, gender{f}#6, hire_date{f}#11, jo..]<[],[]> * \_LimitExec[1000[INTEGER]] * \_FilterExec[LIKE(first_name{f}#5, "FOO*", true) OR IN(1[INTEGER],2[INTEGER],3[INTEGER],emp_no{f}#4 + 1[INTEGER])] - * \_FieldExtractExec[first_name{f}#5, emp_no{f}#4]<[],[]> + * \_FieldExtractExec[first_name{f}#5, emp_no{f}#4]<[],[]> * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#26], limit[], sort[] estimatedRowSize[332] */ public void testChangeCaseAsInsensitiveWildcardLikeNotPushedDown() { @@ -2460,22 +2460,17 @@ public void testPushDownEvalFilter() { /** * - * ProjectExec[[last_name{f}#21 AS name, first_name{f}#18 AS last_name, last_name{f}#21 AS first_name]] - * \_TopNExec[[Order[last_name{f}#21,ASC,LAST]],10[INTEGER],0] - * \_ExchangeExec[[last_name{f}#21, first_name{f}#18],false] - * \_ProjectExec[[last_name{f}#21, first_name{f}#18]] - * \_FieldExtractExec[last_name{f}#21, first_name{f}#18][] - * \_EsQueryExec[test], indexMode[standard], query[{ - * "bool":{"must":[ - * {"esql_single_value":{ - * "field":"last_name", - * "next":{"range":{"last_name":{"gt":"B","boost":1.0}}}, - * "source":"first_name > \"B\"@3:9" - * }}, - * {"exists":{"field":"first_name","boost":1.0}} - * ],"boost":1.0}}][_doc{f}#40], limit[10], sort[[ - * FieldSort[field=last_name{f}#21, direction=ASC, nulls=LAST] - * ]] estimatedRowSize[116] + * ProjectExec[[last_name{f}#13858 AS name#13841, first_name{f}#13855 AS last_name#13844, last_name{f}#13858 AS first_name#13 + * 847]] + * \_TopNExec[[Order[last_name{f}#13858,ASC,LAST]],10[INTEGER],100] + * \_ExchangeExec[[first_name{f}#13855, last_name{f}#13858],false] + * \_ProjectExec[[first_name{f}#13855, last_name{f}#13858]] + * \_FieldExtractExec[first_name{f}#13855, last_name{f}#13858]<[],[]> + * \_EsQueryExec[test], indexMode[standard], query[ + * {"bool":{"must":[{"esql_single_value":{"field":"last_name","next": + * {"range":{"last_name":{"gt":"B","boost":0.0}}},"source":"first_name > \"B\"@3:9"}}, + * {"exists":{"field":"first_name","boost":0.0}}],"boost":1.0}} + * ][_doc{f}#13879], limit[10], sort[[FieldSort[field=last_name{f}#13858, direction=ASC, nulls=LAST]]] estimatedRowSize[116] * */ public void testPushDownEvalSwapFilter() { @@ -2609,7 +2604,7 @@ public void testDissect() { * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2],false] * \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang * uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2]] - * \_FieldExtractExec[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, ge..]<[],[]> + * \_FieldExtractExec[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, ge..]<[],[]> * \_EsQueryExec[test], indexMode[standard], query[{"wildcard":{"_index":{"wildcard":"test*","boost":0.0}}}][_doc{f}#27], * limit[1000], sort[] estimatedRowSize[382] * @@ -3180,15 +3175,16 @@ public void testProjectAwayAllColumnsWhenOnlyTheCountMattersInStats() { /** * LimitExec[1000[INTEGER],336] - * \_MvExpandExec[foo_1{r}#3,foo_1{r}#20] - * \_TopNExec[[Order[emp_no{f}#9,ASC,LAST]],1000[INTEGER],336] - * \_ExchangeExec[[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, gender{f}#11, hire_date{f}#16, job{f}#17, job.raw{f}#18, la - * nguages{f}#12, last_name{f}#13, long_noidx{f}#19, salary{f}#14, foo_1{r}#3, foo_2{r}#5],false] - * \_ProjectExec[[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, gender{f}#11, hire_date{f}#16, job{f}#17, job.raw{f}#18, la - * nguages{f}#12, last_name{f}#13, long_noidx{f}#19, salary{f}#14, foo_1{r}#3, foo_2{r}#5]] - * \_FieldExtractExec[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..]>[],[]< - * \_EvalExec[[1[INTEGER] AS foo_1#3, 1[INTEGER] AS foo_2#5]] - * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#35], limit[1000], sort[[FieldSort[field=emp_no{f}#9, + * \_MvExpandExec[foo_1{r}#4236,foo_1{r}#4253] + * \_TopNExec[[Order[emp_no{f}#4242,ASC,LAST]],1000[INTEGER],336] + * \_ExchangeExec[[_meta_field{f}#4248, emp_no{f}#4242, first_name{f}#4243, gender{f}#4244, hire_date{f}#4249, job{f}#4250, job. + * raw{f}#4251, languages{f}#4245, last_name{f}#4246, long_noidx{f}#4252, salary{f}#4247, foo_1{r}#4236, foo_2{r}#4238], + * false] + * \_ProjectExec[[_meta_field{f}#4248, emp_no{f}#4242, first_name{f}#4243, gender{f}#4244, hire_date{f}#4249, job{f}#4250, job. + * raw{f}#4251, languages{f}#4245, last_name{f}#4246, long_noidx{f}#4252, salary{f}#4247, foo_1{r}#4236, foo_2{r}#4238]] + * \_FieldExtractExec[_meta_field{f}#4248, emp_no{f}#4242, first_name{f}#..]<[],[]> + * \_EvalExec[[1[INTEGER] AS foo_1#4236, 1[INTEGER] AS foo_2#4238]] + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#4268], limit[1000], sort[[FieldSort[field=emp_no{f}#4242, * direction=ASC, nulls=LAST]]] estimatedRowSize[352] */ public void testProjectAwayMvExpandColumnOrder() { @@ -5725,16 +5721,15 @@ public void testPushTopNWithFilterToSource() { } /** - * ProjectExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327]] - * \_TopNExec[[Order[abbrev{f}#12321,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327],false] - * \_ProjectExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327]] - * \_FieldExtractExec[abbrev{f}#12321, name{f}#12322, location{f}#12325, ..][] + * ProjectExec[[abbrev{f}#4474, name{f}#4475, location{f}#4478, country{f}#4479, city{f}#4480]] + * \_TopNExec[[Order[abbrev{f}#4474,ASC,LAST]],5[INTEGER],221] + * \_ExchangeExec[[abbrev{f}#4474, city{f}#4480, country{f}#4479, location{f}#4478, name{f}#4475],false] + * \_ProjectExec[[abbrev{f}#4474, city{f}#4480, country{f}#4479, location{f}#4478, name{f}#4475]] + * \_FieldExtractExec[abbrev{f}#4474, city{f}#4480, country{f}#4479, loca..]<[],[]> * \_EsQueryExec[airports], - * indexMode[standard], - * query[][_doc{f}#12337], - * limit[5], - * sort[[FieldSort[field=abbrev{f}#12321, direction=ASC, nulls=LAST]]] estimatedRowSize[237] + * indexMode[standard], + * query[][_doc{f}#4490], + * limit[5], sort[[FieldSort[field=abbrev{f}#4474, direction=ASC, nulls=LAST]]] estimatedRowSize[237] */ public void testPushTopNKeywordToSource() { var optimized = optimizedPlan(physicalPlan(""" @@ -5767,13 +5762,13 @@ public void testPushTopNKeywordToSource() { /** * - * ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, abbrev{f}#12 AS code]] - * \_TopNExec[[Order[abbrev{f}#12,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18],false] - * \_ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18]] - * \_FieldExtractExec[abbrev{f}#12, name{f}#13, location{f}#16, country{f..][] - * \_EsQueryExec[airports], indexMode[standard], query[][_doc{f}#29], limit[5], - * sort[[FieldSort[field=abbrev{f}#12, direction=ASC, nulls=LAST]]] estimatedRowSize[237] + * ProjectExec[[abbrev{f}#7828, name{f}#7829, location{f}#7832, country{f}#7833, city{f}#7834, abbrev{f}#7828 AS code#7820]] + * \_TopNExec[[Order[abbrev{f}#7828,ASC,LAST]],5[INTEGER],221] + * \_ExchangeExec[[abbrev{f}#7828, city{f}#7834, country{f}#7833, location{f}#7832, name{f}#7829],false] + * \_ProjectExec[[abbrev{f}#7828, city{f}#7834, country{f}#7833, location{f}#7832, name{f}#7829]] + * \_FieldExtractExec[abbrev{f}#7828, city{f}#7834, country{f}#7833, loca..]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[][_doc{f}#7845], limit[5], + * sort[[FieldSort[field=abbrev{f}#7828, direction=ASC, nulls=LAST]]] estimatedRowSize[237] * */ public void testPushTopNAliasedKeywordToSource() { @@ -5808,19 +5803,19 @@ public void testPushTopNAliasedKeywordToSource() { } /** - * ProjectExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17]] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#11, name{f}#12, country{f}#16, city{f}#17][] - * \_EvalExec[[STDISTANCE(location{f}#15,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) - * AS distance]] - * \_FieldExtractExec[location{f}#15][] + * ProjectExec[[abbrev{f}#7283, name{f}#7284, location{f}#7287, country{f}#7288, city{f}#7289]] + * \_TopNExec[[Order[distance{r}#7276,ASC,LAST]],5[INTEGER],229] + * \_ExchangeExec[[abbrev{f}#7283, city{f}#7289, country{f}#7288, location{f}#7287, name{f}#7284, distance{r}#7276],false] + * \_ProjectExec[[abbrev{f}#7283, city{f}#7289, country{f}#7288, location{f}#7287, name{f}#7284, distance{r}#7276]] + * \_FieldExtractExec[abbrev{f}#7283, city{f}#7289, country{f}#7288, name..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#7287,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distan + * ce#7276]] + * \_FieldExtractExec[location{f}#7287]<[],[]> * \_EsQueryExec[airports], - * indexMode[standard], - * query[][_doc{f}#28], - * limit[5], - * sort[[GeoDistanceSort[field=location{f}#15, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + * indexMode[standard], + * query[][_doc{f}#7300], + * limit[5], + * sort[[GeoDistanceSort[field=location{f}#7287, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] */ public void testPushTopNDistanceToSource() { var optimized = optimizedPlan(physicalPlan(""" @@ -5865,20 +5860,19 @@ public void testPushTopNDistanceToSource() { } /** - * ProjectExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14]] - * \_TopNExec[[Order[$$order_by$0$0{r}#16,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14, $$order_by$0$0{r}#16],false] - * \_ProjectExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14, $$order_by$0$0{r}#16]] - * \_FieldExtractExec[abbrev{f}#8, name{f}#9, country{f}#13, city{f}#14][] - * \_EvalExec[[ - * STDISTANCE(location{f}#12,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS $$order_by$0$0 - * ]] - * \_FieldExtractExec[location{f}#12][] + *ProjectExec[[abbrev{f}#5258, name{f}#5259, location{f}#5262, country{f}#5263, city{f}#5264]] + * \_TopNExec[[Order[$$order_by$0$0{r}#5266,ASC,LAST]],5[INTEGER],229] + * \_ExchangeExec[[abbrev{f}#5258, city{f}#5264, country{f}#5263, location{f}#5262, name{f}#5259, $$order_by$0$0{r}#5266],false] + * \_ProjectExec[[abbrev{f}#5258, city{f}#5264, country{f}#5263, location{f}#5262, name{f}#5259, $$order_by$0$0{r}#5266]] + * \_FieldExtractExec[abbrev{f}#5258, city{f}#5264, country{f}#5263, name..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#5262,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS $$orde + * r_by$0$0#5266]] + * \_FieldExtractExec[location{f}#5262]<[],[]> * \_EsQueryExec[airports], - * indexMode[standard], - * query[][_doc{f}#26], - * limit[5], - * sort[[GeoDistanceSort[field=location{f}#12, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + * indexMode[standard], + * query[][_doc{f}#5276], + * limit[5], + * sort[[GeoDistanceSort[field=location{f}#5262, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] */ public void testPushTopNInlineDistanceToSource() { var optimized = optimizedPlan(physicalPlan(""" @@ -5935,14 +5929,14 @@ public void testPushTopNInlineDistanceToSource() { /** * - * ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18]] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#12, name{f}#13, country{f}#17, city{f}#18][] - * \_EvalExec[[STDISTANCE(location{f}#16,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance - * ]] - * \_FieldExtractExec[location{f}#16][] + * ProjectExec[[abbrev{f}#361, name{f}#362, location{f}#365, country{f}#366, city{f}#367]] + * \_TopNExec[[Order[distance{r}#353,ASC,LAST]],5[INTEGER],229] + * \_ExchangeExec[[abbrev{f}#361, city{f}#367, country{f}#366, location{f}#365, name{f}#362, distance{r}#353],false] + * \_ProjectExec[[abbrev{f}#361, city{f}#367, country{f}#366, location{f}#365, name{f}#362, distance{r}#353]] + * \_FieldExtractExec[abbrev{f}#361, city{f}#367, country{f}#366, name{f}..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#365,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distanc + * e#353]] + * \_FieldExtractExec[location{f}#365]<[],[]> * \_EsQueryExec[airports], indexMode[standard], query[ * { * "geo_shape":{ @@ -5955,7 +5949,7 @@ public void testPushTopNInlineDistanceToSource() { * } * } * } - * }][_doc{f}#29], limit[5], sort[[GeoDistanceSort[field=location{f}#16, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + * ][_doc{f}#378], limit[5], sort[[GeoDistanceSort[field=location{f}#365, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] * */ public void testPushTopNDistanceWithFilterToSource() { @@ -6011,48 +6005,25 @@ public void testPushTopNDistanceWithFilterToSource() { /** * - * ProjectExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20]] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#14, name{f}#15, country{f}#19, city{f}#20][] - * \_EvalExec[[STDISTANCE(location{f}#18,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) - * AS distance]] - * \_FieldExtractExec[location{f}#18][] - * \_EsQueryExec[airports], indexMode[standard], query[{ - * "bool":{ - * "filter":[ - * { - * "esql_single_value":{ - * "field":"scalerank", - * "next":{"range":{"scalerank":{"lt":6,"boost":1.0}}}, - * "source":"scalerank lt 6@3:31" - * } - * }, - * { - * "bool":{ - * "must":[ - * {"geo_shape":{ - * "location":{ - * "relation":"INTERSECTS", - * "shape":{"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]} - * } - * }}, - * {"geo_shape":{ - * "location":{ - * "relation":"DISJOINT", - * "shape":{"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]} - * } - * }} - * ], - * "boost":1.0 - * } - * } - * ], - * "boost":1.0 - * }}][_doc{f}#31], limit[5], sort[[ - * GeoDistanceSort[field=location{f}#18, direction=ASC, lat=55.673, lon=12.565] - * ]] estimatedRowSize[245] + * ProjectExec[[abbrev{f}#6367, name{f}#6368, location{f}#6371, country{f}#6372, city{f}#6373]] + * \_TopNExec[[Order[distance{r}#6357,ASC,LAST]],5[INTEGER],229] + * \_ExchangeExec[[abbrev{f}#6367, city{f}#6373, country{f}#6372, location{f}#6371, name{f}#6368, distance{r}#6357],false] + * \_ProjectExec[[abbrev{f}#6367, city{f}#6373, country{f}#6372, location{f}#6371, name{f}#6368, distance{r}#6357]] + * \_FieldExtractExec[abbrev{f}#6367, city{f}#6373, country{f}#6372, name..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#6371,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distan + * ce#6357]] + * \_FieldExtractExec[location{f}#6371]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[ + * {"bool":{"filter":[{"esql_single_value":{"field":"scalerank","next":{"range": + * {"scalerank":{"lt":6,"boost":0.0}}},"source":"scalerank < 6@3:31"}}, + * {"bool":{"must":[{"geo_shape": + * {"location":{"relation":"INTERSECTS","shape": + * {"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape": + * {"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]}}}}] + * ,"boost":1.0}}],"boost":1.0}} + * ][_doc{f}#6384], limit[5], sort[ + * [GeoDistanceSort[field=location{f}#6371, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] * */ public void testPushTopNDistanceWithCompoundFilterToSource() { @@ -6110,35 +6081,28 @@ public void testPushTopNDistanceWithCompoundFilterToSource() { /** * Tests that multiple sorts, including distance and a field, are pushed down to the source. * - * ProjectExec[[abbrev{f}#25, name{f}#26, location{f}#29, country{f}#30, city{f}#31, scalerank{f}#27, scale{r}#7]] - * \_TopNExec[[ - * Order[distance{r}#4,ASC,LAST], - * Order[scalerank{f}#27,ASC,LAST], - * Order[scale{r}#7,DESC,FIRST], - * Order[loc{r}#10,DESC,FIRST] - * ],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#25, name{f}#26, location{f}#29, country{f}#30, city{f}#31, scalerank{f}#27, scale{r}#7, - * distance{r}#4, loc{r}#10],false] - * \_ProjectExec[[abbrev{f}#25, name{f}#26, location{f}#29, country{f}#30, city{f}#31, scalerank{f}#27, scale{r}#7, - * distance{r}#4, loc{r}#10]] - * \_FieldExtractExec[abbrev{f}#25, name{f}#26, country{f}#30, city{f}#31][] - * \_EvalExec[[ - * STDISTANCE(location{f}#29,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance, - * 10[INTEGER] - scalerank{f}#27 AS scale, TOSTRING(location{f}#29) AS loc - * ]] - * \_FieldExtractExec[location{f}#29, scalerank{f}#27][] - * \_EsQueryExec[airports], indexMode[standard], query[{ - * "bool":{ - * "filter":[ - * {"esql_single_value":{"field":"scalerank","next":{...},"source":"scalerank < 6@3:31"}}, - * {"bool":{ - * "must":[ - * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, - * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} - * ],"boost":1.0}}],"boost":1.0}}][_doc{f}#44], limit[5], sort[[ - * GeoDistanceSort[field=location{f}#29, direction=ASC, lat=55.673, lon=12.565], - * FieldSort[field=scalerank{f}#27, direction=ASC, nulls=LAST] - * ]] estimatedRowSize[303] + * ProjectExec[[abbrev{f}#7429, name{f}#7430, location{f}#7433, country{f}#7434, city{f}#7435, scalerank{f}#7431, scale{r}#74 + * 11]] + * \_TopNExec[[Order[distance{r}#7408,ASC,LAST], Order[scalerank{f}#7431,ASC,LAST], Order[scale{r}#7411,DESC,FIRST], Order[l + * oc{r}#7414,DESC,FIRST]],5[INTEGER],287] + * \_ExchangeExec[[abbrev{f}#7429, city{f}#7435, country{f}#7434, location{f}#7433, name{f}#7430, scalerank{f}#7431, distance{r} + * #7408, scale{r}#7411, loc{r}#7414],false] + * \_ProjectExec[[abbrev{f}#7429, city{f}#7435, country{f}#7434, location{f}#7433, name{f}#7430, scalerank{f}#7431, distance{r} + * #7408, scale{r}#7411, loc{r}#7414]] + * \_FieldExtractExec[abbrev{f}#7429, city{f}#7435, country{f}#7434, name..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#7433,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distan + * ce#7408, 10[INTEGER] - scalerank{f}#7431 AS scale#7411, TOSTRING(location{f}#7433) AS loc#7414]] + * \_FieldExtractExec[location{f}#7433, scalerank{f}#7431]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[ + * {"bool":{"filter":[{"esql_single_value":{"field":"scalerank","next": + * {"range":{"scalerank":{"lt":6,"boost":0.0}}},"source":"scalerank < 6@3:31"}}, + * {"bool":{"must":[{"geo_shape":{"location":{"relation":"INTERSECTS","shape": + * {"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape": + * {"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]}}}}], + * "boost":1.0}}],"boost":1.0}}][_doc{f}#7448], limit[5], sort[ + * [GeoDistanceSort[field=location{f}#7433, direction=ASC, lat=55.673, lon=12.565], + * FieldSort[field=scalerank{f}#7431, direction=ASC, nulls=LAST]]] estimatedRowSize[303] * */ public void testPushTopNDistanceAndPushableFieldWithCompoundFilterToSource() { @@ -6204,26 +6168,30 @@ public void testPushTopNDistanceAndPushableFieldWithCompoundFilterToSource() { /** * This test shows that if the filter contains a predicate on the same field that is sorted, we cannot push down the sort. * - * ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25 AS scale]] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scalerank{f}#25,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#23, name{f}#24, country{f}#28, city{f}#29][] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scalerank{f}#25,ASC,LAST]],5[INTEGER],208] - * \_FieldExtractExec[scalerank{f}#25][] - * \_FilterExec[SUBSTRING(position{r}#7,1[INTEGER],5[INTEGER]) == [50 4f 49 4e 54][KEYWORD]] - * \_EvalExec[[ - * STDISTANCE(location{f}#27,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance, - * TOSTRING(location{f}#27) AS position - * ]] - * \_FieldExtractExec[location{f}#27][] - * \_EsQueryExec[airports], indexMode[standard], query[{ - * "bool":{"filter":[ - * {"esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":6,"boost":1.0}}},"source":...}}, - * {"bool":{"must":[ - * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, - * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} - * ],"boost":1.0}}],"boost":1.0}}][_doc{f}#42], limit[], sort[] estimatedRowSize[87] + * ProjectExec[[abbrev{f}#4856, name{f}#4857, location{f}#4860, country{f}#4861, city{f}#4862, scalerank{f}#4858 AS scale#484 + * 3]] + * \_TopNExec[[Order[distance{r}#4837,ASC,LAST], Order[scalerank{f}#4858,ASC,LAST]],5[INTEGER],233] + * \_ExchangeExec[[abbrev{f}#4856, city{f}#4862, country{f}#4861, location{f}#4860, name{f}#4857, scalerank{f}#4858, distance{r} + * #4837],false] + * \_ProjectExec[[abbrev{f}#4856, city{f}#4862, country{f}#4861, location{f}#4860, name{f}#4857, scalerank{f}#4858, distance{r} + * #4837]] + * \_FieldExtractExec[abbrev{f}#4856, city{f}#4862, country{f}#4861, name..]<[],[]> + * \_TopNExec[[Order[distance{r}#4837,ASC,LAST], Order[scalerank{f}#4858,ASC,LAST]],5[INTEGER],303] + * \_FieldExtractExec[scalerank{f}#4858]<[],[]> + * \_FilterExec[SUBSTRING(position{r}#4840,1[INTEGER],5[INTEGER]) == POINT[KEYWORD]] + * \_EvalExec[[STDISTANCE(location{f}#4860,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS + * distance#4837, TOSTRING(location{f}#4860) AS position#4840]] + * \_FieldExtractExec[location{f}#4860]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[ + * {"bool":{"filter":[ + * {"esql_single_value": + * {"field":"scalerank","next":{"range":{"scalerank":{"lt":6,"boost":0.0}}},"source":"scale < 6@3:93"}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape": + * {"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape": + * {"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]}}}} + * ],"boost":1.0}}],"boost":1.0}}][_doc{f}#4875], limit[], sort[] estimatedRowSize[87] * */ public void testPushTopNDistanceAndNonPushableEvalWithCompoundFilterToSource() { @@ -6279,27 +6247,25 @@ public void testPushTopNDistanceAndNonPushableEvalWithCompoundFilterToSource() { /** * This test shows that if the filter contains a predicate on the same field that is sorted, we cannot push down the sort. * - * ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10]] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scale{r}#10,ASC,LAST]],5[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#23, name{f}#24, country{f}#28, city{f}#29][] - * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scale{r}#10,ASC,LAST]],5[INTEGER],208] - * \_FilterExec[ - * SUBSTRING(position{r}#7,1[INTEGER],5[INTEGER]) == [50 4f 49 4e 54][KEYWORD] - * AND scale{r}#10 > 3[INTEGER] - * ] - * \_EvalExec[[ - * STDISTANCE(location{f}#27,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance, - * TOSTRING(location{f}#27) AS position, - * 10[INTEGER] - scalerank{f}#25 AS scale - * ]] - * \_FieldExtractExec[location{f}#27, scalerank{f}#25][] - * \_EsQueryExec[airports], indexMode[standard], query[{ - * "bool":{"must":[ - * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, - * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} - * ],"boost":1.0}}][_doc{f}#42], limit[], sort[] estimatedRowSize[91] + *ProjectExec[[abbrev{f}#1447, name{f}#1448, location{f}#1451, country{f}#1452, city{f}#1453, scalerank{r}#1434]] + * \_TopNExec[[Order[distance{r}#1428,ASC,LAST], Order[scalerank{r}#1434,ASC,LAST]],5[INTEGER],233] + * \_ExchangeExec[[abbrev{f}#1447, city{f}#1453, country{f}#1452, location{f}#1451, name{f}#1448, distance{r}#1428, scalerank{r} + * #1434],false] + * \_ProjectExec[[abbrev{f}#1447, city{f}#1453, country{f}#1452, location{f}#1451, name{f}#1448, distance{r}#1428, scalerank{r} + * #1434]] + * \_FieldExtractExec[abbrev{f}#1447, city{f}#1453, country{f}#1452, name..]<[],[]> + * \_TopNExec[[Order[distance{r}#1428,ASC,LAST], Order[scalerank{r}#1434,ASC,LAST]],5[INTEGER],303] + * \_FilterExec[SUBSTRING(position{r}#1431,1[INTEGER],5[INTEGER]) == POINT[KEYWORD] AND scalerank{r}#1434 > 3[INTEGER]] + * \_EvalExec[[STDISTANCE(location{f}#1451,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distan + * ce#1428, TOSTRING(location{f}#1451) AS position#1431, 10[INTEGER] - scalerank{f}#1449 AS scalerank#1434]] + * \_FieldExtractExec[location{f}#1451, scalerank{f}#1449]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[ + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape": + * {"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape": + * {"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]}}}} + * ],"boost":1.0}}][_doc{f}#1466], limit[], sort[] estimatedRowSize[91] * */ public void testPushTopNDistanceAndNonPushableEvalsWithCompoundFilterToSource() { @@ -6424,28 +6390,29 @@ public void testPushTopNDistanceWithCompoundFilterToSourceAndDisjunctiveNonPusha /** * - * ProjectExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21]] - * \_TopNExec[[Order[scalerank{f}#17,ASC,LAST], Order[distance{r}#4,ASC,LAST]],15[INTEGER],0] - * \_ExchangeExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21, scalerank{f}#17, distance{r}#4],false] - * \_ProjectExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21, scalerank{f}#17, distance{r}#4]] - * \_FieldExtractExec[abbrev{f}#15, name{f}#16, country{f}#20, city{f}#21, ..][] - * \_EvalExec[[STDISTANCE(location{f}#19,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) - * AS distance]] - * \_FieldExtractExec[location{f}#19][] - * \_EsQueryExec[airports], indexMode[standard], query[{ - * "bool":{ - * "filter":[ - * {"esql_single_value":{"field":"scalerank",...,"source":"scalerank lt 6@3:31"}}, - * {"bool":{"must":[ - * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, - * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} - * ],"boost":1.0}} - * ],"boost":1.0 - * } - * }][_doc{f}#32], limit[], sort[[ - * FieldSort[field=scalerank{f}#17, direction=ASC, nulls=LAST], - * GeoDistanceSort[field=location{f}#19, direction=ASC, lat=55.673, lon=12.565] - * ]] estimatedRowSize[37] + * ProjectExec[[abbrev{f}#6090, name{f}#6091, location{f}#6094, country{f}#6095, city{f}#6096]] + * \_TopNExec[[Order[scalerank{f}#6092,ASC,LAST], Order[distance{r}#6079,ASC,LAST]],15[INTEGER],233] + * \_ExchangeExec[[abbrev{f}#6090, city{f}#6096, country{f}#6095, location{f}#6094, name{f}#6091, scalerank{f}#6092, distance{r} + * #6079],false] + * \_ProjectExec[[abbrev{f}#6090, city{f}#6096, country{f}#6095, location{f}#6094, name{f}#6091, scalerank{f}#6092, distance{r} + * #6079]] + * \_FieldExtractExec[abbrev{f}#6090, city{f}#6096, country{f}#6095, name..]<[],[]> + * \_EvalExec[[STDISTANCE(location{f}#6094,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distan + * ce#6079]] + * \_FieldExtractExec[location{f}#6094]<[],[]> + * \_EsQueryExec[airports], indexMode[standard], query[ + * {"bool":{"filter":[ + * {"esql_single_value":{"field":"scalerank","next":{"range": + * {"scalerank":{"lt":6,"boost":0.0}}},"source":"scalerank < 6@3:31"}}, + * {"bool":{"must":[ + * {"geo_shape": {"location":{"relation":"INTERSECTS","shape": + * {"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape": + * {"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]}}}} + * ],"boost":1.0}}],"boost":1.0}} + * ][_doc{f}#6107], limit[15], sort[ + * [FieldSort[field=scalerank{f}#6092, direction=ASC, nulls=LAST], + * GeoDistanceSort[field=location{f}#6094, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[249] * */ public void testPushCompoundTopNDistanceWithCompoundFilterToSource() { @@ -8104,7 +8071,7 @@ public void testNotEqualsPushdownToDelegate() { * ges{f}#5, last_name{f}#6, long_noidx{f}#12, salary{f}#7],false] * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, hire_date{f}#9, job{f}#10, job.raw{f}#11, langua * ges{f}#5, last_name{f}#6, long_noidx{f}#12, salary{f}#7]] - * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen..]<[],[]> + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen..]<[],[]> * \_EsQueryExec[test], indexMode[standard], * query[{"bool":{"filter":[{"sampling":{"probability":0.1,"seed":234,"hash":0}}],"boost":1.0}}] * [_doc{f}#24], limit[1000], sort[] estimatedRowSize[332] From 9ff4935eb89795da56280381b1221ab5fb85143d Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 16 Jul 2025 17:24:35 +0200 Subject: [PATCH 6/7] Revert to kanoshiou's initial approach Keep all the attributes from requiredAttrBuilder in the projection, even when they're not in the fragment's output due to a bug. --- .../esql/optimizer/rules/physical/ProjectAwayColumns.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java index 118534c408a6b..887fb039a14cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/ProjectAwayColumns.java @@ -81,10 +81,14 @@ public PhysicalPlan apply(PhysicalPlan plan) { for (Attribute attribute : logicalFragment.output()) { if (requiredAttrBuilder.contains(attribute)) { output.add(attribute); + requiredAttrBuilder.remove(attribute); } } - assert output.size() == requiredAttrBuilder.build().size() - : "output should be the same size with requiredAttrBuilder"; + // requiredAttrBuilder should be empty unless the plan is inconsistent due to a bug. + // This can happen in case of remote ENRICH, see https://github.com/elastic/elasticsearch/issues/118531 + // TODO: stop adding the remaining required attributes once remote ENRICH is fixed. + output.addAll(requiredAttrBuilder.build()); + // if all the fields are filtered out, it's only the count that matters // however until a proper fix (see https://github.com/elastic/elasticsearch/issues/98703) // add a synthetic field (so it doesn't clash with the user defined one) to return a constant From 66ed7d83583f3bda5bcf750fc0f6612eb78412a6 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Wed, 16 Jul 2025 23:39:13 +0800 Subject: [PATCH 7/7] Remove 129000 --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 2fefcccb4d14f..41b8658302bd7 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -53,7 +53,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase { "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072 // Awaiting fixes for correctness - "Expecting the following columns \\[.*\\], got", // https://github.com/elastic/elasticsearch/issues/129000 "Expecting at most \\[.*\\] columns, got \\[.*\\]" // https://github.com/elastic/elasticsearch/issues/129561 );