From 94fad74694785762c06578bf9b92ab489722dbfb Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 10 Jun 2026 17:50:42 +0800 Subject: [PATCH 1/2] Refactor and rename cast test suites for ANSI mode on and off --- .../AuronAnsiCastSuiteWithAnsiModeOff.scala | 23 ------------------- .../AuronAnsiCastSuiteWithAnsiModeOn.scala | 23 ------------------- .../AuronCastWithAnsiOffSuite.scala} | 4 +--- .../AuronCastWithAnsiOnSuite.scala} | 2 +- .../AuronAnsiCastSuiteWithAnsiModeOff.scala | 23 ------------------- .../AuronAnsiCastSuiteWithAnsiModeOn.scala | 23 ------------------- .../AuronCastWithAnsiOffSuite.scala} | 4 +--- .../AuronCastWithAnsiOnSuite.scala} | 2 +- 8 files changed, 4 insertions(+), 100 deletions(-) delete mode 100644 auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala delete mode 100644 auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala rename auron-spark-tests/{spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala => spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala} (89%) rename auron-spark-tests/{spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala => spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala} (90%) delete mode 100644 auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala delete mode 100644 auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala rename auron-spark-tests/{spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala => spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala} (89%) rename auron-spark-tests/{spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala => spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala} (90%) diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala deleted file mode 100644 index 3197b4f44..000000000 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.SparkExpressionTestsBase - -class AuronAnsiCastSuiteWithAnsiModeOff - extends AnsiCastSuiteWithAnsiModeOff - with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala deleted file mode 100644 index a86e5b714..000000000 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.SparkExpressionTestsBase - -class AuronAnsiCastSuiteWithAnsiModeOn - extends AnsiCastSuiteWithAnsiModeOn - with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala similarity index 89% rename from auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala rename to auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala index fe1fa724d..968114a59 100644 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala @@ -18,6 +18,4 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.SparkExpressionTestsBase -class AuronCastSuiteWithAnsiModeOn - extends CastSuiteWithAnsiModeOn - with SparkExpressionTestsBase {} +class AuronCastWithAnsiOffSuite extends CastWithAnsiOffSuite with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala similarity index 90% rename from auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala rename to auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala index 7b3526204..a582404ef 100644 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala @@ -18,4 +18,4 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.SparkExpressionTestsBase -class AuronCastSuite extends CastSuite with SparkExpressionTestsBase {} +class AuronCastWithAnsiOnSuite extends CastWithAnsiOnSuite with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala deleted file mode 100644 index 3197b4f44..000000000 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOff.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.SparkExpressionTestsBase - -class AuronAnsiCastSuiteWithAnsiModeOff - extends AnsiCastSuiteWithAnsiModeOff - with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala deleted file mode 100644 index a86e5b714..000000000 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronAnsiCastSuiteWithAnsiModeOn.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.SparkExpressionTestsBase - -class AuronAnsiCastSuiteWithAnsiModeOn - extends AnsiCastSuiteWithAnsiModeOn - with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala similarity index 89% rename from auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala rename to auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala index fe1fa724d..968114a59 100644 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuiteWithAnsiModeOn.scala +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOffSuite.scala @@ -18,6 +18,4 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.SparkExpressionTestsBase -class AuronCastSuiteWithAnsiModeOn - extends CastSuiteWithAnsiModeOn - with SparkExpressionTestsBase {} +class AuronCastWithAnsiOffSuite extends CastWithAnsiOffSuite with SparkExpressionTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala similarity index 90% rename from auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala rename to auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala index 7b3526204..a582404ef 100644 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastSuite.scala +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/AuronCastWithAnsiOnSuite.scala @@ -18,4 +18,4 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.SparkExpressionTestsBase -class AuronCastSuite extends CastSuite with SparkExpressionTestsBase {} +class AuronCastWithAnsiOnSuite extends CastWithAnsiOnSuite with SparkExpressionTestsBase {} From 527dd39de1261085978c060ffaf38ee5328d57f0 Mon Sep 17 00:00:00 2001 From: sychen Date: Fri, 12 Jun 2026 12:21:49 +0800 Subject: [PATCH 2/2] test --- .../auron/utils/AuronSparkTestSettings.scala | 30 +++----------- .../auron/utils/AuronSparkTestSettings.scala | 39 +++++-------------- native-engine/auron/src/rt.rs | 20 ++++++---- 3 files changed, 27 insertions(+), 62 deletions(-) diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 913015ea0..a9155abe6 100644 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -29,18 +29,8 @@ class AuronSparkTestSettings extends SparkTestSettings { .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDateFunctionsSuite] - // Native execution wraps Spark parsing/format validation exceptions in SparkException. - .exclude("function to_date") - .exclude("unix_timestamp") - .exclude("to_unix_timestamp") - // Native date_trunc does not support all Spark aliases such as "yy". - .exclude("function date_trunc") - // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. - .exclude("unsupported fmt fields for trunc/date_trunc results null") - // Native date_trunc may produce incorrect results for historical timestamps with - // non-UTC timezones due to timezone handling differences in the DataFusion engine. - .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") - .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") + .disable( + "Native execution can crash in Spark 4 date/partition suites causing cascade failures") enableSuite[AuronMathFunctionsSuite] .disable("Native execution can crash in Spark 4") @@ -98,8 +88,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetInteroperabilitySuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetPartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetProtobufCompatibilitySuite] .exclude("unannotated array of primitive type") .exclude("unannotated array of struct") @@ -136,14 +125,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetV1FilterSuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetV1PartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("read partitioned table - partition key included in Parquet file") - .exclude( - "read partitioned table - with nulls and partition keys are included in Parquet file") - .exclude( - "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") - .exclude( - "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetV1QuerySuite] .exclude("simple select queries") .exclude("appending") @@ -161,9 +143,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetV2FilterSuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetV2PartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude( - "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetV2QuerySuite] .exclude("simple select queries") .exclude("appending") diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 6f1abd4be..060872689 100644 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -29,18 +29,8 @@ class AuronSparkTestSettings extends SparkTestSettings { .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDateFunctionsSuite] - // Native execution wraps Spark parsing/format validation exceptions in SparkException. - .exclude("function to_date") - .exclude("unix_timestamp") - .exclude("to_unix_timestamp") - // Native date_trunc does not support all Spark aliases such as "yy". - .exclude("function date_trunc") - // Native date_trunc throws for unsupported fields instead of returning NULL as Spark does. - .exclude("unsupported fmt fields for trunc/date_trunc results null") - // Native date_trunc may produce incorrect results for historical timestamps with - // non-UTC timezones due to timezone handling differences in the DataFusion engine. - .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") - .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") + .disable( + "Native execution can crash in Spark 4 date/partition suites causing cascade failures") enableSuite[AuronMathFunctionsSuite] .disable("Native execution can crash in Spark 4") @@ -101,8 +91,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetInteroperabilitySuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetPartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("Infer the TIME data type from partition values") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetProtobufCompatibilitySuite] .exclude("unannotated array of primitive type") .exclude("unannotated array of struct") @@ -125,6 +114,11 @@ class AuronSparkTestSettings extends SparkTestSettings { .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") .exclude("SPARK-31159: rebasing dates in write") .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + // Spark 4.1 changed datetimeRebaseModeInWrite default to EXCEPTION, which causes these + // tests to fail when writing ancient dates (before 1582-10-15) in LEGACY mode. + .exclude( + "SPARK-33163, SPARK-37705: write the metadata keys 'org.apache.spark.legacyDateTime' and 'org.apache.spark.timeZone'") + .exclude("SPARK-33160, SPARK-37705: write the metadata key 'org.apache.spark.legacyINT96' and 'org.apache.spark.timeZone'") enableSuite[AuronParquetRebaseDatetimeV1Suite] .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") enableSuite[AuronParquetRebaseDatetimeV2Suite] @@ -140,15 +134,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetV1FilterSuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetV1PartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("Infer the TIME data type from partition values") - .exclude("read partitioned table - partition key included in Parquet file") - .exclude( - "read partitioned table - with nulls and partition keys are included in Parquet file") - .exclude( - "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") - .exclude( - "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetV1QuerySuite] .exclude("simple select queries") .exclude("appending") @@ -167,12 +153,7 @@ class AuronSparkTestSettings extends SparkTestSettings { enableSuite[AuronParquetV2FilterSuite] .disable("Native execution can crash in Spark 4") enableSuite[AuronParquetV2PartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("Infer the TIME data type from partition values") - .exclude("_SUCCESS should not break partitioning discovery") - .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") - .exclude( - "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") + .disable("Native execution can crash in Spark 4 Parquet partition discovery") enableSuite[AuronParquetV2QuerySuite] .exclude("simple select queries") .exclude("appending") diff --git a/native-engine/auron/src/rt.rs b/native-engine/auron/src/rt.rs index b03ccdeb3..932ae3136 100644 --- a/native-engine/auron/src/rt.rs +++ b/native-engine/auron/src/rt.rs @@ -205,11 +205,12 @@ impl NativeExecutionRuntime { }; let native_wrapper_cloned = native_wrapper.clone(); + let is_finalizing_for_handler = is_finalizing.clone(); let join_handle = tokio_runtime.spawn(async move { consume_stream.await.unwrap_or_else(|err| { handle_unwinded_scope(|| { let task_running = is_task_running(); - if !task_running { + if !task_running || is_finalizing_for_handler.load(Ordering::Acquire) { log::warn!("task completed before native execution done"); return Ok(()); } @@ -271,11 +272,13 @@ impl NativeExecutionRuntime { match next_batch() { Ok(ret) => ret, Err(err) => { - let _ = set_error( - &self.native_wrapper, - &format!("poll record batch error: {err}"), - None, - ); + if !self.is_finalizing.load(Ordering::Acquire) { + let _ = set_error( + &self.native_wrapper, + &format!("poll record batch error: {err}"), + None, + ); + } false } } @@ -288,8 +291,9 @@ impl NativeExecutionRuntime { self.update_metrics().unwrap_or_default(); drop(self.plan); - // Set finalizing flag before dropping receiver to allow graceful SendError - // handling + // Set finalizing flag before dropping receiver and native_wrapper to prevent + // concurrent set_error calls from next_batch/tokio workers from accessing a + // freed GlobalRef after finalize completes. self.is_finalizing.store(true, Ordering::Release); drop(self.batch_receiver);