File tree Expand file tree Collapse file tree 2 files changed +19
-4
lines changed
spark/src/test/scala/org/apache/comet/exec Expand file tree Collapse file tree 2 files changed +19
-4
lines changed Original file line number Diff line number Diff line change @@ -245,10 +245,8 @@ impl Accumulator for VarianceAccumulator {
245245
246246 Ok ( ScalarValue :: Float64 ( match self . count {
247247 count if count == 0.0 => None ,
248- count if count == 1.0 => {
249- if let StatsType :: Population = self . stats_type {
250- Some ( 0.0 )
251- } else if self . null_on_divide_by_zero {
248+ count if count == 1.0 && StatsType :: Sample == self . stats_type => {
249+ if self . null_on_divide_by_zero {
252250 None
253251 } else {
254252 Some ( f64:: NAN )
Original file line number Diff line number Diff line change @@ -38,6 +38,23 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark34Plus
3838class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
3939 import testImplicits ._
4040
41+ test(" stddev_pop should return NaN for some cases" ) {
42+ withSQLConf(
43+ CometConf .COMET_EXEC_SHUFFLE_ENABLED .key -> " true" ,
44+ CometConf .COMET_EXPR_STDDEV_ENABLED .key -> " true" ) {
45+ Seq (true , false ).foreach { nullOnDivideByZero =>
46+ withSQLConf(" spark.sql.legacy.statisticalAggregate" -> nullOnDivideByZero.toString) {
47+
48+ val data : Seq [(Float , Int )] = Seq ((Float .PositiveInfinity , 1 ))
49+ withParquetTable(data, " tbl" , false ) {
50+ val df = sql(" SELECT stddev_pop(_1), stddev_pop(_2) FROM tbl" )
51+ checkSparkAnswer(df)
52+ }
53+ }
54+ }
55+ }
56+ }
57+
4158 test(" count with aggregation filter" ) {
4259 withSQLConf(
4360 CometConf .COMET_ENABLED .key -> " true" ,
You can’t perform that action at this time.
0 commit comments