Skip to content

[SPARK-24562][TESTS] Support different configs for same test in SQLQueryTestSuite #21568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mgaido91 wants to merge 3 commits into from
Closed

[SPARK-24562][TESTS] Support different configs for same test in SQLQueryTestSuite #21568

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed01ff0
Different config for same test in SQLQueryTestSuite
mgaido91 Jun 13, 2018
94cbe24
Revert "Different config for same test in SQLQueryTestSuite"
mgaido91 Jul 11, 2018
6f90e63
use matrix of configs in the files
mgaido91 Jul 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 0 additions & 29 deletions ...e/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,33 +54,4 @@ select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.1
select 123456789123456789.1234567890 * 1.123456789123456789;
select 12345678912345.123456789123 / 0.000000012345678;

-- return NULL instead of rounding, according to old Spark versions' behavior
set spark.sql.decimalOperations.allowPrecisionLoss=false;

-- test decimal operations
select id, a+b, a-b, a*b, a/b from decimals_test order by id;

-- test operations between decimals and constants
select id, a*10, b/10 from decimals_test order by id;

-- test operations on constants
select 10.3 * 3.0;
select 10.3000 * 3.0;
select 10.30000 * 30.0;
select 10.300000000000000000 * 3.000000000000000000;
select 10.300000000000000000 * 3.0000000000000000000;
select 2.35E10 * 1.0;

-- arithmetic operations causing an overflow return NULL
select (5e36 + 0.1) + 5e36;
select (-4e36 - 0.1) - 7e36;
select 12345678901234567890.0 * 12345678901234567890.0;
select 1e35 / 0.1;
select 1.2345678901234567890E30 * 1.2345678901234567890E25;

-- arithmetic operations causing a precision loss return NULL
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345;
select 123456789123456789.1234567890 * 1.123456789123456789;
select 12345678912345.123456789123 / 0.000000012345678;

drop table decimals_test;
193 changes: 193 additions & 0 deletions .../decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
-- Automatically generated by SQLQueryTestSuite
Copy link
Member

@maropu maropu Jun 16, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This new feature looks good to me. Btw, if we set multiple configurations, filenames get too long? How about adding the configuration info. in the head of files? Then, filenames are;

./decimalArithmeticOperations.sql.out.1
./decimalArithmeticOperations.sql.out.2
...

Copy link
Contributor Author

@mgaido91 mgaido91 Jun 16, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we have two cases:
1 - Different configs produce different results (so different files) and in this case your suggestion is fine;
2 - Different configs produce the same results (so we have one golden file for all of them), how would you address this case?

Copy link
Member

@maropu maropu Jun 18, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking the second case also output multiple same result files for each config.

// these files have the same result
subquery/in-subquery/in-joins.sql.out.1 <- sort-merge joins
subquery/in-subquery/in-joins.sql.out.2 <- hash joins

-- Number of queries: 23


-- !query 0
CREATE TEMPORARY VIEW t AS SELECT 1.0 as a, 0.0 as b
-- !query 0 schema
struct<>
-- !query 0 output



-- !query 1
select a / b from t
-- !query 1 schema
struct<(CAST(a AS DECIMAL(2,1)) / CAST(b AS DECIMAL(2,1))):decimal(8,6)>
-- !query 1 output
NULL


-- !query 2
select a % b from t
-- !query 2 schema
struct<(CAST(a AS DECIMAL(2,1)) % CAST(b AS DECIMAL(2,1))):decimal(1,1)>
-- !query 2 output
NULL


-- !query 3
select pmod(a, b) from t
-- !query 3 schema
struct<pmod(CAST(a AS DECIMAL(2,1)), CAST(b AS DECIMAL(2,1))):decimal(1,1)>
-- !query 3 output
NULL


-- !query 4
create table decimals_test(id int, a decimal(38,18), b decimal(38,18)) using parquet
-- !query 4 schema
struct<>
-- !query 4 output



-- !query 5
insert into decimals_test values(1, 100.0, 999.0), (2, 12345.123, 12345.123),
(3, 0.1234567891011, 1234.1), (4, 123456789123456789.0, 1.123456789123456789)
-- !query 5 schema
struct<>
-- !query 5 output



-- !query 6
select id, a+b, a-b, a*b, a/b from decimals_test order by id
-- !query 6 schema
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
-- !query 6 output
1 1099 -899 NULL 0.1001001001001001
2 24690.246 0 NULL 1
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436


-- !query 7
select id, a*10, b/10 from decimals_test order by id
-- !query 7 schema
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
-- !query 7 output
1 1000 99.9
2 123451.23 1234.5123
3 1.234567891011 123.41
4 1234567891234567890 0.1123456789123456789


-- !query 8
select 10.3 * 3.0
-- !query 8 schema
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
-- !query 8 output
30.9


-- !query 9
select 10.3000 * 3.0
-- !query 9 schema
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
-- !query 9 output
30.9


-- !query 10
select 10.30000 * 30.0
-- !query 10 schema
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
-- !query 10 output
309


-- !query 11
select 10.300000000000000000 * 3.000000000000000000
-- !query 11 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
-- !query 11 output
30.9


-- !query 12
select 10.300000000000000000 * 3.0000000000000000000
-- !query 12 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
-- !query 12 output
NULL


-- !query 13
select 2.35E10 * 1.0
-- !query 13 schema
struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)>
-- !query 13 output
23500000000


-- !query 14
select (5e36 + 0.1) + 5e36
-- !query 14 schema
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 14 output
NULL


-- !query 15
select (-4e36 - 0.1) - 7e36
-- !query 15 schema
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 15 output
NULL


-- !query 16
select 12345678901234567890.0 * 12345678901234567890.0
-- !query 16 schema
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
-- !query 16 output
NULL


-- !query 17
select 1e35 / 0.1
-- !query 17 schema
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
-- !query 17 output
NULL


-- !query 18
select 1.2345678901234567890E30 * 1.2345678901234567890E25
-- !query 18 schema
struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)>
-- !query 18 output
NULL


-- !query 19
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
-- !query 19 schema
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)>
-- !query 19 output
NULL


-- !query 20
select 123456789123456789.1234567890 * 1.123456789123456789
-- !query 20 schema
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
-- !query 20 output
NULL


-- !query 21
select 12345678912345.123456789123 / 0.000000012345678
-- !query 21 schema
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)>
-- !query 21 output
NULL


-- !query 22
drop table decimals_test
-- !query 22 schema
struct<>
-- !query 22 output

148 changes: 3 additions & 145 deletions ...ative/decimalArithmeticOperations.sql.out → ...malOperations.allowPrecisionLoss-true.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 40
-- Number of queries: 23


-- !query 0
Expand Down Expand Up @@ -186,150 +186,8 @@ struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8


-- !query 22
set spark.sql.decimalOperations.allowPrecisionLoss=false
-- !query 22 schema
struct<key:string,value:string>
-- !query 22 output
spark.sql.decimalOperations.allowPrecisionLoss false


-- !query 23
select id, a+b, a-b, a*b, a/b from decimals_test order by id
-- !query 23 schema
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
-- !query 23 output
1 1099 -899 NULL 0.1001001001001001
2 24690.246 0 NULL 1
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436


-- !query 24
select id, a*10, b/10 from decimals_test order by id
-- !query 24 schema
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
-- !query 24 output
1 1000 99.9
2 123451.23 1234.5123
3 1.234567891011 123.41
4 1234567891234567890 0.1123456789123456789


-- !query 25
select 10.3 * 3.0
-- !query 25 schema
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
-- !query 25 output
30.9


-- !query 26
select 10.3000 * 3.0
-- !query 26 schema
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
-- !query 26 output
30.9


-- !query 27
select 10.30000 * 30.0
-- !query 27 schema
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
-- !query 27 output
309


-- !query 28
select 10.300000000000000000 * 3.000000000000000000
-- !query 28 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
-- !query 28 output
30.9


-- !query 29
select 10.300000000000000000 * 3.0000000000000000000
-- !query 29 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
-- !query 29 output
NULL


-- !query 30
select 2.35E10 * 1.0
-- !query 30 schema
struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)>
-- !query 30 output
23500000000


-- !query 31
select (5e36 + 0.1) + 5e36
-- !query 31 schema
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 31 output
NULL


-- !query 32
select (-4e36 - 0.1) - 7e36
-- !query 32 schema
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 32 output
NULL


-- !query 33
select 12345678901234567890.0 * 12345678901234567890.0
-- !query 33 schema
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
-- !query 33 output
NULL


-- !query 34
select 1e35 / 0.1
-- !query 34 schema
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
-- !query 34 output
NULL


-- !query 35
select 1.2345678901234567890E30 * 1.2345678901234567890E25
-- !query 35 schema
struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)>
-- !query 35 output
NULL


-- !query 36
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
-- !query 36 schema
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)>
-- !query 36 output
NULL


-- !query 37
select 123456789123456789.1234567890 * 1.123456789123456789
-- !query 37 schema
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
-- !query 37 output
NULL


-- !query 38
select 12345678912345.123456789123 / 0.000000012345678
-- !query 38 schema
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)>
-- !query 38 output
NULL


-- !query 39
drop table decimals_test
-- !query 39 schema
-- !query 22 schema
struct<>
-- !query 39 output
-- !query 22 output

Loading