Skip to content

[SPARK-22036][SQL][FOLLOWUP] Fix decimalArithmeticOperations.sql #20498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wangyum wants to merge 4 commits into from
Closed

[SPARK-22036][SQL][FOLLOWUP] Fix decimalArithmeticOperations.sql #20498

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2f532ea
Fix test error.
wangyum Feb 3, 2018
dac2dc8
Copy from test from https://github.com/apache/spark/pull/20350
wangyum Feb 4, 2018
a42523d
Add more precision loss case.
wangyum Feb 4, 2018
aee5a55
Add more precision loss case
wangyum Feb 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...e/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ select 1e35 / 0.1;

-- arithmetic operations causing a precision loss are truncated
select 123456789123456789.1234567890 * 1.123456789123456789;
select 0.001 / 9876543210987654321098765432109876543.2
select 0.001 / 9876543210987654321098765432109876543.2;
Copy link
Member

@dongjoon-hyun dongjoon-hyun Feb 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, @wangyum .
If this is only for a specific test file, can we have more proper title? Fix decimalArithmeticOperations.sql?

Copy link
Member

@gatorsmile gatorsmile Feb 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

... A good catch! We need to review the PR more carefully.

Copy link
Member

@gatorsmile gatorsmile Feb 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not a good test case. The results and schemas are the same no matter whether we set spark.sql.decimalOperations.allowPrecisionLoss to true or false.


-- return NULL instead of rounding, according to old Spark versions' behavior
set spark.sql.decimalOperations.allowPrecisionLoss=false;
Expand All @@ -75,6 +75,6 @@ select 1e35 / 0.1;

-- arithmetic operations causing a precision loss return NULL
select 123456789123456789.1234567890 * 1.123456789123456789;
select 0.001 / 9876543210987654321098765432109876543.2
select 0.001 / 9876543210987654321098765432109876543.2;

drop table decimals_test;
110 changes: 53 additions & 57 deletions .../test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 32
-- Number of queries: 34


-- !query 0
Expand Down Expand Up @@ -155,137 +155,133 @@ struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789

-- !query 18
select 0.001 / 9876543210987654321098765432109876543.2

set spark.sql.decimalOperations.allowPrecisionLoss=false
-- !query 18 schema
struct<>
struct<(CAST(0.001 AS DECIMAL(38,3)) / CAST(9876543210987654321098765432109876543.2 AS DECIMAL(38,3))):decimal(38,37)>
-- !query 18 output
org.apache.spark.sql.catalyst.parser.ParseException

mismatched input 'spark' expecting <EOF>(line 3, pos 4)

== SQL ==
select 0.001 / 9876543210987654321098765432109876543.2

set spark.sql.decimalOperations.allowPrecisionLoss=false
----^^^
NULL


-- !query 19
select id, a+b, a-b, a*b, a/b from decimals_test order by id
set spark.sql.decimalOperations.allowPrecisionLoss=false
-- !query 19 schema
struct<id:int,(a + b):decimal(38,17),(a - b):decimal(38,17),(a * b):decimal(38,6),(a / b):decimal(38,6)>
struct<key:string,value:string>
-- !query 19 output
1 1099 -899 99900 0.1001
2 24690.246 0 152402061.885129 1
3 1234.2234567891011 -1233.9765432108989 152.358023 0.0001
4 123456789123456790.12345678912345679 123456789123456787.87654321087654321 138698367904130467.515623 109890109097814272.043109
spark.sql.decimalOperations.allowPrecisionLoss false


-- !query 20
select id, a*10, b/10 from decimals_test order by id
select id, a+b, a-b, a*b, a/b from decimals_test order by id
-- !query 20 schema
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
-- !query 20 output
1 1000 99.9
2 123451.23 1234.5123
3 1.234567891011 123.41
4 1234567891234567890 0.112345678912345679
1 1099 -899 NULL 0.1001001001001001
2 24690.246 0 NULL 1
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436


-- !query 21
select 10.3 * 3.0
select id, a*10, b/10 from decimals_test order by id
-- !query 21 schema
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
-- !query 21 output
30.9
1 1000 99.9
2 123451.23 1234.5123
3 1.234567891011 123.41
4 1234567891234567890 0.1123456789123456789


-- !query 22
select 10.3000 * 3.0
select 10.3 * 3.0
-- !query 22 schema
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
-- !query 22 output
30.9


-- !query 23
select 10.30000 * 30.0
select 10.3000 * 3.0
-- !query 23 schema
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
-- !query 23 output
309
30.9


-- !query 24
select 10.300000000000000000 * 3.000000000000000000
select 10.30000 * 30.0
-- !query 24 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
-- !query 24 output
30.9
309


-- !query 25
select 10.300000000000000000 * 3.0000000000000000000
select 10.300000000000000000 * 3.000000000000000000
-- !query 25 schema
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
-- !query 25 output
30.9


-- !query 26
select (5e36 + 0.1) + 5e36
select 10.300000000000000000 * 3.0000000000000000000
-- !query 26 schema
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
-- !query 26 output
NULL


-- !query 27
select (-4e36 - 0.1) - 7e36
select (5e36 + 0.1) + 5e36
-- !query 27 schema
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 27 output
NULL


-- !query 28
select 12345678901234567890.0 * 12345678901234567890.0
select (-4e36 - 0.1) - 7e36
-- !query 28 schema
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
-- !query 28 output
NULL


-- !query 29
select 1e35 / 0.1
select 12345678901234567890.0 * 12345678901234567890.0
-- !query 29 schema
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,6)>
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
-- !query 29 output
NULL


-- !query 30
select 123456789123456789.1234567890 * 1.123456789123456789
select 1e35 / 0.1
-- !query 30 schema
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
-- !query 30 output
138698367904130467.654320988515622621
NULL


-- !query 31
select 0.001 / 9876543210987654321098765432109876543.2

drop table decimals_test
select 123456789123456789.1234567890 * 1.123456789123456789
-- !query 31 schema
struct<>
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
-- !query 31 output
org.apache.spark.sql.catalyst.parser.ParseException
NULL

mismatched input 'table' expecting <EOF>(line 3, pos 5)

== SQL ==
-- !query 32
select 0.001 / 9876543210987654321098765432109876543.2
-- !query 32 schema
struct<(CAST(0.001 AS DECIMAL(38,3)) / CAST(9876543210987654321098765432109876543.2 AS DECIMAL(38,3))):decimal(38,37)>
-- !query 32 output
NULL


-- !query 33
drop table decimals_test
-----^^^
-- !query 33 schema
struct<>
-- !query 33 output