Skip to content

[SPARK-28390][SQL][PYTHON][TESTS] Convert and port 'pgSQL/select_having.sql' into UDF test base #25161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shivusondur wants to merge 8 commits into from
Closed

[SPARK-28390][SQL][PYTHON][TESTS] Convert and port 'pgSQL/select_having.sql' into UDF test base #25161

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
80f2894
for SPARK-28390
shivusondur Jul 15, 2019
624fad5
Handled th review comments
shivusondur Jul 15, 2019
b0a4fd1
Handled the few more comments
shivusondur Jul 19, 2019
fc65bcf
Handled the few more comments
shivusondur Jul 21, 2019
a1ba9a8
Just added the TODO comment
shivusondur Jul 22, 2019
ef39cdd
Removed the extra queries(made like original) and added udf() whereve…
shivusondur Jul 22, 2019
c4ad657
Added different combination of udf
shivusondur Jul 22, 2019
8795d66
Only "TODO" comment is moved up
shivusondur Jul 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/udf/pgSQL/udf-select_having.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
--
-- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
--
--
-- SELECT_HAVING
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_having.sql
--
-- This test file was converted from inputs/pgSQL/select_having.sql
Copy link
Member

@HyukjinKwon HyukjinKwon Jul 22, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add this todo on the top:

-- TODO: We should add UDFs in GROUP BY clause when [SPARK-28445] is resolved.

and remove same lines below?

Copy link
Contributor Author

@shivusondur shivusondur Jul 23, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done


-- load test data
CREATE TABLE test_having (a int, b int, c string, d string) USING parquet;
INSERT INTO test_having VALUES (0, 1, 'XXXX', 'A');
INSERT INTO test_having VALUES (1, 2, 'AAAA', 'b');
INSERT INTO test_having VALUES (2, 2, 'AAAA', 'c');
INSERT INTO test_having VALUES (3, 3, 'BBBB', 'D');
INSERT INTO test_having VALUES (4, 3, 'BBBB', 'e');
INSERT INTO test_having VALUES (5, 3, 'bbbb', 'F');
INSERT INTO test_having VALUES (6, 4, 'cccc', 'g');
INSERT INTO test_having VALUES (7, 4, 'cccc', 'h');
INSERT INTO test_having VALUES (8, 4, 'CCCC', 'I');
INSERT INTO test_having VALUES (9, 4, 'CCCC', 'j');

SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY b, c;
SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY udf(b), c;
SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY udf(b), udf(c);

-- HAVING is effectively equivalent to WHERE in this case
SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(b) = 3 ORDER BY b, c;
SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(b) = 3 ORDER BY udf(b), c;
SELECT udf(b), udf(c) FROM test_having
GROUP BY b, c HAVING udf(b) = 3 ORDER BY udf(b), udf(c);

-- [SPARK-28386] Cannot resolve ORDER BY columns with GROUP BY and HAVING
-- SELECT lower(c), count(c) FROM test_having
-- GROUP BY lower(c) HAVING count(*) > 2 OR min(a) = max(a)
-- ORDER BY lower(c);

SELECT udf(c), max(udf(a)) FROM test_having
GROUP BY c HAVING udf(count(*)) > 2 OR udf(min(a)) = udf(max(a))
ORDER BY c;

-- test degenerate cases involving HAVING without GROUP BY
-- Per SQL spec, these should generate 0 or 1 row, even without aggregates

SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(a)) = udf(max(a));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(a))) = udf(max(a));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(a))) = udf(udf(max(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(a)) = udf(udf(max(a)));

SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(a)) < udf(max(a));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(a))) < udf(max(a));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(a))) < udf(udf(max(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(a)) < udf(udf(max(a)));

SELECT udf(min(udf(a))), udf(max(udf(a))) FROM test_having HAVING udf(min(udf(a))) = udf(max(udf(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(udf(a)))) = udf(max(udf(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(udf(a)))) = udf(udf(max(udf(a))));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(udf(a))) = udf(udf(max(udf(a))));

SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(udf(a))) < udf(max(udf(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(udf(a)))) < udf(max(udf(a)));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(udf(min(udf(a)))) < udf(udf(max(udf(a))));
SELECT udf(min(a)), udf(max(a)) FROM test_having HAVING udf(min(udf(a))) < udf(udf(max(udf(a))));


-- errors: ungrouped column references
SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a));
SELECT 1 AS one FROM test_having HAVING udf(a) > 1;

-- the really degenerate case: need not scan table at all
SELECT 1 AS one FROM test_having HAVING udf(1 > 2);
SELECT 1 AS one FROM test_having HAVING udf(udf(1) > udf(2));
SELECT 1 AS one FROM test_having HAVING udf(1 < 2);
SELECT 1 AS one FROM test_having HAVING udf(udf(1) < udf(2));

-- and just to prove that we aren't scanning the table:
SELECT 1 AS one FROM test_having WHERE 1/udf(a) = 1 HAVING 1 < 2;

DROP TABLE test_having;
Loading