Skip to content

Automatic columns selector for joins function #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pcbldi wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Automatic columns selector for joins function #168

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cb0bfad
Use automatic column selector in left-join
prakash-defnx Aug 21, 2024
a1ca0ee
Add automatic selector functionality for all join functions
prakash-defnx Aug 23, 2024
59604c4
Update api ns with new function signature
pcbldi Aug 31, 2024
81f627b
Add another test
pcbldi Sep 2, 2024
bb1c463
remove :all from column-selector
pcbldi Sep 9, 2024
d42c600
Fix typo
pcbldi Sep 16, 2024
6702403
Add tests for join functions
pcbldi Sep 16, 2024
eeb0691
Add docstring
pcbldi Sep 16, 2024
47d8d9d
Add more doc-strings
pcbldi Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/tablecloth/api.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,8 @@


(defn anti-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/anti-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/anti-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -315,6 +317,8 @@


(defn asof-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/asof-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/asof-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -1180,6 +1184,8 @@ column-names function returns names according to columns-selector

(defn full-join
"Join keeping all rows"
([ds-left ds-right]
(tablecloth.api.join-concat-ds/full-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/full-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -1337,6 +1343,8 @@ column-names function returns names according to columns-selector


(defn inner-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/inner-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/inner-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -1393,6 +1401,8 @@ column-names function returns names according to columns-selector


(defn left-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/left-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/left-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -2078,6 +2088,8 @@ column-names function returns names according to columns-selector


(defn right-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/right-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/right-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down Expand Up @@ -2187,6 +2199,8 @@ column-names function returns names according to columns-selector


(defn semi-join
([ds-left ds-right]
(tablecloth.api.join-concat-ds/semi-join ds-left ds-right))
([ds-left ds-right columns-selector]
(tablecloth.api.join-concat-ds/semi-join ds-left ds-right columns-selector))
([ds-left ds-right columns-selector options]
Expand Down
19 changes: 19 additions & 0 deletions src/tablecloth/api/join_concat_ds.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,22 +53,35 @@
(impl [(first cols-left) (first cols-right)] ds-left ds-right (or options {}))
(multi-join impl ds-left ds-right cols-left cols-right options))))

(defn- automatic-columns-selector [ds-left ds-right]
(let [cols-l (set (column-names ds-left :all))
Copy link
Member

@genmeblog genmeblog Sep 3, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:all can be skipped.

Copy link

@prakash-defnx prakash-defnx Sep 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done,thanks.

cols-r (set (column-names ds-right :all))]
Copy link
Member

@genmeblog genmeblog Sep 3, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:all can be skipped.

Copy link

@prakash-defnx prakash-defnx Sep 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done, thanks

(vec (s/intersection cols-l cols-r))) )

(defn left-join
([ds-left ds-right]
(left-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (left-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(apply-join j/left-join ds-left ds-right columns-selector options)))

(defn right-join
([ds-left ds-right]
(right-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (right-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(apply-join j/right-join ds-left ds-right columns-selector options)))

(defn inner-join
([ds-left ds-right]
(inner-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (inner-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(apply-join j/inner-join ds-left ds-right columns-selector options)))

(defn asof-join
([ds-left ds-right]
(asof-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (asof-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(apply-join j/left-join-asof ds-left ds-right columns-selector options)))
Expand All @@ -82,6 +95,8 @@

(defn full-join
"Join keeping all rows"
([ds-left ds-right]
(full-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (full-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(apply-join full-join-wrapper ds-left ds-right columns-selector options)))
Expand All @@ -95,12 +110,16 @@
(distinct)))

(defn semi-join
([ds-left ds-right]
(semi-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (semi-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(->> (semi-anti-join-indexes ds-left ds-right columns-selector options)
(select-rows ds-left))))

(defn anti-join
([ds-left ds-right]
(anti-join ds-left ds-right (automatic-columns-selector ds-left ds-right)))
([ds-left ds-right columns-selector] (anti-join ds-left ds-right columns-selector nil))
([ds-left ds-right columns-selector options]
(->> (semi-anti-join-indexes ds-left ds-right columns-selector options)
Expand Down
16 changes: 16 additions & 0 deletions test/tablecloth/api/join_concat_ds_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@
[:i :y])
(api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}])

(fact "int-string join with automatic column selector"
(-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}]))
(-> (api/dataset [{:i "foo" :y 2022 :s "2022"}
{:i "foo" :y 2023 :s "2023"}])))
(api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}]
(-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}])
(api/convert-types {:y :int16}))
(-> (api/dataset [{:i "foo" :y 2022 :s "2022"}
{:i "foo" :y 2023 :s "2023"}])))
(api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}])

(fact "int-string join with automatic column selector - when there are no common columns"
(-> (api/left-join (-> (api/dataset [{:i "foo" :x 2022}]))
(-> (api/dataset [{:y 2022 :z "bar"}])))
(api/rows :as-maps)) => [{:i "foo", :x 2022, :y 2022 :z "bar"}])

(fact "left join on shorts packed into the vector"
(-> (api/left-join (-> (api/dataset [{:iy ["foo" (short 2022)]}]))
(-> (api/dataset [{:iy ["foo" (long 2022)] :s "2022"}
Expand Down