Skip to content

Initial dpctl tensor removal #2779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Initial dpctl tensor removal #2779

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c869820
Initial dpctl tensor removal
ethanglaser Nov 6, 2025
a78832f
fixes and formatting
ethanglaser Nov 6, 2025
9f59916
re-add internal dpctl handling
ethanglaser Nov 18, 2025
7878669
minor
ethanglaser Nov 20, 2025
860c6d7
Merge branch 'main' into dev/eglaser-dcptl-rm-pt1
ethanglaser Nov 20, 2025
8dde020
remove unnecessary dpnp.asnumpy calls
ethanglaser Nov 24, 2025
a933b3c
fixes
ethanglaser Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions examples/sklearnex/basic_statistics_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# limitations under the License.
# ==============================================================================

import dpctl.tensor as dpt
import dpnp
import numpy as np
from dpctl import SyclQueue
from mpi4py import MPI
Expand Down Expand Up @@ -51,14 +51,14 @@ def generate_data(par, size, seed=777):
data, weights = generate_data(params_spmd, size, seed=rank)
weighted_data = np.diag(weights) @ data

dpt_data = dpt.asarray(data, usm_type="device", sycl_queue=q)
dpt_weights = dpt.asarray(weights, usm_type="device", sycl_queue=q)
dpnp_data = dpnp.asarray(data, usm_type="device", sycl_queue=q)
dpnp_weights = dpnp.asarray(weights, usm_type="device", sycl_queue=q)

gtr_mean = np.mean(weighted_data, axis=0)
gtr_std = np.std(weighted_data, axis=0)

bss = BasicStatisticsSpmd(["mean", "standard_deviation"])
bss.fit(dpt_data, dpt_weights)
bss.fit(dpnp_data, dpnp_weights)

print(f"Computed mean on rank {rank}:\n", bss.mean_)
print(f"Computed std on rank {rank}:\n", bss.standard_deviation_)
6 changes: 3 additions & 3 deletions examples/sklearnex/covariance_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# ==============================================================================

import dpctl
import dpctl.tensor as dpt
import dpnp
import numpy as np
from mpi4py import MPI

Expand All @@ -35,8 +35,8 @@ def get_data(data_seed):
size = comm.Get_size()

X = get_data(rank)
dpt_X = dpt.asarray(X, usm_type="device", sycl_queue=q)
dpnp_X = dpnp.asarray(X, usm_type="device", sycl_queue=q)

cov = EmpiricalCovariance().fit(dpt_X)
cov = EmpiricalCovariance().fit(dpnp_X)

print(f"Computed covariance values on rank {rank}:\n", cov.covariance_)
6 changes: 3 additions & 3 deletions examples/sklearnex/dbscan_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from warnings import warn

import dpctl.tensor as dpt
import dpnp
import numpy as np
from dpctl import SyclQueue
from mpi4py import MPI
Expand Down Expand Up @@ -55,8 +55,8 @@ def get_test_data(size):

queue = SyclQueue("gpu")

dpt_X = dpt.asarray(X, usm_type="device", sycl_queue=queue)
dpnp_X = dpnp.asarray(X, usm_type="device", sycl_queue=queue)

model = DBSCAN(eps=3, min_samples=2).fit(dpt_X)
model = DBSCAN(eps=3, min_samples=2).fit(dpnp_X)

print(f"Labels on rank {rank} (slice of 2):\n", model.labels_[:2])
12 changes: 6 additions & 6 deletions ...nex/incremental_basic_statistics_dpctl.py → ...rnex/incremental_basic_statistics_dpnp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,25 @@
# ==============================================================================

import dpctl
import dpctl.tensor as dpt
import dpnp

from sklearnex.basic_statistics import IncrementalBasicStatistics

# We create GPU SyclQueue and then put data to dpctl tensor using
# We create GPU SyclQueue and then put data to dpnp arrays using
# the queue. It allows us to do computation on GPU.

queue = dpctl.SyclQueue("gpu")
Copy link
Contributor

@Vika-F Vika-F Nov 25, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it Ok to still have dpctl.SyclQueue after dpctl tensor removal?

Copy link
Contributor

@david-cortes-intel david-cortes-intel Nov 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's not going to be removed.

But since this is using a queue object, maybe it'd be better to offload to that instead of creating dpnp arrays on GPU that will then be moved to CPU and back again to GPU during the call.


incbs = IncrementalBasicStatistics(result_options=["mean", "max", "sum"])

# We do partial_fit for each batch and then print final result.
X_1 = dpt.asarray([[0, 1], [0, 1]], sycl_queue=queue)
X_1 = dpnp.asarray([[0, 1], [0, 1]], sycl_queue=queue)
result = incbs.partial_fit(X_1)

X_2 = dpt.asarray([[1, 2]], sycl_queue=queue)
X_2 = dpnp.asarray([[1, 2]], sycl_queue=queue)
result = incbs.partial_fit(X_2)

X_3 = dpt.asarray([[1, 1], [1, 2], [2, 3]], sycl_queue=queue)
X_3 = dpnp.asarray([[1, 1], [1, 2], [2, 3]], sycl_queue=queue)
result = incbs.partial_fit(X_3)

print(f"Mean:\n{result.mean_}")
Expand All @@ -43,7 +43,7 @@
# We put the whole data to fit method, it is split automatically and then
# partial_fit is called for each batch.
incbs = IncrementalBasicStatistics(result_options=["mean", "max", "sum"], batch_size=3)
X = dpt.asarray([[0, 1], [0, 1], [1, 2], [1, 1], [1, 2], [2, 3]], sycl_queue=queue)
X = dpnp.asarray([[0, 1], [0, 1], [1, 2], [1, 1], [1, 2], [2, 3]], sycl_queue=queue)
result = incbs.fit(X)

print(f"Mean:\n{result.mean_}")
Expand Down
6 changes: 3 additions & 3 deletions examples/sklearnex/incremental_covariance_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# ===============================================================================

import dpctl
import dpctl.tensor as dpt
import dpnp
import numpy as np
from mpi4py import MPI

Expand Down Expand Up @@ -50,8 +50,8 @@ def get_local_data(data, comm):
# Partial fit is called for each batch on each GPU

for i in range(num_batches):
dpt_X = dpt.asarray(X_split[i], usm_type="device", sycl_queue=q)
cov.partial_fit(dpt_X)
dpnp_X = dpnp.asarray(X_split[i], usm_type="device", sycl_queue=q)
cov.partial_fit(dpnp_X)

# Finalization of results is performed in a lazy way after requesting results like in non-SPMD incremental estimators.

Expand Down
14 changes: 7 additions & 7 deletions ...ex/incremental_linear_regression_dpctl.py → ...nex/incremental_linear_regression_dpnp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,27 @@
# ==============================================================================

import dpctl
import dpctl.tensor as dpt
import dpnp

from sklearnex.linear_model import IncrementalLinearRegression

# We create GPU SyclQueue and then put data to dpctl tensors using
# We create GPU SyclQueue and then put data to dpnp arrays using
# the queue. It allows us to do computation on GPU.

queue = dpctl.SyclQueue("gpu")

inclin = IncrementalLinearRegression()

# We do partial_fit for each batch and then print final result.
X_1, y_1 = dpt.asarray([[0, 1], [1, 2]], sycl_queue=queue), dpt.asarray(
X_1, y_1 = dpnp.asarray([[0, 1], [1, 2]], sycl_queue=queue), dpnp.asarray(
[2, 4], sycl_queue=queue
)
result = inclin.partial_fit(X_1, y_1)

X_2, y_2 = dpt.asarray([[2, 3]], sycl_queue=queue), dpt.asarray([6], sycl_queue=queue)
X_2, y_2 = dpnp.asarray([[2, 3]], sycl_queue=queue), dpnp.asarray([6], sycl_queue=queue)
result = inclin.partial_fit(X_2, y_2)

X_3, y_3 = dpt.asarray([[0, 2], [1, 3], [2, 4]], sycl_queue=queue), dpt.asarray(
X_3, y_3 = dpnp.asarray([[0, 2], [1, 3], [2, 4]], sycl_queue=queue), dpnp.asarray(
[3, 5, 7], sycl_queue=queue
)
result = inclin.partial_fit(X_3, y_3)
Expand All @@ -46,9 +46,9 @@
# We put the whole data to fit method, it is split automatically and then
# partial_fit is called for each batch.
inclin = IncrementalLinearRegression(batch_size=3)
X, y = dpt.asarray(
X, y = dpnp.asarray(
[[0, 1], [1, 2], [2, 3], [0, 2], [1, 3], [2, 4]], sycl_queue=queue
), dpt.asarray([2, 4, 6, 3, 5, 7], sycl_queue=queue)
), dpnp.asarray([2, 4, 6, 3, 5, 7], sycl_queue=queue)
result = inclin.fit(X, y)

print(f"Coefs:\n{result.coef_}")
Expand Down
16 changes: 8 additions & 8 deletions examples/sklearnex/incremental_pca_dpctl.py → examples/sklearnex/incremental_pca_dpnp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
# limitations under the License.
# ==============================================================================

# sklearnex IncrementalPCA example for GPU offloading with DPCtl usm ndarray:
# sklearnex IncrementalPCA example for GPU offloading with DPNP ndarray:
Copy link
Contributor

@david-cortes-intel david-cortes-intel Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to have examples that offload to GPU through DPNP arrays if there's already support for array API that can avoid transferring data back and forth with a one-liner change?

# SKLEARNEX_PREVIEW=YES python ./incremental_pca_dpctl.py

import dpctl
import dpctl.tensor as dpt
import dpnp

# Import estimator via sklearnex's patch mechanism from sklearn
from sklearnex import patch_sklearn, sklearn_is_patched
Expand All @@ -35,23 +35,23 @@
# Or just directly import estimator from sklearnex namespace.
# from sklearnex.preview.decomposition import IncrementalPCA

# We create GPU SyclQueue and then put data to dpctl tensor using
# We create GPU SyclQueue and then put data to dpnp arrays using
# the queue. It allows us to do computation on GPU.
queue = dpctl.SyclQueue("gpu")

incpca = IncrementalPCA()

# We do partial_fit for each batch and then print final result.
X_1 = dpt.asarray([[-1, -1], [-2, -1]], sycl_queue=queue)
X_1 = dpnp.asarray([[-1, -1], [-2, -1]], sycl_queue=queue)
result = incpca.partial_fit(X_1)

X_2 = dpt.asarray([[-3, -2], [1, 1]], sycl_queue=queue)
X_2 = dpnp.asarray([[-3, -2], [1, 1]], sycl_queue=queue)
result = incpca.partial_fit(X_2)

X_3 = dpt.asarray([[2, 1], [3, 2]], sycl_queue=queue)
X_3 = dpnp.asarray([[2, 1], [3, 2]], sycl_queue=queue)
result = incpca.partial_fit(X_3)

X = dpt.concat((X_1, X_2, X_3))
X = dpnp.concat((X_1, X_2, X_3))
transformed_X = incpca.transform(X)

print(f"Principal components:\n{result.components_}")
Expand All @@ -61,7 +61,7 @@
# We put the whole data to fit method, it is split automatically and then
# partial_fit is called for each batch.
incpca = IncrementalPCA(batch_size=3)
X = dpt.asarray([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
X = dpnp.asarray([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
result = incpca.fit(X)
transformed_X = incpca.transform(X)

Expand Down
12 changes: 6 additions & 6 deletions examples/sklearnex/kmeans_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from warnings import warn

import dpctl.tensor as dpt
import dpnp
import numpy as np
from dpctl import SyclQueue
from mpi4py import MPI
Expand Down Expand Up @@ -51,17 +51,17 @@ def get_test_data(size):

queue = SyclQueue("gpu")

dpt_X = dpt.asarray(X, usm_type="device", sycl_queue=queue)
dpnp_X = dpnp.asarray(X, usm_type="device", sycl_queue=queue)

model = KMeans(n_clusters=10).fit(dpt_X)
model = KMeans(n_clusters=10).fit(dpnp_X)

print(f"Number of iterations on {rank}:\n", model.n_iter_)
print(f"Labels on rank {rank} (slice of 2):\n", model.labels_[:2])
print(f"Centers on rank {rank} (slice of 2):\n", model.cluster_centers_[:2, :])

X_test, _ = get_test_data(size)
dpt_X_test = dpt.asarray(X_test, usm_type="device", sycl_queue=queue)
dpnp_X_test = dpnp.asarray(X_test, usm_type="device", sycl_queue=queue)

result = model.predict(dpt_X_test)
result = model.predict(dpnp_X_test)

print(f"Result labels on rank {rank} (slice of 5):\n", dpt.to_numpy(result)[:5])
print(f"Result labels on rank {rank} (slice of 5):\n", dpnp.asnumpy(result)[:5])
58 changes: 0 additions & 58 deletions examples/sklearnex/knn_bf_classification_dpnp.py
View file
Open in desktop

This file was deleted.

17 changes: 8 additions & 9 deletions examples/sklearnex/knn_bf_classification_spmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from warnings import warn

import dpctl
import dpctl.tensor as dpt
import dpnp
import numpy as np
from mpi4py import MPI
from sklearn.metrics import accuracy_score
Expand Down Expand Up @@ -53,27 +53,26 @@ def generate_X_y(par, seed):
X_train, y_train = generate_X_y(params_train, rank)
X_test, y_test = generate_X_y(params_test, rank + 99)

dpt_X_train = dpt.asarray(X_train, usm_type="device", sycl_queue=q)
dpt_y_train = dpt.asarray(y_train, usm_type="device", sycl_queue=q)
dpt_X_test = dpt.asarray(X_test, usm_type="device", sycl_queue=q)
dpt_y_test = dpt.asarray(y_test, usm_type="device", sycl_queue=q)
dpnp_X_train = dpnp.asarray(X_train, usm_type="device", sycl_queue=q)
dpnp_y_train = dpnp.asarray(y_train, usm_type="device", sycl_queue=q)
dpnp_X_test = dpnp.asarray(X_test, usm_type="device", sycl_queue=q)

model_spmd = KNeighborsClassifier(
algorithm="brute", n_neighbors=20, weights="uniform", p=2, metric="minkowski"
)
model_spmd.fit(dpt_X_train, dpt_y_train)
model_spmd.fit(dpnp_X_train, dpnp_y_train)

y_predict = model_spmd.predict(dpt_X_test)
y_predict = model_spmd.predict(dpnp_X_test)

print("Brute Force Distributed kNN classification results:")
print("Ground truth (first 5 observations on rank {}):\n{}".format(rank, y_test[:5]))
print(
"Classification results (first 5 observations on rank {}):\n{}".format(
rank, dpt.to_numpy(y_predict)[:5]
rank, dpnp.asnumpy(y_predict)[:5]
)
)
print(
"Accuracy for entire rank {} (256 classes): {}\n".format(
rank, accuracy_score(y_test, dpt.to_numpy(y_predict))
rank, accuracy_score(y_test, dpnp.asnumpy(y_predict))
)
)
Loading
Loading