Skip to content

Commit 9677b9b

Browse files
HyukjinKwonHyukjinKwon
committed
Move connect server and common to builtin module
1 parent 399980e commit 9677b9b

2,254 files changed

Lines changed: 80 additions & 162 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/labeler.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ DEPLOY:
225225
CONNECT:
226226
- changed-files:
227227
- any-glob-to-any-file: [
228+
'connect/**/*',
228229
'connector/connect/**/*',
229230
'python/pyspark/sql/**/connect/**/*',
230231
'python/pyspark/ml/**/connect/**/*'

‎.github/workflows/build_and_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ jobs:
612612
- name: Breaking change detection against branch-3.5
613613
uses: bufbuild/buf-breaking-action@v1
614614
with:
615-
input: connector/connect/common/src/main
615+
input: connect/common/src/main
616616
against: 'https://github.com/apache/spark.git#branch=branch-3.5,subdir=connector/connect/common/src/main'
617617
- name: Install Python 3.9
618618
uses: actions/setup-python@v5

‎.github/workflows/build_python_connect.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ jobs:
8484
# Start a Spark Connect server for local
8585
PYTHONPATH="python/lib/pyspark.zip:python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH" ./sbin/start-connect-server.sh \
8686
--driver-java-options "-Dlog4j.configurationFile=file:$GITHUB_WORKSPACE/conf/log4j2.properties" \
87-
--jars "`find connector/connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
87+
--jars "`find connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
8888
8989
# Remove Py4J and PySpark zipped library to make sure there is no JVM connection
9090
mv python/lib lib.back
@@ -104,7 +104,7 @@ jobs:
104104
PYTHONPATH="python/lib/pyspark.zip:python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH" ./sbin/start-connect-server.sh \
105105
--master "local-cluster[2, 4, 1024]" \
106106
--driver-java-options "-Dlog4j.configurationFile=file:$GITHUB_WORKSPACE/conf/log4j2.properties" \
107-
--jars "`find connector/connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
107+
--jars "`find connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
108108
109109
# Remove Py4J and PySpark zipped library to make sure there is no JVM connection
110110
mv python/lib lib.back

‎.github/workflows/build_python_connect35.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ jobs:
8787
# Start a Spark Connect server for local
8888
PYTHONPATH="python/lib/pyspark.zip:python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH" ./sbin/start-connect-server.sh \
8989
--driver-java-options "-Dlog4j.configurationFile=file:$GITHUB_WORKSPACE/conf/log4j2.properties" \
90-
--jars "`find connector/connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
90+
--jars "`find connect/server/target -name spark-connect-*SNAPSHOT.jar`,`find connector/protobuf/target -name spark-protobuf-*SNAPSHOT.jar`,`find connector/avro/target -name spark-avro*SNAPSHOT.jar`"
9191
9292
# Checkout to branch-3.5 to use the tests in branch-3.5.
9393
cd ..

‎.github/workflows/maven_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ jobs:
194194
if [[ "$INCLUDED_TAGS" != "" ]]; then
195195
./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae
196196
elif [[ "$MODULES_TO_TEST" == "connect" ]]; then
197-
./build/mvn $MAVEN_CLI_OPTS -Dtest.exclude.tags="$EXCLUDED_TAGS" -Djava.version=${JAVA_VERSION/-ea} -pl connector/connect/client/jvm,connector/connect/common,connector/connect/server test -fae
197+
./build/mvn $MAVEN_CLI_OPTS -Dtest.exclude.tags="$EXCLUDED_TAGS" -Djava.version=${JAVA_VERSION/-ea} -pl connector/connect/client/jvm,connect/common,connect/server test -fae
198198
elif [[ "$EXCLUDED_TAGS" != "" ]]; then
199199
./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.exclude.tags="$EXCLUDED_TAGS" test -fae
200200
elif [[ "$MODULES_TO_TEST" == *"sql#hive-thriftserver"* ]]; then

‎assembly/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@
7474
<artifactId>spark-repl_${scala.binary.version}</artifactId>
7575
<version>${project.version}</version>
7676
</dependency>
77+
<dependency>
78+
<groupId>org.apache.spark</groupId>
79+
<artifactId>spark-connect_${scala.binary.version}</artifactId>
80+
<version>${project.version}</version>
81+
</dependency>
7782

7883
<!--
7984
Because we don't shade dependencies anymore, we need to restore Guava to compile scope so

‎connect/common/README.md‎

Lines changed: 5 additions & 0 deletions

connector/connect/common/pom.xml renamed to connect/common/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
<groupId>org.apache.spark</groupId>
2424
<artifactId>spark-parent_2.13</artifactId>
2525
<version>4.0.0-SNAPSHOT</version>
26-
<relativePath>../../../pom.xml</relativePath>
26+
<relativePath>../../pom.xml</relativePath>
2727
</parent>
2828

2929
<artifactId>spark-connect-common_2.13</artifactId>

connector/connect/common/src/main/buf.gen.yaml renamed to connect/common/src/main/buf.gen.yaml

File renamed without changes.

connector/connect/common/src/main/buf.work.yaml renamed to connect/common/src/main/buf.work.yaml

File renamed without changes.

0 commit comments

Comments
 (0)