Skip to content

[FLINK-5005] WIP: publish scala 2.12 artifacts #3703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
DieBauer wants to merge 31 commits into from
Closed

[FLINK-5005] WIP: publish scala 2.12 artifacts #3703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
f5bb06c
initial commit.
DieBauer Jan 3, 2017
c702228
Merge branch 'master' of https://github.com/apache/flink into feature…
DieBauer Apr 7, 2017
92faa20
Change shell scripts
DieBauer Apr 7, 2017
80696a5
part1
DieBauer Apr 7, 2017
d6560a3
flink-tests
DieBauer Apr 7, 2017
dd69422
fix compile error
DieBauer Apr 7, 2017
8d4e345
since scala 2.12 use java.util.concurrent.forkjoinpool directly
DieBauer Apr 7, 2017
bf13083
fix ambiguous methods
DieBauer Apr 7, 2017
edcf0e3
take into account that only kafka 0.10 is for scala 2.12
DieBauer Apr 7, 2017
3027e0f
gelly
DieBauer Apr 7, 2017
b944578
extract version breeze
DieBauer Apr 7, 2017
ac89e24
fix flink ml
DieBauer Apr 7, 2017
f9918c8
add iloopcompat for scala-2.12
DieBauer Apr 7, 2017
3e6482a
make it compile with 2.12
DieBauer Apr 8, 2017
566c467
build all with travis
DieBauer Apr 8, 2017
64c99b6
profiles
DieBauer Apr 8, 2017
2f3079d
update pom profile
DieBauer Apr 8, 2017
4e62b3f
updated pom profile
DieBauer Apr 8, 2017
83d5059
update javassist
DieBauer Apr 8, 2017
b86648d
bump scalatest
DieBauer Apr 8, 2017
b46632e
Add types to foreach
DieBauer Apr 8, 2017
595aa45
make profiles better
DieBauer Apr 8, 2017
9541498
update chill to 2.12 version compat
DieBauer Apr 8, 2017
3a335f8
use jdk8 profile for travis and scala 2.12
DieBauer Apr 8, 2017
7c08869
clean up
DieBauer Apr 8, 2017
5c6b296
move akka version to parent pom iso profile
DieBauer Apr 8, 2017
2e480d9
add flag for kafkamodules
DieBauer Apr 9, 2017
dbe541b
Bump asm version to 5.2 and implement visitMethodInsn correct
DieBauer Apr 16, 2017
9973b2a
wip closurecleaner for lambdas
DieBauer Apr 16, 2017
ec4fb48
rename methods for flink iso spark
DieBauer Apr 16, 2017
76a7395
add license file
DieBauer Apr 17, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ language: java
#See https://issues.apache.org/jira/browse/FLINK-1072
matrix:
include:
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"

# Always run test groups A and B together
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
Expand All @@ -23,11 +30,18 @@ matrix:
env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"

- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
env: PROFILE="-Dhadoop.version=2.6.3 -Dscala-2.10 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
env: PROFILE="-Dhadoop.version=2.6.3 -Dscala-2.10 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
env: PROFILE="-Dhadoop.version=2.6.3 -Dscala-2.10 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"

- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis,jdk8"
- jdk: "oraclejdk8"
env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.12 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8"

- jdk: "openjdk7"
env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis"
Expand All @@ -37,17 +51,16 @@ matrix:
env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis"

- jdk: "oraclejdk7"
env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-a,include-kinesis"
env: PROFILE="-Dhadoop.version=2.3.0 -Dscala-2.10 -Pflink-fast-tests-a,include-kinesis"
- jdk: "oraclejdk7"
env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis"
env: PROFILE="-Dhadoop.version=2.3.0 -Dscala-2.10 -Pflink-fast-tests-b,include-kinesis"
- jdk: "oraclejdk7"
env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-c,include-kinesis"
env: PROFILE="-Dhadoop.version=2.3.0 -Dscala-2.10 -Pflink-fast-tests-c,include-kinesis"


git:
depth: 100


env:
global:
# Global variable to avoid hanging travis builds when downloading cache archives.
Expand Down
40 changes: 34 additions & 6 deletions flink-clients/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,7 @@ under the License.
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.data-artisans</groupId>
<artifactId>flakka-testkit_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<!-- More information on this:
Expand Down Expand Up @@ -192,4 +187,37 @@ under the License.
</plugins>
</pluginManagement>
</build>

<profiles>
<profile>
<id>scala-2.12</id>
<activation>
<property>
<!-- this is the default scala profile -->
<name>scala-2.12</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_2.12</artifactId>
</dependency>
</dependencies>
</profile>
<profile>
<id>scala</id>
<activation>
<property>
<!-- this is the default scala profile -->
<name>!scala-2.12</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>com.data-artisans</groupId>
<artifactId>flakka-testkit_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion flink-connectors/flink-connector-kafka-0.10/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ under the License.

<!-- Allow users to pass custom connector versions -->
<properties>
<kafka.version>0.10.0.1</kafka.version>
<kafka.version>0.10.1.1</kafka.version>
</properties>

<dependencies>
Expand Down
6 changes: 5 additions & 1 deletion ...0/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import kafka.admin.AdminUtils;
import kafka.common.KafkaException;
import kafka.metrics.KafkaMetricsReporter;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
import kafka.utils.SystemTime$;
Expand All @@ -42,6 +43,7 @@
import org.apache.kafka.common.requests.MetadataResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.collection.Seq$;

import java.io.File;
import java.net.BindException;
Expand Down Expand Up @@ -381,7 +383,9 @@ protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Except

try {
scala.Option<String> stringNone = scala.Option.apply(null);
KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
scala.collection.Seq emptyReporter =
Seq$.MODULE$.<KafkaMetricsReporter>empty();
KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone, emptyReporter);
server.startup();
return server;
}
Expand Down
2 changes: 1 addition & 1 deletion flink-connectors/flink-connector-kafka-base/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ under the License.

<!-- Allow users to pass custom connector versions -->
<properties>
<kafka.version>0.8.2.2</kafka.version>
<kafka.version>0.10.1.1</kafka.version>
</properties>

<dependencies>
Expand Down
21 changes: 19 additions & 2 deletions flink-connectors/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ under the License.
<module>flink-hbase</module>
<module>flink-hcatalog</module>
<module>flink-connector-kafka-base</module>
<module>flink-connector-kafka-0.8</module>
<module>flink-connector-kafka-0.9</module>
<module>flink-connector-kafka-0.10</module>
<module>flink-connector-elasticsearch-base</module>
<module>flink-connector-elasticsearch</module>
Expand Down Expand Up @@ -100,6 +98,25 @@ under the License.
<module>flink-connector-elasticsearch5</module>
</modules>
</profile>

<profile>
<id>scala-2.12</id>
<properties>
<kafka.version>0.10.1.1</kafka.version>
</properties>
</profile>
<profile>
<id>scala</id>
<activation>
<property>
<name>!scala-2.12</name>
</property>
</activation>
<modules>
<module>flink-connector-kafka-0.8</module>
<module>flink-connector-kafka-0.9</module>
</modules>
</profile>
</profiles>

</project>
38 changes: 31 additions & 7 deletions flink-contrib/flink-streaming-contrib/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,6 @@ under the License.
<jvmArg>-Xms128m</jvmArg>
<jvmArg>-Xmx512m</jvmArg>
</jvmArgs>
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>

Expand Down Expand Up @@ -184,4 +177,35 @@ under the License.
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>scala</id>
<activation>
<property>
<name>!scala-2.12</name>
</property>
</activation>
<build>
<plugins>

<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>

</plugins>
</build>
</profile>
</profiles>
</project>
4 changes: 2 additions & 2 deletions ...k-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/DeltaPageRank.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ object DeltaPageRank {
(solutionSet, workset) =>
{
val deltas = workset.join(adjacency).where(0).equalTo(0) {
(lastDeltas, adj, out: Collector[Page]) =>
(lastDeltas: (Long, Double), adj: (Long, Array[Long]), out: Collector[Page]) =>
{
val targets = adj._2
val deltaPerTarget = DAMPENING_FACTOR * lastDeltas._2 / targets.length
Expand All @@ -88,7 +88,7 @@ object DeltaPageRank {
}
}
.groupBy(0).sum(1)
.filter(x => Math.abs(x._2) > THRESHOLD)
.filter((x: (Long, Double)) => Math.abs(x._2) > THRESHOLD)

val rankUpdates = solutionSet.join(deltas).where(0).equalTo(0) {
(current, delta) => (current._1, current._2 + delta._2)
Expand Down
2 changes: 1 addition & 1 deletion ...k-examples-batch/src/main/scala/org/apache/flink/examples/scala/graph/PageRankBasic.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ object PageRankBasic {
// collect ranks and sum them up
.groupBy("pageId").aggregate(SUM, "rank")
// apply dampening factor
.map { p =>
.map { p: Page =>
Page(p.pageId, (p.rank * DAMPENING_FACTOR) + ((1 - DAMPENING_FACTOR) / numPages))
}.withForwardedFields("pageId")

Expand Down
5 changes: 3 additions & 2 deletions ...s-batch/src/main/scala/org/apache/flink/examples/scala/graph/TransitiveClosureNaive.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ object TransitiveClosureNaive {
val nextPaths = prevPaths
.join(edges)
.where(1).equalTo(0) {
(left, right) => (left._1,right._2)
(left: (Long, Long), right: (Long, Long)) => (left._1,right._2)
}.withForwardedFieldsFirst("_1").withForwardedFieldsSecond("_2")
.union(prevPaths)
.groupBy(0, 1)
Expand All @@ -67,7 +67,8 @@ object TransitiveClosureNaive {
val terminate = prevPaths
.coGroup(nextPaths)
.where(0).equalTo(0) {
(prev, next, out: Collector[(Long, Long)]) => {
(prev: Iterator[(Long, Long)], next: Iterator[(Long, Long)],
out: Collector[(Long, Long)]) => {
val prevPaths = prev.toSet
for (n <- next)
if (!prevPaths.contains(n)) out.collect(n)
Expand Down
12 changes: 7 additions & 5 deletions ...ples-batch/src/main/scala/org/apache/flink/examples/scala/relational/WebLogAnalysis.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,20 +105,22 @@ object WebLogAnalysis {
val visits = getVisitsDataSet(env, params)

val filteredDocs = documents
.filter(doc => doc._2.contains(" editors ") && doc._2.contains(" oscillations "))
.filter((doc: (String, String)) =>
doc._2.contains(" editors ") && doc._2.contains(" oscillations "))

val filteredRanks = ranks
.filter(rank => rank._1 > 40)
.filter((rank: (Int, String, Int)) => rank._1 > 40)

val filteredVisits = visits
.filter(visit => visit._2.substring(0, 4).toInt == 2007)
.filter((visit: (String, String)) => visit._2.substring(0, 4).toInt == 2007)

val joinDocsRanks = filteredDocs.join(filteredRanks).where(0).equalTo(1) {
(doc, rank) => rank
(doc: (String, String), rank: (Int, String, Int)) => rank
}.withForwardedFieldsSecond("*")

val result = joinDocsRanks.coGroup(filteredVisits).where(1).equalTo(0) {
(ranks, visits, out: Collector[(Int, String, Int)]) =>
(ranks: Iterator[(Int, String, Int)], visits: Iterator[(String, String)],
out: Collector[(Int, String, Int)]) =>
if (visits.isEmpty) for (rank <- ranks) out.collect(rank)
}.withForwardedFieldsFirst("*")

Expand Down
40 changes: 32 additions & 8 deletions flink-libraries/flink-gelly-examples/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,16 +124,9 @@
<jvmArg>-Xms128m</jvmArg>
<jvmArg>-Xmx512m</jvmArg>
</jvmArgs>
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>

<!-- Eclipse Integration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand Down Expand Up @@ -222,4 +215,35 @@
</plugins>
</build>


<profiles>
<profile>
<id>scala</id>
<activation>
<property>
<name>!scala-2.12</name>
</property>
</activation>

<build>
<plugins>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>

</project>
Loading