Skip to content

Commit b000948

Browse files
raphaelsolarskiraphaelsolarski
authored
Support for scala 2.12. Versions bump: scala 2.13.16 flink 1.20.1. Flink docker images publication
* flink bump to 1.20.1 * scala 2.12 support added (2.13 removed from project name) * building flink images with patched scala * publishSigned configuration * publish as manual ci action
1 parent 1f4faca commit b000948

File tree

11 files changed

+183
-50
lines changed

11 files changed

+183
-50
lines changed

‎.github/workflows/ci.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ jobs:
66
runs-on: ubuntu-latest
77
steps:
88
- uses: actions/checkout@v1
9-
- uses: olafurpg/setup-scala@v11
9+
- uses: actions/setup-java@v4
1010
with:
11-
java-version: [email protected]
11+
distribution: temurin
12+
java-version: 17
13+
cache: sbt
14+
- uses: sbt/setup-sbt@v1
1215
- name: Test
13-
run: sbt test
16+
run: sbt "++test"

‎.github/workflows/publish.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Publish
2+
on:
3+
workflow_dispatch:
4+
jobs:
5+
build:
6+
runs-on: ubuntu-latest
7+
env:
8+
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
9+
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
10+
steps:
11+
- uses: actions/checkout@v1
12+
- uses: actions/setup-java@v4
13+
with:
14+
distribution: temurin
15+
java-version: 11
16+
cache: sbt
17+
- uses: sbt/setup-sbt@v1
18+
- name: Import GPG key
19+
id: import_gpg
20+
uses: crazy-max/ghaction-import-gpg@v5
21+
with:
22+
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
23+
passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }}
24+
- name: Build and publish maven artifacts
25+
run: sbt "+clean; +publishSigned"
26+
- name: Setup buildx builder
27+
uses: docker/setup-buildx-action@v3
28+
with:
29+
platforms: linux/amd64,linux/arm64
30+
config-inline: |
31+
[worker.oci]
32+
max-parallelism = 1
33+
- name: Login to Docker Hub
34+
uses: docker/login-action@v1
35+
with:
36+
username: ${{ secrets.DOCKERHUB_USER }}
37+
password: ${{ secrets.DOCKERHUB_TOKEN }}
38+
- name: Build and push images
39+
run: ./build-images.sh --push

‎.sbtopts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-J--add-opens=java.base/java.util=ALL-UNNAMED

‎Dockerfile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ARG FLINK_VERSION="invalid"
2+
3+
FROM flink:${FLINK_VERSION}-scala_2.12-java17
4+
5+
RUN rm $FLINK_HOME/lib/flink-scala*.jar
6+
7+
ARG FLINK_SCALA_VERSION="invalid"
8+
COPY --chown=flink:flink flink-scala-assembly-${FLINK_SCALA_VERSION}.jar $FLINK_HOME/lib/

‎README.md

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,25 @@
1-
# flink-scala-2.13
1+
# flink-scala
22

3-
For now Flink does not support Scala 2.13. For more refer to <https://issues.apache.org/jira/browse/FLINK-13414>.
3+
This module is a replacement for the `org.apache.flink:flink-scala` lib shipped with flink distribution,
4+
and allows using flink with higher scala versions than 2.12.8.
45

5-
Our solution to deploy Scala 2.13 code to Flink, until it's officially supported (or Flink becomes really scala-free):
6+
For more refer to <https://issues.apache.org/jira/browse/FLINK-13414>.
67

8+
## Replacing flink-scala in flink distribution
79
```bash
810
rm $FLINK_HOME/lib/flink-scala*.jar
9-
wget https://repo1.maven.org/maven2/pl/touk/flink-scala-2-13_2.13/1.1.2/flink-scala-2-13_2.13-1.1.2-assembly.jar -O $FLINK_HOME/lib/flink-scala-2-13_2.13-1.1.1-assembly.jar
11+
12+
wget https://central.sonatype.com/repository/maven-snapshots/pl/touk/flink-scala_2.13/1.1.3-SNAPSHOT/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar -O $FLINK_HOME/lib/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar
1013
```
1114

15+
## Using as a lib (probably only sufficient when child-first classloading is enabled on flink)
1216
```scala
13-
libraryDependencies += "pl.touk" %% "flink-scala-2-13" % "1.1.2"
17+
libraryDependencies += "pl.touk" %% "flink-scala" % "1.1.3-SNAPSHOT"
1418
```
1519

20+
## Prebuild flink images
21+
* we provide prebuild flink docker images for scala 2.12 and 2.13 on [Docker Hub](https://hub.docker.com/r/touk/flink)
22+
1623
## Publishing
1724
```
1825
sbt publishSigned sonatypeBundleRelease

‎build-images.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
set -e
3+
4+
VERSION=$(sbt --no-colors -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}')
5+
FLINK_VERSION=$(sbt --no-colors -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}')
6+
echo "FLINK_SCALA_VERSION: ${VERSION}"
7+
echo "FLINK_VERSION: ${FLINK_VERSION}"
8+
9+
if [[ "$1" == "--push" ]]; then
10+
OUTPUT_TYPE="registry"
11+
else
12+
OUTPUT_TYPE="docker"
13+
fi
14+
15+
sbt "+clean; +assembly"
16+
17+
IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.12"
18+
echo "Building Docker image with version: $IMAGE_TAG"
19+
20+
cp target/scala-2.12/flink-scala-assembly-${VERSION}.jar .
21+
22+
docker buildx build \
23+
--build-arg FLINK_VERSION=$FLINK_VERSION \
24+
--build-arg FLINK_SCALA_VERSION=$VERSION \
25+
--platform linux/amd64,linux/arm64 \
26+
--tag touk/flink:$IMAGE_TAG \
27+
--output=type=$OUTPUT_TYPE .
28+
29+
IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.13"
30+
echo "Building Docker image with version: $IMAGE_TAG"
31+
32+
cp target/scala-2.13/flink-scala-assembly-${VERSION}.jar .
33+
34+
docker buildx build \
35+
--build-arg FLINK_VERSION=$FLINK_VERSION \
36+
--build-arg FLINK_SCALA_VERSION=$VERSION \
37+
--platform linux/amd64,linux/arm64 \
38+
--tag touk/flink:$IMAGE_TAG \
39+
--output=type=$OUTPUT_TYPE .

‎build.sbt

Lines changed: 76 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,103 @@
1-
version := "1.1.2"
1+
import sbtassembly.MergeStrategy
22

3-
scalaVersion := "2.13.15"
3+
name := "flink-scala"
4+
version := "1.1.3-SNAPSHOT"
45

5-
name := "flink-scala-2.13"
6+
val scala212 = "2.12.20"
7+
val scala213 = "2.13.16"
68

7-
lazy val flinkV = "1.16.2"
8-
lazy val scalaTestV = "3.2.17"
9+
scalaVersion := scala212
10+
crossScalaVersions := List(scala212, scala213)
911

12+
val flinkV = settingKey[String]("Flink version") // to extract using `show flinkV`
13+
flinkV := "1.20.1"
1014

11-
assembly / artifact := {
12-
val art = (assembly / artifact).value
13-
art.withClassifier(Some("assembly"))
14-
}
15+
lazy val scalaTestV = "3.2.19"
1516

16-
addArtifact(assembly / artifact, assembly)
17+
lazy val assemblySettings = Seq(
18+
assembly / artifact := {
19+
val art = (assembly / artifact).value
20+
art.withClassifier(Some("assembly"))
21+
},
22+
assembly / assemblyMergeStrategy := {
23+
case PathList(ps@_*) if ps.last == "module-info.class" => MergeStrategy.discard
24+
case x => MergeStrategy.defaultMergeStrategy(x)
25+
},
26+
addArtifact(assembly / artifact, assembly)
27+
)
1728

1829
lazy val publishSettings = Seq(
1930
publishMavenStyle := true,
31+
sonatypeCredentialHost := "central.sonatype.com",
2032
publishTo := {
21-
val defaultNexusUrl = "https://oss.sonatype.org/"
22-
if (isSnapshot.value)
23-
Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots")
24-
else {
25-
sonatypePublishToBundle.value
26-
}
33+
if (isSnapshot.value)
34+
Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/")
35+
else {
36+
sonatypePublishToBundle.value //todo: full release not tested yet
37+
}
2738
},
2839
Test / publishArtifact := false,
2940
//We don't put scm information here, it will be added by release plugin and if scm provided here is different than the one from scm
3041
//we'll end up with two scm sections and invalid pom...
3142
pomExtra in Global := {
3243
<scm>
33-
<connection>scm:git:github.com/TouK/flink-scala-2.13.git</connection>
34-
<developerConnection>scm:git:git@github.com:TouK/flink-scala-2.13.git</developerConnection>
35-
<url>github.com/TouK/flink-scala-2.13</url>
44+
<connection>scm:git:github.com/TouK/flink-scala.git</connection>
45+
<developerConnection>scm:git:git@github.com:TouK/flink-scala.git</developerConnection>
46+
<url>github.com/TouK/flink-scala</url>
3647
</scm>
37-
<developers>
38-
<developer>
39-
<id>TouK</id>
40-
<name>TouK</name>
41-
<url>https://touk.pl</url>
42-
</developer>
43-
</developers>
48+
<developers>
49+
<developer>
50+
<id>TouK</id>
51+
<name>TouK</name>
52+
<url>https://touk.pl</url>
53+
</developer>
54+
</developers>
4455
},
4556
organization := "pl.touk",
4657
)
4758

4859
lazy val root = (project in file("."))
4960
.settings(
50-
name := "flink-scala-2.13",
61+
name := "flink-scala",
5162
organization := "pl.touk",
5263
licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")),
53-
homepage := Some(url("https://github.com/TouK/flink-scala-2.13")),
54-
libraryDependencies ++= {
55-
Seq(
56-
"org.scala-lang" % "scala-compiler" % scalaVersion.value,
57-
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
58-
59-
"org.apache.flink" % "flink-streaming-java" % flinkV % "provided",
60-
"com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"),
61-
"com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided",
62-
63-
"org.scalatest" %% "scalatest" % scalaTestV % "test",
64-
)
65-
}
64+
homepage := Some(url("https://github.com/TouK/flink-scala")),
65+
libraryDependencies ++= (forScalaVersion(scalaVersion.value) {
66+
case (2, 12) =>
67+
Seq(
68+
"org.apache.flink" %% "flink-scala" % flinkV.value excludeAll(
69+
ExclusionRule(organization = "org.apache.flink", name = "flink-core"),
70+
ExclusionRule(organization = "org.apache.flink", name = "flink-java"),
71+
ExclusionRule(organization = "org.apache.flink", name = "flink-shaded-asm-9"),
72+
ExclusionRule(organization = "org.slf4j", name = "slf4j-api"),
73+
ExclusionRule(organization = "com.google.code.findbugs", name = "jsr305"),
74+
),
75+
"com.esotericsoftware.kryo" % "kryo" % "2.24.0" % Test,
76+
"org.apache.flink" % "flink-java" % flinkV.value % Test,
77+
)
78+
case (2, 13) =>
79+
Seq(
80+
"org.apache.flink" % "flink-streaming-java" % flinkV.value % "provided",
81+
"com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"),
82+
"com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided",
83+
)
84+
} ++ Seq(
85+
"org.scala-lang" % "scala-library" % scalaVersion.value,
86+
"org.scala-lang" % "scala-compiler" % scalaVersion.value,
87+
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
88+
"org.scalatest" %% "scalatest" % scalaTestV % Test,
89+
))
6690
)
91+
.settings(assemblySettings *)
6792
.settings(publishSettings)
93+
94+
def forScalaVersion[T](version: String)(provide: PartialFunction[(Int, Int), T]): T = {
95+
CrossVersion.partialVersion(version) match {
96+
case Some((major, minor)) if provide.isDefinedAt((major.toInt, minor.toInt)) =>
97+
provide((major.toInt, minor.toInt))
98+
case Some(_) =>
99+
throw new IllegalArgumentException(s"Scala version $version is not handled")
100+
case None =>
101+
throw new IllegalArgumentException(s"Invalid Scala version $version")
102+
}
103+
}

‎project/plugins.sbt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4")
1+
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1")
22
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0")
33
addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1")

‎src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala renamed to ‎src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,4 +233,4 @@ class ScalaCollectionsRegistrarCompat extends IKryoRegistrar {
233233
)
234234
)
235235
}
236-
}
236+
}

‎src/main/scala/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala renamed to ‎src/main/scala-2.13/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala

File renamed without changes.

0 commit comments

Comments
 (0)