Separate Docker Layers for Dependencies and App jars.

src/main/scala/com/typesafe/sbt/packager/docker/DockerPlugin.scala

@@ -5,7 +5,7 @@ import java.util.UUID
 import java.util.concurrent.atomic.AtomicBoolean
 
 import sbt._
-import sbt.Keys.{clean, mappings, name, publish, publishLocal, sourceDirectory, streams, target, version}
+import sbt.Keys.{clean, mappings, name, organization, publish, publishLocal, sourceDirectory, streams, target, version}
 import com.typesafe.sbt.packager.Keys._
 import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport.{daemonUser, defaultLinuxInstallLocation}
 import com.typesafe.sbt.packager.universal.UniversalPlugin
@@ -98,6 +98,20 @@ object DockerPlugin extends AutoPlugin {
     ),
     dockerUpdateLatest := false,
     dockerAutoremoveMultiStageIntermediateImages := true,
+    dockerLayerGrouping := {
+      val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
+      (path: String) =>
+        {
+          val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
+          if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
+            2
+          else if (pathInWorkdir.startsWith("/lib/"))
+            1
+          else if (pathInWorkdir.startsWith("/bin/"))
+            1
+          else 0
+        }
+    },
     dockerAliases := {
       val alias = dockerAlias.value
       if (dockerUpdateLatest.value) {
@@ -123,13 +137,14 @@ object DockerPlugin extends AutoPlugin {
     dockerBuildCommand := dockerExecCommand.value ++ Seq("build") ++ dockerBuildOptions.value ++ Seq("."),
     dockerAdditionalPermissions := {
       val basePath = (defaultLinuxInstallLocation in Docker).value
-      (mappings in Docker).value
+      (dockerLayerMappings in Docker).value
         .collect {
           // by default we assume everything in the bin/ folder should be executable that is not a .bat file
-          case (_, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
-            DockerChmodType.UserGroupPlusExecute -> path
+          case (layerIdx, _, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
+            DockerChmodType.UserGroupPlusExecute -> pathInLayer(path, layerIdx)
           // sh files should also be marked as executable
-          case (_, path) if path.endsWith(".sh") => DockerChmodType.UserGroupPlusExecute -> path
+          case (layerIdx, _, path) if path.endsWith(".sh") =>
+            DockerChmodType.UserGroupPlusExecute -> pathInLayer(path, layerIdx)
         }
     },
     dockerCommands := {
@@ -142,19 +157,20 @@ object DockerPlugin extends AutoPlugin {
       val base = dockerBaseImage.value
       val addPerms = dockerAdditionalPermissions.value
       val multiStageId = UUID.randomUUID().toString
-
       val generalCommands = makeFrom(base) +: makeMaintainer((maintainer in Docker).value).toSeq
       val stage0name = "stage0"
+      val layerIdsAscending = (dockerLayerMappings in Docker).value.map(_._1).distinct.sorted
       val stage0: Seq[CmdLike] = strategy match {
         case DockerPermissionStrategy.MultiStage =>
           Seq(
             makeFromAs(base, stage0name),
             makeLabel("snp-multi-stage" -> "intermediate"),
             makeLabel("snp-multi-stage-id" -> multiStageId),
-            makeWorkdir(dockerBaseDirectory),
-            makeCopy(dockerBaseDirectory),
-            makeUser("root"),
-            makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))
+            makeWorkdir(dockerBaseDirectory)
+          ) ++
+            layerIdsAscending.map(l => makeCopy(s"$l", s"/$l/")) ++
+            Seq(makeUser("root")) ++ layerIdsAscending.map(
+            l => makeChmodRecursive(dockerChmodType.value, Seq(s"/$l$dockerBaseDirectory"))
           ) ++
             (addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) }) ++
             Seq(DockerStageBreak)
@@ -169,7 +185,10 @@ object DockerPlugin extends AutoPlugin {
         Seq(makeWorkdir(dockerBaseDirectory)) ++
         (strategy match {
           case DockerPermissionStrategy.MultiStage =>
-            Seq(makeCopyFrom(dockerBaseDirectory, stage0name, user, group))
+            val layerIdsAscending = (dockerLayerMappings in Docker).value.map(_._1).distinct.sorted
+            layerIdsAscending.map { layerId =>
+              makeCopyFrom(s"/$layerId$dockerBaseDirectory", dockerBaseDirectory, stage0name, user, group)
+            }
           case DockerPermissionStrategy.Run =>
             Seq(makeCopy(dockerBaseDirectory), makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))) ++
               (addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) })
@@ -231,9 +250,19 @@ object DockerPlugin extends AutoPlugin {
         }
       },
       sourceDirectory := sourceDirectory.value / "docker",
-      stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, mappings.value),
+      stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, dockerLayerMappings.value.map {
+        case (layerIdx, file, path) => (file, pathInLayer(path, layerIdx))
+      }),
       stage := (stage dependsOn dockerGenerateConfig).value,
       stagingDirectory := (target in Docker).value / "stage",
+      dockerLayerMappings := {
+        val dockerGroups = dockerLayerGrouping.value
+        val dockerFinalFiles = (mappings in Docker).value
+        for {
+          (file, path) <- dockerFinalFiles
+          layerIdx = dockerGroups(path)
+        } yield (layerIdx, file, path)
+      },
       target := target.value / "docker",
       // pick a user name that's unlikely to exist in base images
       daemonUser := "demiourgos728",
@@ -327,22 +356,25 @@ object DockerPlugin extends AutoPlugin {
     Cmd("COPY", s"$files /$files")
   }
 
+  private final def makeCopy(src: String, dest: String): CmdLike =
+    Cmd("COPY", s"$src $dest")
+
   /**
-    * @param dockerBaseDirectory the installation directory
-    * @param from files are copied from the given build stage
+    * @param src the installation directory
+    * @param stage files are copied from the given build stage
     * @param daemonUser
     * @param daemonGroup
     * @return COPY command copying all files inside the directory from another build stage.
     */
-  private final def makeCopyFrom(dockerBaseDirectory: String,
-                                 from: String,
+  private final def makeCopyFrom(src: String,
+                                 dest: String,
+                                 stage: String,
                                  daemonUser: String,
                                  daemonGroup: String): CmdLike =
-    Cmd("COPY", s"--from=$from --chown=$daemonUser:$daemonGroup $dockerBaseDirectory $dockerBaseDirectory")
+    Cmd("COPY", s"--from=$stage --chown=$daemonUser:$daemonGroup $src $dest")
 
   /**
     * @param dockerBaseDirectory the installation directory
-    * @param from files are copied from the given build stage
     * @param daemonUser
     * @param daemonGroup
     * @return COPY command copying all files inside the directory from another build stage.
@@ -501,6 +533,8 @@ object DockerPlugin extends AutoPlugin {
     inConfig(Docker)(Seq(mappings := renameDests((mappings in Universal).value, defaultLinuxInstallLocation.value)))
   }
 
+  private final def pathInLayer(path: String, layer: Int) = s"/$layer$path"
+
   private[packager] def publishLocalLogger(log: Logger) =
     new sys.process.ProcessLogger {
       override def err(err: => String): Unit =
@@ -716,5 +750,4 @@ object DockerPlugin extends AutoPlugin {
         case _ => List.empty
       }
     }
-
 }

src/main/scala/com/typesafe/sbt/packager/docker/Keys.scala

@@ -57,4 +57,10 @@ private[packager] trait DockerKeysEx extends DockerKeys {
   lazy val dockerAdditionalPermissions =
     taskKey[Seq[(DockerChmodType, String)]]("Explicit chmod calls to some of the paths.")
   val dockerApiVersion = TaskKey[Option[DockerApiVersion]]("dockerApiVersion", "The docker server api version")
+  val dockerLayerGrouping = settingKey[String => Int](
+    "Group files by path into in layers to increase docker cache hits. " +
+      "Lower index means the file would be a part of an earlier layer."
+  )
+  val dockerLayerMappings =
+    taskKey[Seq[(Int, File, String)]]("List of layer, source file and destination in Docker image.")
 }

src/sbt-test/docker/file-permission/build.sbt

@@ -19,16 +19,19 @@ lazy val root = (project in file("."))
       assert(lines(2).substring(0, 25) == "LABEL snp-multi-stage-id=") // random generated id is hard to test
       assertEquals(lines.drop(3),
         """WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1 /1/
+          |COPY 2 /2/
           |USER root
-          |RUN ["chmod", "-R", "u=rX,g=rX", "/opt/docker"]
-          |RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
+          |RUN ["chmod", "-R", "u=rX,g=rX", "/1/opt/docker"]
+          |RUN ["chmod", "-R", "u=rX,g=rX", "/2/opt/docker"]
+          |RUN ["chmod", "u+x,g+x", "/1/opt/docker/bin/file-permission-test"]
           |
           |FROM fabric8/java-centos-openjdk8-jdk
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY --from=stage0 --chown=demiourgos728:root /opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /1/opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /2/opt/docker /opt/docker
           |USER 1001:0
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)

src/sbt-test/docker/staging/test

@@ -1,4 +1,4 @@
 # Stage the distribution and ensure files show up.
 > docker:stage
 $ exists target/docker/stage/Dockerfile
-$ exists target/docker/stage/opt
+$ exists target/docker/stage/0/opt

src/sbt-test/docker/test-executableScriptName/test

@@ -1,6 +1,6 @@
 # Generate the Docker image locally
 > docker:publishLocal
 $ exists target/docker/stage/Dockerfile
-$ exists target/docker/stage/opt/docker/bin/docker-exec
+$ exists target/docker/stage/1/opt/docker/bin/docker-exec
 > checkDockerfile
-$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
+$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'

src/sbt-test/docker/test-layer-groups/build.sbt

@@ -0,0 +1,19 @@
+
+enablePlugins(JavaAppPackaging)
+
+organization := "com.example"
+name := "docker-groups"
+version := "0.1.0"
+
+dockerLayerGrouping in Docker := {
+  val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
+  (path: String) =>
+  {
+    val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
+    if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
+      2
+    else if (pathInWorkdir.startsWith("/bin/"))
+      123
+    else 0
+  }
+}

src/sbt-test/docker/test-layer-groups/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))

src/sbt-test/docker/test-layer-groups/src/main/scala/Main.scala

@@ -0,0 +1,3 @@
+object Main extends App {
+  println("Hello world")
+}

src/sbt-test/docker/test-layer-groups/test

@@ -0,0 +1,8 @@
+# Generate the Docker image locally
+> docker:publishLocal
+$ exists target/docker/stage/Dockerfile
+$ exists target/docker/stage/123/opt/docker/bin/docker-groups
+$ exists target/docker/stage/0
+$ exists target/docker/stage/2
+
+$ exec bash -c 'docker run --entrypoint=ls docker-groups:0.1.0 |tr "\n" "," | grep -q "bin,lib"'

src/sbt-test/docker/test-packageName-universal/test

@@ -1,6 +1,6 @@
 # Generate the Docker image locally
 > docker:publishLocal
 $ exists target/docker/stage/Dockerfile
-$ exists target/docker/stage/opt/docker/bin/docker-test
+$ exists target/docker/stage/1/opt/docker/bin/docker-test
 > checkDockerfile
-$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
+$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'

src/sbt-test/docker/test-packageName/test

@@ -1,6 +1,6 @@
 # Generate the Docker image locally
 > docker:publishLocal
 $ exists target/docker/stage/Dockerfile
-$ exists target/docker/stage/opt/docker/bin/docker-test
+$ exists target/docker/stage/1/opt/docker/bin/docker-test
 > checkDockerfile
-$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
+$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'

src/sphinx/formats/docker.rst

@@ -145,6 +145,13 @@ Environment Settings
   ``dockerApiVersion``
     The docker server API version. Used to leverage new docker features while maintaining backwards compatibility.
 
+  ``dockerLayerGrouping``
+    The function mapping files into separate layers to increase docker cache hits.
+    Lower index means the file would be a part of an earlier layer.
+    The main idea behind this is to COPY dependencies *.jar's first as they should change rarely.
+    In separate command COPY the application *.jar's that should change more often.
+    Defaults to detect whether the file name starts with ``ThisBuild / organization``.
+
 Publishing Settings
 ~~~~~~~~~~~~~~~~~~~