Merge pull request swiftlang#131 from apple/debian-docker-2

refactor debian packager

Author: shahmishal

platforms/Linux/DEB/Shared/build_deb.sh

@@ -0,0 +1,46 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+#!/bin/bash
+
+set -eux
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+# working in /tmp since docker file sharing makes this very slow to do dirctly on the share
+staging_dir=/tmp/swift-deb-builder
+package_dir=${staging_dir}/swiftlang-${debversion}
+
+# clean
+rm -rf ${package_dir} && mkdir -p ${package_dir}
+
+# copy control files to pakcage build directory
+cp -r ${here}/debian ${package_dir}/
+cp -r ${package_dir}/debian/control.in ${package_dir}/debian/control
+
+# build the source package
+${here}/build_source_package.sh ${staging_dir}
+
+# install the build dependencies
+cd ${staging_dir}
+mk-build-deps --install ${package_dir}/debian/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'
+
+# build the installable package
+# TODO: add signing key information
+cd ${package_dir}
+DEB_BUILD_OPTIONS=parallel=64 debuild
+
+# copy the final packages to /output
+cd ${staging_dir}
+cp *.deb /output/
+cp *.ddeb /output/
+cp *.dsc /output/
+cp *.tar.* /output/

platforms/Linux/DEB/Ubuntu/focal/debian/getsource.sh renamed to platforms/Linux/DEB/Shared/build_source_package.sh

@@ -1,3 +1,11 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
 #!/bin/sh
 
 # This helper script will fetch the Swift sources, save the
@@ -27,16 +35,22 @@
 #   $ cd swiftlang-X.Y.Z
 #   $ DEB_BUILD_OPTIONS=parallel=64 debuild
 
-set -eu
+set -eux
 
-. $(dirname $0)/source-versions.sh
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+staging_dir=$1
+package_dir=$staging_dir/swiftlang-${debversion}
 
 get_component ()
 {
     component=$1
     url="$2"
 
-    dest=swiftlang_${debversion}.orig-${component}
+    dest=${staging_dir}/swiftlang_${debversion}.orig-${component}
 
     echo "Downloading ${component} from ${url}"
 
@@ -69,8 +83,9 @@ get_component ()
 	    exit 1
     esac
 
-    mkdir swiftlang-${debversion}/${component}
-    tar -C swiftlang-${debversion}/${component} --strip-components=1 -axf ${dest}
+    echo "Extracting ${component}"
+    mkdir ${package_dir}/${component}
+    tar -C ${package_dir}/${component} --strip-components=1 -axf ${dest}
 }
 
 get_component swift https://github.com/apple/swift/archive/swift-${swift_version}.tar.gz
@@ -112,7 +127,7 @@ get_component swift-markdown https://github.com/apple/swift-markdown/archive/swi
 if [ -s swiftlang-${debversion}/debian/patches/series ]; then
     cd swiftlang-${debversion}
 
-    export QUILT_PATCHES=debian/patches 
+    export QUILT_PATCHES=debian/patches
     export QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index"
 
     while quilt push; do quilt refresh; done
@@ -121,6 +136,6 @@ if [ -s swiftlang-${debversion}/debian/patches/series ]; then
     cd -
 fi
 
-cp -f swiftlang-${debversion}/debian/control.in swiftlang-${debversion}/debian/control
-
-dpkg-source --create-empty-orig -b swiftlang-${debversion}
+# create a source package
+cd $staging_dir
+dpkg-source --create-empty-orig -b ${package_dir}

platforms/Linux/DEB/Shared/copyright

@@ -0,0 +1,37 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: Swift.org
+Upstream-Contact: Swift Maintainers <[email protected]>
+Source: https://github.com/apple/swift-installer-scripts
+
+Files: *
+Copyright: Apple Inc. and the Swift project authors
+License: Apache-2.0
+
+Files: debian/*
+Copyright: Apple Inc. and the Swift project authors
+License: Apache-2.0
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+ https://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache version 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
+
+# Please also look if there are files or directories which have a
+# different copyright/license attached and list them here.
+# Please avoid picking licenses with terms that are more restrictive than the
+# packaged work, as it may make Debian's contributions unacceptable upstream.
+#
+# If you need, there are some extra license texts available in two places:
+#   /usr/share/debhelper/dh_make/licenses/
+#   /usr/share/common-licenses/

platforms/Linux/DEB/Ubuntu/focal/debian/source-versions.sh renamed to platforms/Linux/DEB/Shared/versions.sh

@@ -1 +1,71 @@
 ## Ubuntu
+
+Ubuntu uses the [Deb package format]() to install software packages.
+The Swift Deb package can be built either by creating a Linux container image or manually on a computer running Ubuntu.
+
+There are separate directories for each version of Ubuntu.
+The instructions below are applicable to all versions.
+
+## Important file and directories
+
+**build_deb.sh**
+Driver file to build the deb package
+
+**build_source.sh**
+Driver file to build the deb source package (which is a step in `build_deb`)
+
+**patches/*.patch**
+Any post-release patches that have not yet been merged upstream that are temporarily necessary to build Swift.
+
+**control.in**
+Debian package metadata, including the `BuildDepends` and `Depends` definitions.
+
+**changelog**
+Debian package changelog
+
+**rules**
+Debian package recipe
+
+**Dockerfile**
+Defines the base docker image to run the install scripts in.
+
+**docker-compose**
+Defines docker compose tasks to drive the pacakge build in Docker.
+
+## Importand file and directories
+
+**Shared/version.sh**
+Shell fragment versions.sh containing version information for all source components, and the Debian package upstream version (debversion).
+
+**Shared/copyright**
+Copyright information
+
+### Building with docker-compose
+
+* to run the build end-to-end
+
+```
+docker-compose run build
+```
+
+* to enter the docker env in shell mode
+
+```
+docker-compose run shell
+```
+
+then you can run `./build_deb.sh` to run the build manually inside the docker
+
+
+* to rebuild the base image
+
+```
+docker-compose build --pull
+```
+
+note this still uses the docker cache, so will rebuild only if the version of the underlying base image changed upstream
+
+### Building locally on an Ubuntu machine
+
+1. Install required development tools (see Dockerfile)
+2. Run `./build_deb.sh`

platforms/Linux/DEB/Ubuntu/README.md

@@ -0,0 +1 @@
+.output

platforms/Linux/DEB/Ubuntu/focal/.dockerignore

@@ -0,0 +1,21 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+FROM ubuntu:focal
+LABEL PURPOSE="This image is configured to build Swift for the version of Ubuntu listed above"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+
+# Required deb packaging tools
+RUN apt-get install -y curl devscripts equivs quilt tar
+
+# Optimization: Install Swift build requirements listed in the control file
+ADD debian/control.in /tmp/control.in
+RUN mk-build-deps --install /tmp/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'

platforms/Linux/DEB/Ubuntu/focal/Dockerfile

@@ -0,0 +1 @@
+../../Shared/build_deb.sh

platforms/Linux/DEB/Ubuntu/focal/README.md

@@ -0,0 +1 @@
+../../Shared/build_source_package.sh

platforms/Linux/DEB/Ubuntu/focal/build_deb.sh

@@ -0,0 +1 @@
+../../../Shared/copyright

platforms/Linux/DEB/Ubuntu/focal/build_source_package.sh

@@ -0,0 +1,47 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+# this setup is designed to build the Deb with docker
+# usage:
+# docker-compose -f platforms/Linux/DEB/ubuntu/focal/docker-compose.yaml run build
+
+# to rebuild the base image
+# docker-compose -f platforms/Linux/DEB/ubuntu/focal/docker-compose.yaml build
+
+# to shell into the container for debugging purposes:
+# docker-compose -f platforms/Linux/DEB/ubuntu/focal/docker-compose.yaml run shell
+
+version: "2"
+
+services:
+
+  docker-setup:
+    image: ubuntu-focal-deb-builder
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+  common: &common
+    image: ubuntu-focal-deb-builder
+    depends_on: [docker-setup]
+    volumes:
+      - ../../Ubuntu/focal:/code/Ubuntu/focal:z
+      - ../../Shared:/code/Shared:z
+      - ./.output:/output:z
+    working_dir: /code/Ubuntu/focal
+    cap_drop:
+      - CAP_NET_RAW
+      - CAP_NET_BIND_SERVICE
+
+  build:
+    <<: *common
+    command: /bin/bash -cl "./build_deb.sh"
+
+  shell:
+    <<: *common
+    entrypoint: /bin/bash -l

platforms/Linux/DEB/Ubuntu/focal/debian/copyright

@@ -0,0 +1 @@
+../../Shared/versions.sh