update fedora setup to use docke copose and follow conventions of other RPM builders

motivation: consistentcy across RPM builders

changes:
* add docker-compose setup
* update rpmspec to use share metadata
* update rpmspec to download cmake and not rely on the one that ships with the distro
* other minor formatting adjustments to make rpmspec consistent with other rpmspecs
* update Dockerfil to use updated rpmspec
* update build script to use updated docker file and rpmspect
* add readme

Author: tomerd

platforms/Linux/RPM/Fedora/34/Dockerfile

@@ -1,30 +1,20 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
 FROM fedora:34
 LABEL PURPOSE="This image is configured to build Swift for the version of Fedora listed above"
 
-WORKDIR /root
-
 RUN yum -y update
 
 # RPM and yum development tools
-RUN yum install -y rpmdevtools yum-utils
-
-# Add the spec
-RUN mkdir -p /root/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-ADD swiftlang.spec /root/rpmbuild/SPECS/swiftlang.spec
-
-
-# Install all the dependencies needed to build Swift from the spec file itself
-RUN yum-builddep -y /root/rpmbuild/SPECS/swiftlang.spec
-
-# Get the sources for Swift as defined in the spec file
-RUN spectool -g -R /root/rpmbuild/SPECS/swiftlang.spec
-
-# Add the patches
-ADD patches/*.patch /root/rpmbuild/SOURCES/
-
-# Add the driver script
-ADD build_rpm.sh /root/build_rpm.sh
-RUN chmod +x /root/build_rpm.sh
-
+RUN yum install -y rpmdevtools yum-utils createrepo
 
-CMD ["/root/build_rpm.sh"]
+# Optimization: Install all the dependencies needed to build Swift from the spec file itself
+ADD swiftlang.spec /tmp/swiftlang.spec
+RUN touch /tmp/metadata.inc # fake metadata is okay for this optimization
+RUN cd /tmp && yum-builddep -y swiftlang.spec

platforms/Linux/RPM/Fedora/34/README.md

@@ -0,0 +1,31 @@
+# Building Swift on Fedora Linux
+
+
+### building with docker-compose
+
+* to run the build end-to-end
+
+```
+docker-compose run build
+```
+
+* to enter the docker env in shell mode
+
+```
+docker-compose run shell
+```
+
+then you can run `./build_rpm.sh` to run the build manually inside the docker
+
+
+* to rebuild the base image
+
+```
+docker-compose build --pull
+```
+
+note this still uses the docker cache, so will rebuild only if the version of the underlying base image changed upstream
+
+
+### Open Issues / TODO
+* the list of build requirements (BuildRequires) and especially requirements (Requires) should come from an external file, likely one per swift release version (which we can use it to also drive documentation)

platforms/Linux/RPM/Fedora/34/build_rpm.sh

@@ -1,25 +1,43 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
 #!/usr/bin/env bash
 
-# This script assumes it's running in a container as root
-# and that /out is mounted to a directory on the local
-# filesystem so the build output and artifacts can be
-# copied out and used
+set -ex
 
-OUTDIR=/out
-if [[ ! -d "$OUTDIR" ]]
-then
+OUTDIR=/output
+if [[ ! -d "$OUTDIR" ]]; then
     echo "$OUTDIR does not exist, so no place to copy the artifacts!"
     exit 1
 fi
 
-# Always make sure we're up to date
-yum -y update
+# always make sure we're up to date
+yum update -y
+
+# prepare direcoties
+mkdir -p $HOME/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
+
+# Add the spec
+cp swiftlang.spec $HOME/rpmbuild/SPECS/
+# Add the metadata for this swift version
+cp /shared/metadata.inc $HOME/rpmbuild/SPECS/
+# Add any patches
+cp patches/*.patch $HOME/rpmbuild/SOURCES/
 
+pushd $HOME/rpmbuild/SPECS
+# install all the dependencies needed to build Swift from the spec file itself
+yum-builddep -y ./swiftlang.spec
+# get the sources for Swift as defined in the spec file
+spectool -g -R ./swiftlang.spec
 # Now we proceed to build Swift. If this is successful, we
 # will have two files: a SRPM file which contains the source files
 # as well as a regular RPM file that can be installed via `dnf' or `yum'
-pushd $HOME/rpmbuild/SPECS
-rpmbuild -ba ./swiftlang.spec 2>&1 | tee $HOME/build-output.txt
+rpmbuild -ba ./swiftlang.spec 2>&1 | tee /root/build-output.txt
 popd
 
 # Include the build log which can be used to determine what went

platforms/Linux/RPM/Fedora/34/docker-compose.yaml

@@ -0,0 +1,47 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+# this setup is designed to build the RPM with docker
+# usage:
+# docker-compose -f platforms/Linux/fedora/34/docker-compose.yaml build
+# to shell into the container for debugging purposes:
+# docker-compose -f platforms/Linux/fedora/34/docker-compose.yaml run build
+
+version: "2"
+
+services:
+
+  docker-setup:
+    image: fedora-34-rpm-builder
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+  common: &common
+    image: fedora-34-rpm-builder
+    depends_on: [docker-setup]
+    volumes:
+      - .:/code:z
+      - ../../../RPM:/shared:ro
+      - ./.output:/output:z
+    working_dir: /code
+    cap_drop:
+      - CAP_NET_RAW
+      - CAP_NET_BIND_SERVICE
+
+  build:
+    <<: *common
+    command: /bin/bash -cl "./build_rpm.sh"
+
+  createrepo:
+    <<: *common
+    command: /bin/bash -cl "../shared/createrepo_rpm.sh"
+
+  shell:
+    <<: *common
+    entrypoint: /bin/bash -l

platforms/Linux/RPM/Fedora/34/swiftlang.spec

@@ -1,45 +1,39 @@
+%include metadata.inc
+
 %global debug_package %{nil}
-%global linux_version fedora
-%global swifttag 5.5.3-RELEASE
-%global swiftbuild swift-source
-%global icu_version 65-1
-%global yams_version 4.0.2
-%global sap_version 0.4.3
-%global swift_crypto_version 1.1.5
-%global ninja_version 1.10.2
-%global package_version 5.5.3
-
-Name:           swiftlang
-Version:        5.5.3
+
+Name:           %{package_name}
+Version:        %{package_version}
 Release:        1%{?dist}
-Summary:        The Swift programming language
-License:        Apache 2.0
-URL:            https://www.swift.org
-
-Source0:        https://github.com/apple/swift/archive/swift-%{swifttag}.tar.gz#/swift.tar.gz
-Source1:        https://github.com/apple/swift-corelibs-libdispatch/archive/swift-%{swifttag}.tar.gz#/corelibs-libdispatch.tar.gz
-Source2:        https://github.com/apple/swift-corelibs-foundation/archive/swift-%{swifttag}.tar.gz#/corelibs-foundation.tar.gz
-Source3:        https://github.com/apple/swift-integration-tests/archive/swift-%{swifttag}.tar.gz#/swift-integration-tests.tar.gz
-Source4:        https://github.com/apple/swift-corelibs-xctest/archive/swift-%{swifttag}.tar.gz#/corelibs-xctest.tar.gz
-Source5:        https://github.com/apple/swift-package-manager/archive/swift-%{swifttag}.tar.gz#/package-manager.tar.gz
-Source6:        https://github.com/apple/swift-llbuild/archive/swift-%{swifttag}.tar.gz#/llbuild.tar.gz
-Source7:        https://github.com/apple/swift-cmark/archive/swift-%{swifttag}.tar.gz#/cmark.tar.gz
-Source8:        https://github.com/apple/swift-xcode-playground-support/archive/swift-%{swifttag}.tar.gz#/swift-xcode-playground-support.tar.gz
-Source9:        https://github.com/apple/sourcekit-lsp/archive/swift-%{swifttag}.tar.gz#/sourcekit-lsp.tar.gz
-Source10:       https://github.com/apple/indexstore-db/archive/swift-%{swifttag}.tar.gz#/indexstore-db.tar.gz
-Source11:       https://github.com/apple/llvm-project/archive/swift-%{swifttag}.tar.gz#/llvm-project.tar.gz
-Source12:       https://github.com/apple/swift-tools-support-core/archive/swift-%{swifttag}.tar.gz#/swift-tools-support-core.tar.gz
-Source13:       https://github.com/apple/swift-argument-parser/archive/%{sap_version}.tar.gz
-Source14:       https://github.com/apple/swift-driver/archive/swift-%{swifttag}.tar.gz#/swift-driver.tar.gz
+Summary:        %{package_summary}
+License:        %{package_license}
+URL:            %{package_url}
+
+Source0:        https://github.com/apple/swift/archive/swift-%{swift_version}.tar.gz#/swift.tar.gz
+Source1:        https://github.com/apple/swift-corelibs-libdispatch/archive/swift-%{swift_version}.tar.gz#/corelibs-libdispatch.tar.gz
+Source2:        https://github.com/apple/swift-corelibs-foundation/archive/swift-%{swift_version}.tar.gz#/corelibs-foundation.tar.gz
+Source3:        https://github.com/apple/swift-integration-tests/archive/swift-%{swift_version}.tar.gz#/swift-integration-tests.tar.gz
+Source4:        https://github.com/apple/swift-corelibs-xctest/archive/swift-%{swift_version}.tar.gz#/corelibs-xctest.tar.gz
+Source5:        https://github.com/apple/swift-package-manager/archive/swift-%{swift_version}.tar.gz#/package-manager.tar.gz
+Source6:        https://github.com/apple/swift-llbuild/archive/swift-%{swift_version}.tar.gz#/llbuild.tar.gz
+Source7:        https://github.com/apple/swift-cmark/archive/swift-%{swift_version}.tar.gz#/cmark.tar.gz
+Source8:        https://github.com/apple/swift-xcode-playground-support/archive/swift-%{swift_version}.tar.gz#/swift-xcode-playground-support.tar.gz
+Source9:        https://github.com/apple/sourcekit-lsp/archive/swift-%{swift_version}.tar.gz#/sourcekit-lsp.tar.gz
+Source10:       https://github.com/apple/indexstore-db/archive/swift-%{swift_version}.tar.gz#/indexstore-db.tar.gz
+Source11:       https://github.com/apple/llvm-project/archive/swift-%{swift_version}.tar.gz#/llvm-project.tar.gz
+Source12:       https://github.com/apple/swift-tools-support-core/archive/swift-%{swift_version}.tar.gz#/swift-tools-support-core.tar.gz
+Source13:       https://github.com/apple/swift-argument-parser/archive/%{swift_argument_parser_version}.tar.gz
+Source14:       https://github.com/apple/swift-driver/archive/swift-%{swift_version}.tar.gz#/swift-driver.tar.gz
 Source15:       https://github.com/unicode-org/icu/archive/release-%{icu_version}.tar.gz
-Source16:       https://github.com/apple/swift-syntax/archive/swift-%{swifttag}.zip#/swift-syntax.tar.gz
-Source17:       https://github.com/jpsim/Yams/archive/%{yams_version}.zip
+Source16:       https://github.com/apple/swift-syntax/archive/swift-%{swift_version}.zip#/swift-syntax.tar.gz
+Source17:       https://github.com/jpsim/Yams/archive/%{yams_version}.zip#/yams.tar.gz
 Source18:       https://github.com/apple/swift-crypto/archive/refs/tags/%{swift_crypto_version}.tar.gz
 Source19:       https://github.com/ninja-build/ninja/archive/refs/tags/v%{ninja_version}.tar.gz#/ninja.tar.gz
+Source20:       https://github.com/KitWare/CMake/archive/refs/tags/v%{cmake_version}.tar.gz#/cmake.tar.gz
 
 Patch0:         nocyclades.patch
-Patch1:			unusedvariable.patch
- 
+Patch1:			    unusedvariable.patch
+
 BuildRequires:  clang
 BuildRequires:  swig
 BuildRequires:  rsync
@@ -69,78 +63,68 @@ Requires:       gcc
 Requires:       ncurses-devel
 Requires:       ncurses-compat-libs
 
-ExclusiveArch:  x86_64 aarch64 
-
-Provides: 	swiftlang = %{version}-%{release}
+ExclusiveArch:  x86_64 aarch64
 
 %description
-Swift is a general-purpose programming language built using 
-a modern approach to safety, performance, and software design 
+Swift is a general-purpose programming language built using
+a modern approach to safety, performance, and software design
 patterns.
 
-The goal of the Swift project is to create the best available 
-language for uses ranging from systems programming, to mobile 
-and desktop apps, scaling up to cloud services. Most 
-importantly, Swift is designed to make writing and maintaining 
-correct programs easier for the developer. 
-
+The goal of the Swift project is to create the best available
+language for uses ranging from systems programming, to mobile
+and desktop apps, scaling up to cloud services. Most
+importantly, Swift is designed to make writing and maintaining
+correct programs easier for the developer.
 
 %prep
-%setup -q -c -n %{swiftbuild} -a 0 -a 1 -a 2 -a 3 -a 4 -a 5 -a 6 -a 7 -a 8 -a 9 -a 10 -a 11 -a 12 -a 13 -a 14 -a 15 -a 16 -a 17 -a 18 -a 19
+%setup -q -c -n %{swift_source_location} -a 0 -a 1 -a 2 -a 3 -a 4 -a 5 -a 6 -a 7 -a 8 -a 9 -a 10 -a 11 -a 12 -a 13 -a 14 -a 15 -a 16 -a 17 -a 18 -a 19 -a 20
 # The Swift build script requires directories to be named
 # in a specific way so renaming the source directories is
 # necessary
-mv swift-cmark-swift-%{swifttag} cmark
-mv swift-corelibs-foundation-swift-%{swifttag} swift-corelibs-foundation
-mv swift-corelibs-libdispatch-swift-%{swifttag} swift-corelibs-libdispatch
-mv swift-corelibs-xctest-swift-%{swifttag} swift-corelibs-xctest
-mv swift-integration-tests-swift-%{swifttag} swift-integration-tests
-mv swift-llbuild-swift-%{swifttag} llbuild
-mv swift-package-manager-swift-%{swifttag} swiftpm
-mv swift-swift-%{swifttag} swift
-mv swift-xcode-playground-support-swift-%{swifttag} swift-xcode-playground-support
-mv sourcekit-lsp-swift-%{swifttag} sourcekit-lsp
-mv indexstore-db-swift-%{swifttag} indexstore-db
-mv llvm-project-swift-%{swifttag} llvm-project
-mv swift-syntax-swift-%{swifttag} swift-syntax
-mv swift-tools-support-core-swift-%{swifttag} swift-tools-support-core
-mv swift-argument-parser-%{sap_version} swift-argument-parser
-mv swift-driver-swift-%{swifttag} swift-driver
+mv swift-cmark-swift-%{swift_version} cmark
+mv swift-corelibs-foundation-swift-%{swift_version} swift-corelibs-foundation
+mv swift-corelibs-libdispatch-swift-%{swift_version} swift-corelibs-libdispatch
+mv swift-corelibs-xctest-swift-%{swift_version} swift-corelibs-xctest
+mv swift-integration-tests-swift-%{swift_version} swift-integration-tests
+mv swift-llbuild-swift-%{swift_version} llbuild
+mv swift-package-manager-swift-%{swift_version} swiftpm
+mv swift-swift-%{swift_version} swift
+mv swift-xcode-playground-support-swift-%{swift_version} swift-xcode-playground-support
+mv sourcekit-lsp-swift-%{swift_version} sourcekit-lsp
+mv indexstore-db-swift-%{swift_version} indexstore-db
+mv llvm-project-swift-%{swift_version} llvm-project
+mv swift-syntax-swift-%{swift_version} swift-syntax
+mv swift-tools-support-core-swift-%{swift_version} swift-tools-support-core
+mv swift-argument-parser-%{swift_argument_parser_version} swift-argument-parser
+mv swift-driver-swift-%{swift_version} swift-driver
 mv swift-crypto-%{swift_crypto_version} swift-crypto
-
-# ICU 
+mv ninja-%{ninja_version} ninja
+mv CMake-%{cmake_version} cmake
 mv icu-release-%{icu_version} icu
-
-# Yams
 mv Yams-%{yams_version} yams
 
-# Ninja
-mv ninja-%{ninja_version} ninja
-
 # Remove Cyclades as it has been removed from the Linux kernel
 %patch0 -p0
 
 # Temp patch to test libdispatch issue with clang 13
 %patch1 -p0
 
-# Fix python to python3 
+# Fix python to python3
 pathfix.py -pni "%{__python3} %{py3_shbang_opts}" swift/utils/api_checker/swift-api-checker.py
 pathfix.py -pni "%{__python3} %{py3_shbang_opts}" llvm-project/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
 
-
 %build
 export VERBOSE=1
-# Before Fedora 34, we may not have /usr/bin/python, so we 
+# Before Fedora 34, we may not have /usr/bin/python, so we
 # roll our own because the build script expects there to be one.
 %if 0%{?fedora} < 34 || 0%{?el8}
 mkdir $PWD/binforpython
 ln -s /usr/bin/python3 $PWD/binforpython/python
 export PATH=$PWD/binforpython:$PATH
 %endif
 
-# Here we go!
-swift/utils/build-script --preset=buildbot_linux,no_test install_destdir=%{_builddir} installable_package=%{_builddir}/swift-%{version}-%{linux_version}.tar.gz
-
+# Run the build
+swift/utils/build-script --preset=buildbot_linux,no_test install_destdir=%{_builddir} installable_package=%{_builddir}/swift-%{version}-fedora-34.tar.gz
 
 %install
 mkdir -p %{buildroot}%{_libexecdir}/swift/%{package_version}
@@ -156,7 +140,6 @@ cp %{_builddir}/usr/share/man/man1/swift.1 %{buildroot}%{_mandir}/man1/swift.1
 # how the Swift binaries use RPATH
 export QA_SKIP_RPATHS=1
 
-
 %files
 %license swift/LICENSE.txt
 %{_bindir}/swift
@@ -165,9 +148,7 @@ export QA_SKIP_RPATHS=1
 %{_mandir}/man1/swift.1.gz
 %{_libexecdir}/swift/
 
-
 %post -p /sbin/ldconfig
 %postun -p /sbin/ldconfig
 
-
 %changelog