Merge pull request #13472 from rabbitmq/mergify/bp/v4.1.x/pr-13471

(Updated) Adding a "source-bundle" target to the Makefile (backport #13471)

Author: michaelklishin

Makefile

@@ -134,30 +134,17 @@ define restore_hex_cache_from_erl_term
 endef
 
 # --------------------------------------------------------------------
-# Distribution.
+# Distribution - common variables and generic functions.
 # --------------------------------------------------------------------
 
-.PHONY: source-dist clean-source-dist
-
-SOURCE_DIST_BASE ?= rabbitmq-server
-SOURCE_DIST_SUFFIXES ?= tar.xz
-SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION)
-
-# The first source distribution file is used by packages: if the archive
-# type changes, you must update all packages' Makefile.
-SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(SOURCE_DIST_SUFFIXES))
-
-.PHONY: $(SOURCE_DIST_FILES)
-
-source-dist: $(SOURCE_DIST_FILES)
-	@:
-
 RSYNC ?= rsync
 RSYNC_V_0 =
 RSYNC_V_1 = -v
 RSYNC_V_2 = -v
 RSYNC_V = $(RSYNC_V_$(V))
-RSYNC_FLAGS += -a $(RSYNC_V)		\
+BASE_RSYNC_FLAGS += -a $(RSYNC_V) \
+	       --delete					\
+	       --delete-excluded			\
 	       --exclude '.sw?' --exclude '.*.sw?'	\
 	       --exclude '*.beam'			\
 	       --exclude '*.d'				\
@@ -188,12 +175,10 @@ RSYNC_FLAGS += -a $(RSYNC_V)		\
 	       --exclude '$(notdir $(DEPS_DIR))/'	\
 	       --exclude 'hexer*'			\
 	       --exclude 'logs/'			\
-	       --exclude 'packaging'			\
 	       --exclude 'PKG_*.md'			\
 	       --exclude '/plugins/'			\
 	       --include 'cli/plugins'			\
 	       --exclude '$(notdir $(DIST_DIR))/'	\
-	       --exclude 'test'				\
 	       --exclude '/$(notdir $(PACKAGES_DIR))/'	\
 	       --exclude '/PACKAGES/'			\
 	       --exclude '/amqp_client/doc/'		\
@@ -208,9 +193,21 @@ RSYNC_FLAGS += -a $(RSYNC_V)		\
 	       --exclude '/ranch/doc/'			\
 	       --exclude '/ranch/examples/'		\
 	       --exclude '/sockjs/examples/'		\
-	       --exclude '/workflow_sources/'		\
-	       --delete					\
-	       --delete-excluded
+	       --exclude '/workflow_sources/'
+
+SOURCE_DIST_RSYNC_FLAGS += $(BASE_RSYNC_FLAGS)          \
+	       --exclude 'packaging'			\
+	       --exclude 'test'
+
+# For source-bundle, explicitly include folders that are needed
+# for tests to execute. These are added before excludes from
+# the base flags so rsync honors the first match.
+SOURCE_BUNDLE_RSYNC_FLAGS += \
+	       --include 'rabbit_shovel_test/ebin'      \
+	       --include 'rabbit_shovel_test/ebin/*'    \
+	       --include 'rabbitmq_ct_helpers/tools'    \
+	       --include 'rabbitmq_ct_helpers/tools/*'  \
+               $(BASE_RSYNC_FLAGS)
 
 TAR ?= tar
 TAR_V_0 =
@@ -228,65 +225,125 @@ ZIP_V_1 =
 ZIP_V_2 =
 ZIP_V = $(ZIP_V_$(V))
 
-.PHONY: $(SOURCE_DIST)
-.PHONY: clean-source-dist distclean-packages clean-unpacked-source-dist
-
-$(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
-	$(verbose) mkdir -p $(dir $@)
-	$(gen_verbose) $(RSYNC) $(RSYNC_FLAGS) ./ $@/
-	$(verbose) echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > "$@/git-revisions.txt"
-	$(verbose) echo "$(PROJECT) $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> "$@/git-revisions.txt"
-	$(verbose) echo "$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > "$@.git-times.txt"
-	$(verbose) cat packaging/common/LICENSE.head > $@/LICENSE
-	$(verbose) mkdir -p $@/deps/licensing
-	$(verbose) set -e; for dep in $$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \
-		$(RSYNC) $(RSYNC_FLAGS) \
-		 $$dep \
-		 $@/deps; \
+ifeq ($(shell tar --version | grep -c "GNU tar"),0)
+# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X)
+ifeq ($(shell uname | grep -c "Darwin"),0)
+TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \
+				    --gid 0 \
+				    --numeric-owner \
+				    --no-acls \
+				    --no-fflags \
+				    --no-xattrs
+endif
+else
+TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \
+				    --group 0 \
+				    --numeric-owner
+endif
+
+DIST_SUFFIXES ?= tar.xz
+
+# Function to create distribution targets
+# Args: $(1) - Full distribution path
+#       $(2) - RSYNC flags to use
+define create_dist_target
+$(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
+	$${verbose} mkdir -p $$(dir $$@)
+	$${gen_verbose} $${RSYNC} $(2) ./ $$@/
+	$${verbose} echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > $$@/git-revisions.txt
+	$${verbose} echo "$(PROJECT) $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> $$@/git-revisions.txt
+	$${verbose} echo "$$$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > $$@.git-times.txt
+	$${verbose} cat packaging/common/LICENSE.head > $$@/LICENSE
+	$${verbose} mkdir -p $$@/deps/licensing
+	$${verbose} set -e; for dep in $$$$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \
+		$${RSYNC} $(2) \
+		 $$$$dep \
+		 $$@/deps; \
 		rm -f \
-		 $@/deps/rabbit_common/rebar.config \
-		 $@/deps/rabbit_common/rebar.lock; \
-		if test -f $@/deps/$$(basename $$dep)/erlang.mk && \
-		   test "$$(wc -l $@/deps/$$(basename $$dep)/erlang.mk | awk '{print $$1;}')" = "1" && \
-		   grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$" $@/deps/$$(basename $$dep)/erlang.mk; then \
-			echo "include ../../erlang.mk" > $@/deps/$$(basename $$dep)/erlang.mk; \
+		 $$@/deps/rabbit_common/rebar.config \
+		 $$@/deps/rabbit_common/rebar.lock; \
+		if test -f $$@/deps/$$$$(basename $$$$dep)/erlang.mk && \
+		   test "$$$$(wc -l $$@/deps/$$$$(basename $$$$dep)/erlang.mk | awk '{print $$$$1;}')" = "1" && \
+		   grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$$$" $$@/deps/$$$$(basename $$$$dep)/erlang.mk; then \
+			echo "include ../../erlang.mk" > $$@/deps/$$$$(basename $$$$dep)/erlang.mk; \
 		fi; \
-		sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$|include ../../erlang.mk|" \
-		 $@/deps/$$(basename $$dep)/Makefile && \
-		rm $@/deps/$$(basename $$dep)/Makefile.bak; \
-		mix_exs=$@/deps/$$(basename $$dep)/mix.exs; \
-		if test -f $$mix_exs; then \
-			(cd $$(dirname "$$mix_exs") && \
-			 (test -d $@/deps/.hex || env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
-			 env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
+		sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$$$|include ../../erlang.mk|" \
+		 $$@/deps/$$$$(basename $$$$dep)/Makefile && \
+		rm $$@/deps/$$$$(basename $$$$dep)/Makefile.bak; \
+		mix_exs=$$@/deps/$$$$(basename $$$$dep)/mix.exs; \
+		if test -f $$$$mix_exs; then \
+			(cd $$$$(dirname "$$$$mix_exs") && \
+			 (test -d $$@/deps/.hex || env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
+			 env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
 			 cp $(CURDIR)/mk/rabbitmq-mix.mk . && \
 			 rm -rf _build deps); \
 		fi; \
-		if test -f "$$dep/license_info"; then \
-			cp "$$dep/license_info" "$@/deps/licensing/license_info_$$(basename "$$dep")"; \
-			cat "$$dep/license_info" >> $@/LICENSE; \
+		if test -f "$$$$dep/license_info"; then \
+			cp "$$$$dep/license_info" "$$@/deps/licensing/license_info_$$$$(basename $$$$dep)"; \
+			cat "$$$$dep/license_info" >> $$@/LICENSE; \
 		fi; \
-		find "$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $@/deps/licensing \; ; \
-		(cd $$dep; \
-		 echo "$$(basename "$$dep") $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \
-		 >> "$@/git-revisions.txt"; \
-		! test -d $$dep/.git || (cd $$dep; \
-		 echo "$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \
-		 >> "$@.git-times.txt"; \
+		find "$$$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $$@/deps/licensing \; ; \
+		(cd $$$$dep; \
+		 echo "$$$$(basename "$$$$dep") $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \
+		 >> "$$@/git-revisions.txt"; \
+		! test -d $$$$dep/.git || (cd $$$$dep; \
+		 echo "$$$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \
+		 >> "$$@.git-times.txt"; \
 	done
-	$(verbose) cat packaging/common/LICENSE.tail >> $@/LICENSE
-	$(verbose) find $@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $@ \;
-	$(verbose) rm -rf $@/deps/licensing
-	$(verbose) for file in $$(find $@ -name '*.app.src'); do \
+	$${verbose} cat packaging/common/LICENSE.tail >> $$@/LICENSE
+	$${verbose} find $$@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $$@ \;
+	$${verbose} rm -rf $$@/deps/licensing
+	$${verbose} for file in $$$$(find $$@ -name '*.app.src'); do \
 		sed -E -i.bak \
 		  -e 's/[{]vsn[[:blank:]]*,[[:blank:]]*(""|"0.0.0")[[:blank:]]*}/{vsn, "$(PROJECT_VERSION)"}/' \
 		  -e 's/[{]broker_version_requirements[[:blank:]]*,[[:blank:]]*\[\][[:blank:]]*}/{broker_version_requirements, ["$(PROJECT_VERSION)"]}/' \
-		  $$file; \
-		rm $$file.bak; \
+		  $$$$file; \
+		rm $$$$file.bak; \
 	done
-	$(verbose) echo "PLUGINS := $(PLUGINS)" > $@/plugins.mk
-# Remember the latest Git timestamp.
-	$(verbose) sort -r < "[email protected]" | head -n 1 > "[email protected]"
+	$${verbose} echo "PLUGINS := $(PLUGINS)" > $$@/plugins.mk
+	$${verbose} sort -r < "$$@.git-times.txt" | head -n 1 > "$$@.git-time.txt"
+	$${verbose} $$(call erlang,$$(call dump_hex_cache_to_erl_term,$$(call core_native_path,$$@),$$(call core_native_path,$$@.git-time.txt)))
+	$${verbose} find $$@ -print0 | xargs -0 touch -t "$$$$(cat $$@.git-time.txt)"
+	$${verbose} rm "$$@.git-times.txt" "$$@.git-time.txt"
+
+$(1).manifest: $(1)
+	$${gen_verbose} cd $$(dir $$@) && \
+		find $$(notdir $$<) | LC_COLLATE=C sort > $$@
+
+$(1).tar.xz: $(1).manifest
+	$${gen_verbose} cd $$(dir $$@) && \
+		$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
+		$${XZ} > $$@
+
+$(1).tar.gz: $(1).manifest
+	$${gen_verbose} cd $$(dir $$@) && \
+		$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
+		$${GZIP} --best > $$@
+
+$(1).tar.bz2: $(1).manifest
+	$${gen_verbose} cd $$(dir $$@) && \
+		$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
+		$${BZIP2} > $$@
+
+$(1).zip: $(1).manifest
+	$${verbose} rm -f $$@
+	$${gen_verbose} cd $$(dir $$@) && \
+		$${ZIP} $${ZIP_V} --names-stdin $$@ < $$(notdir $$<)
+
+endef
+
+# Function to create clean targets
+# Args: $(1) - Base name (e.g. SOURCE_DIST_BASE or BUNDLE_DIST_BASE)
+define create_clean_targets
+.PHONY: clean-$(1)
+
+clean-$(1):
+	$${gen_verbose} rm -rf -- $(1)-*
+
+# Add each clean target to the clean:: rule
+clean:: clean-$(1)
+endef
+
 # Mix Hex component requires a cache file, otherwise it refuses to build
 # offline... That cache is an ETS table with all the applications we
 # depend on, plus some versioning informations and checksums. There
@@ -300,11 +357,6 @@ $(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
 #
 # The ETS file must be recreated before compiling RabbitMQ. See the
 # `restore-hex-cache-ets-file` Make target.
-	$(verbose) $(call erlang,$(call dump_hex_cache_to_erl_term,$(call core_native_path,$@),$(call core_native_path,[email protected])))
-# Fix file timestamps to have reproducible source archives.
-	$(verbose) find $@ -print0 | xargs -0 touch -t "$$(cat "[email protected]")"
-	$(verbose) rm "[email protected]" "[email protected]"
-
 define dump_hex_cache_to_erl_term
   In = "$(1)/deps/.hex/cache.ets",
   Out = "$(1)/deps/.hex/cache.erl",
@@ -333,50 +385,35 @@ define dump_hex_cache_to_erl_term
   init:stop().
 endef
 
-$(SOURCE_DIST).manifest: $(SOURCE_DIST)
-	$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
-		find $(notdir $(SOURCE_DIST)) | LC_COLLATE=C sort > $@
+# --------------------------------------------------------------------
+# Distribution - public targets
+# --------------------------------------------------------------------
 
-ifeq ($(shell tar --version | grep -c "GNU tar"),0)
-# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X)
-ifeq ($(shell uname | grep -c "Darwin"),0)
-TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \
-				    --gid 0 \
-				    --numeric-owner \
-				    --no-acls \
-				    --no-fflags \
-				    --no-xattrs
-endif
-else
-TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \
-				    --group 0 \
-				    --numeric-owner
-endif
+SOURCE_DIST_BASE ?= rabbitmq-server
+SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION)
+SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(DIST_SUFFIXES))
 
-$(SOURCE_DIST).tar.gz: $(SOURCE_DIST).manifest
-	$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
-		$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
-		$(GZIP) --best > $@
+.PHONY: source-dist
+source-dist: $(SOURCE_DIST_FILES)
+	@:
+
+$(eval $(call create_dist_target,$(SOURCE_DIST),$(SOURCE_DIST_RSYNC_FLAGS)))
 
-$(SOURCE_DIST).tar.bz2: $(SOURCE_DIST).manifest
-	$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
-		$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
-		$(BZIP2) > $@
+SOURCE_BUNDLE_BASE ?= rabbitmq-server-bundle
+SOURCE_BUNDLE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_BUNDLE_BASE)-$(PROJECT_VERSION)
+SOURCE_BUNDLE_FILES = $(addprefix $(SOURCE_BUNDLE_DIST).,$(DIST_SUFFIXES))
 
-$(SOURCE_DIST).tar.xz: $(SOURCE_DIST).manifest
-	$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
-		$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
-		$(XZ) > $@
+.PHONY: source-bundle
+source-bundle: $(SOURCE_BUNDLE_FILES)
+	@:
 
-$(SOURCE_DIST).zip: $(SOURCE_DIST).manifest
-	$(verbose) rm -f $@
-	$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
-		$(ZIP) $(ZIP_V) --names-stdin $@ < $(SOURCE_DIST).manifest
+$(eval $(call create_dist_target,$(SOURCE_BUNDLE_DIST),$(SOURCE_BUNDLE_RSYNC_FLAGS)))
 
-clean:: clean-source-dist
+# Create the clean targets for both distributions
+$(eval $(call create_clean_targets,$(SOURCE_DIST_BASE)))
+$(eval $(call create_clean_targets,$(SOURCE_BUNDLE_BASE)))
 
-clean-source-dist:
-	$(gen_verbose) rm -rf -- $(SOURCE_DIST_BASE)-*
+.PHONY: distclean-packages clean-unpacked-source-dist
 
 distclean:: distclean-packages