Skip to content

(Updated) Adding a "source-bundle" target to the Makefile (backport #13471) #13473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 8, 2025
Merged

(Updated) Adding a "source-bundle" target to the Makefile (backport #13471) #13473

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e675c97
Adding a "source-bundle" target that largely duplicates the "source-d…
kartg Mar 7, 2025
ca0e667
Refactor "source-dist" and "source-bundle" targets to reduce duplication
kartg Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
261 changes: 149 additions & 112 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,30 +134,17 @@ define restore_hex_cache_from_erl_term
endef

# --------------------------------------------------------------------
# Distribution.
# Distribution - common variables and generic functions.
# --------------------------------------------------------------------

.PHONY: source-dist clean-source-dist

SOURCE_DIST_BASE ?= rabbitmq-server
SOURCE_DIST_SUFFIXES ?= tar.xz
SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION)

# The first source distribution file is used by packages: if the archive
# type changes, you must update all packages' Makefile.
SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(SOURCE_DIST_SUFFIXES))

.PHONY: $(SOURCE_DIST_FILES)

source-dist: $(SOURCE_DIST_FILES)
@:

RSYNC ?= rsync
RSYNC_V_0 =
RSYNC_V_1 = -v
RSYNC_V_2 = -v
RSYNC_V = $(RSYNC_V_$(V))
RSYNC_FLAGS += -a $(RSYNC_V) \
BASE_RSYNC_FLAGS += -a $(RSYNC_V) \
--delete \
--delete-excluded \
--exclude '.sw?' --exclude '.*.sw?' \
--exclude '*.beam' \
--exclude '*.d' \
Expand Down Expand Up @@ -188,12 +175,10 @@ RSYNC_FLAGS += -a $(RSYNC_V) \
--exclude '$(notdir $(DEPS_DIR))/' \
--exclude 'hexer*' \
--exclude 'logs/' \
--exclude 'packaging' \
--exclude 'PKG_*.md' \
--exclude '/plugins/' \
--include 'cli/plugins' \
--exclude '$(notdir $(DIST_DIR))/' \
--exclude 'test' \
--exclude '/$(notdir $(PACKAGES_DIR))/' \
--exclude '/PACKAGES/' \
--exclude '/amqp_client/doc/' \
Expand All @@ -208,9 +193,21 @@ RSYNC_FLAGS += -a $(RSYNC_V) \
--exclude '/ranch/doc/' \
--exclude '/ranch/examples/' \
--exclude '/sockjs/examples/' \
--exclude '/workflow_sources/' \
--delete \
--delete-excluded
--exclude '/workflow_sources/'

SOURCE_DIST_RSYNC_FLAGS += $(BASE_RSYNC_FLAGS) \
--exclude 'packaging' \
--exclude 'test'

# For source-bundle, explicitly include folders that are needed
# for tests to execute. These are added before excludes from
# the base flags so rsync honors the first match.
SOURCE_BUNDLE_RSYNC_FLAGS += \
--include 'rabbit_shovel_test/ebin' \
--include 'rabbit_shovel_test/ebin/*' \
--include 'rabbitmq_ct_helpers/tools' \
--include 'rabbitmq_ct_helpers/tools/*' \
$(BASE_RSYNC_FLAGS)

TAR ?= tar
TAR_V_0 =
Expand All @@ -228,65 +225,125 @@ ZIP_V_1 =
ZIP_V_2 =
ZIP_V = $(ZIP_V_$(V))

.PHONY: $(SOURCE_DIST)
.PHONY: clean-source-dist distclean-packages clean-unpacked-source-dist

$(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
$(verbose) mkdir -p $(dir $@)
$(gen_verbose) $(RSYNC) $(RSYNC_FLAGS) ./ $@/
$(verbose) echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > "$@/git-revisions.txt"
$(verbose) echo "$(PROJECT) $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> "$@/git-revisions.txt"
$(verbose) echo "$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > "[email protected]"
$(verbose) cat packaging/common/LICENSE.head > $@/LICENSE
$(verbose) mkdir -p $@/deps/licensing
$(verbose) set -e; for dep in $$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \
$(RSYNC) $(RSYNC_FLAGS) \
$$dep \
$@/deps; \
ifeq ($(shell tar --version | grep -c "GNU tar"),0)
# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X)
ifeq ($(shell uname | grep -c "Darwin"),0)
TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \
--gid 0 \
--numeric-owner \
--no-acls \
--no-fflags \
--no-xattrs
endif
else
TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \
--group 0 \
--numeric-owner
endif

DIST_SUFFIXES ?= tar.xz

# Function to create distribution targets
# Args: $(1) - Full distribution path
# $(2) - RSYNC flags to use
define create_dist_target
$(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
$${verbose} mkdir -p $$(dir $$@)
$${gen_verbose} $${RSYNC} $(2) ./ $$@/
$${verbose} echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > $$@/git-revisions.txt
$${verbose} echo "$(PROJECT) $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> $$@/git-revisions.txt
$${verbose} echo "$$$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > [email protected]
$${verbose} cat packaging/common/LICENSE.head > $$@/LICENSE
$${verbose} mkdir -p $$@/deps/licensing
$${verbose} set -e; for dep in $$$$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \
$${RSYNC} $(2) \
$$$$dep \
$$@/deps; \
rm -f \
$@/deps/rabbit_common/rebar.config \
$@/deps/rabbit_common/rebar.lock; \
if test -f $@/deps/$$(basename $$dep)/erlang.mk && \
test "$$(wc -l $@/deps/$$(basename $$dep)/erlang.mk | awk '{print $$1;}')" = "1" && \
grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$" $@/deps/$$(basename $$dep)/erlang.mk; then \
echo "include ../../erlang.mk" > $@/deps/$$(basename $$dep)/erlang.mk; \
$$@/deps/rabbit_common/rebar.config \
$$@/deps/rabbit_common/rebar.lock; \
if test -f $$@/deps/$$$$(basename $$$$dep)/erlang.mk && \
test "$$$$(wc -l $$@/deps/$$$$(basename $$$$dep)/erlang.mk | awk '{print $$$$1;}')" = "1" && \
grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$$$" $$@/deps/$$$$(basename $$$$dep)/erlang.mk; then \
echo "include ../../erlang.mk" > $$@/deps/$$$$(basename $$$$dep)/erlang.mk; \
fi; \
sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$|include ../../erlang.mk|" \
$@/deps/$$(basename $$dep)/Makefile && \
rm $@/deps/$$(basename $$dep)/Makefile.bak; \
mix_exs=$@/deps/$$(basename $$dep)/mix.exs; \
if test -f $$mix_exs; then \
(cd $$(dirname "$$mix_exs") && \
(test -d $@/deps/.hex || env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$$$|include ../../erlang.mk|" \
$$@/deps/$$$$(basename $$$$dep)/Makefile && \
rm $$@/deps/$$$$(basename $$$$dep)/Makefile.bak; \
mix_exs=$$@/deps/$$$$(basename $$$$dep)/mix.exs; \
if test -f $$$$mix_exs; then \
(cd $$$$(dirname "$$$$mix_exs") && \
(test -d $$@/deps/.hex || env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
cp $(CURDIR)/mk/rabbitmq-mix.mk . && \
rm -rf _build deps); \
fi; \
if test -f "$$dep/license_info"; then \
cp "$$dep/license_info" "$@/deps/licensing/license_info_$$(basename "$$dep")"; \
cat "$$dep/license_info" >> $@/LICENSE; \
if test -f "$$$$dep/license_info"; then \
cp "$$$$dep/license_info" "$$@/deps/licensing/license_info_$$$$(basename $$$$dep)"; \
cat "$$$$dep/license_info" >> $$@/LICENSE; \
fi; \
find "$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $@/deps/licensing \; ; \
(cd $$dep; \
echo "$$(basename "$$dep") $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \
>> "$@/git-revisions.txt"; \
! test -d $$dep/.git || (cd $$dep; \
echo "$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \
>> "[email protected]"; \
find "$$$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $$@/deps/licensing \; ; \
(cd $$$$dep; \
echo "$$$$(basename "$$$$dep") $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \
>> "$$@/git-revisions.txt"; \
! test -d $$$$dep/.git || (cd $$$$dep; \
echo "$$$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \
>> "$$@.git-times.txt"; \
done
$(verbose) cat packaging/common/LICENSE.tail >> $@/LICENSE
$(verbose) find $@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $@ \;
$(verbose) rm -rf $@/deps/licensing
$(verbose) for file in $$(find $@ -name '*.app.src'); do \
$${verbose} cat packaging/common/LICENSE.tail >> $$@/LICENSE
$${verbose} find $$@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $$@ \;
$${verbose} rm -rf $$@/deps/licensing
$${verbose} for file in $$$$(find $$@ -name '*.app.src'); do \
sed -E -i.bak \
-e 's/[{]vsn[[:blank:]]*,[[:blank:]]*(""|"0.0.0")[[:blank:]]*}/{vsn, "$(PROJECT_VERSION)"}/' \
-e 's/[{]broker_version_requirements[[:blank:]]*,[[:blank:]]*\[\][[:blank:]]*}/{broker_version_requirements, ["$(PROJECT_VERSION)"]}/' \
$$file; \
rm $$file.bak; \
$$$$file; \
rm $$$$file.bak; \
done
$(verbose) echo "PLUGINS := $(PLUGINS)" > $@/plugins.mk
# Remember the latest Git timestamp.
$(verbose) sort -r < "[email protected]" | head -n 1 > "[email protected]"
$${verbose} echo "PLUGINS := $(PLUGINS)" > $$@/plugins.mk
$${verbose} sort -r < "[email protected]" | head -n 1 > "[email protected]"
$${verbose} $$(call erlang,$$(call dump_hex_cache_to_erl_term,$$(call core_native_path,$$@),$$(call core_native_path,[email protected])))
$${verbose} find $$@ -print0 | xargs -0 touch -t "$$$$(cat [email protected])"
$${verbose} rm "[email protected]" "[email protected]"

$(1).manifest: $(1)
$${gen_verbose} cd $$(dir $$@) && \
find $$(notdir $$<) | LC_COLLATE=C sort > $$@

$(1).tar.xz: $(1).manifest
$${gen_verbose} cd $$(dir $$@) && \
$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
$${XZ} > $$@

$(1).tar.gz: $(1).manifest
$${gen_verbose} cd $$(dir $$@) && \
$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
$${GZIP} --best > $$@

$(1).tar.bz2: $(1).manifest
$${gen_verbose} cd $$(dir $$@) && \
$${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \
$${BZIP2} > $$@

$(1).zip: $(1).manifest
$${verbose} rm -f $$@
$${gen_verbose} cd $$(dir $$@) && \
$${ZIP} $${ZIP_V} --names-stdin $$@ < $$(notdir $$<)

endef

# Function to create clean targets
# Args: $(1) - Base name (e.g. SOURCE_DIST_BASE or BUNDLE_DIST_BASE)
define create_clean_targets
.PHONY: clean-$(1)

clean-$(1):
$${gen_verbose} rm -rf -- $(1)-*

# Add each clean target to the clean:: rule
clean:: clean-$(1)
endef

# Mix Hex component requires a cache file, otherwise it refuses to build
# offline... That cache is an ETS table with all the applications we
# depend on, plus some versioning informations and checksums. There
Expand All @@ -300,11 +357,6 @@ $(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
#
# The ETS file must be recreated before compiling RabbitMQ. See the
# `restore-hex-cache-ets-file` Make target.
$(verbose) $(call erlang,$(call dump_hex_cache_to_erl_term,$(call core_native_path,$@),$(call core_native_path,[email protected])))
# Fix file timestamps to have reproducible source archives.
$(verbose) find $@ -print0 | xargs -0 touch -t "$$(cat "[email protected]")"
$(verbose) rm "[email protected]" "[email protected]"

define dump_hex_cache_to_erl_term
In = "$(1)/deps/.hex/cache.ets",
Out = "$(1)/deps/.hex/cache.erl",
Expand Down Expand Up @@ -333,50 +385,35 @@ define dump_hex_cache_to_erl_term
init:stop().
endef

$(SOURCE_DIST).manifest: $(SOURCE_DIST)
$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
find $(notdir $(SOURCE_DIST)) | LC_COLLATE=C sort > $@
# --------------------------------------------------------------------
# Distribution - public targets
# --------------------------------------------------------------------

ifeq ($(shell tar --version | grep -c "GNU tar"),0)
# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X)
ifeq ($(shell uname | grep -c "Darwin"),0)
TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \
--gid 0 \
--numeric-owner \
--no-acls \
--no-fflags \
--no-xattrs
endif
else
TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \
--group 0 \
--numeric-owner
endif
SOURCE_DIST_BASE ?= rabbitmq-server
SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION)
SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(DIST_SUFFIXES))

$(SOURCE_DIST).tar.gz: $(SOURCE_DIST).manifest
$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
$(GZIP) --best > $@
.PHONY: source-dist
source-dist: $(SOURCE_DIST_FILES)
@:

$(eval $(call create_dist_target,$(SOURCE_DIST),$(SOURCE_DIST_RSYNC_FLAGS)))

$(SOURCE_DIST).tar.bz2: $(SOURCE_DIST).manifest
$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
$(BZIP2) > $@
SOURCE_BUNDLE_BASE ?= rabbitmq-server-bundle
SOURCE_BUNDLE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_BUNDLE_BASE)-$(PROJECT_VERSION)
SOURCE_BUNDLE_FILES = $(addprefix $(SOURCE_BUNDLE_DIST).,$(DIST_SUFFIXES))

$(SOURCE_DIST).tar.xz: $(SOURCE_DIST).manifest
$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
$(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \
$(XZ) > $@
.PHONY: source-bundle
source-bundle: $(SOURCE_BUNDLE_FILES)
@:

$(SOURCE_DIST).zip: $(SOURCE_DIST).manifest
$(verbose) rm -f $@
$(gen_verbose) cd $(dir $(SOURCE_DIST)) && \
$(ZIP) $(ZIP_V) --names-stdin $@ < $(SOURCE_DIST).manifest
$(eval $(call create_dist_target,$(SOURCE_BUNDLE_DIST),$(SOURCE_BUNDLE_RSYNC_FLAGS)))

clean:: clean-source-dist
# Create the clean targets for both distributions
$(eval $(call create_clean_targets,$(SOURCE_DIST_BASE)))
$(eval $(call create_clean_targets,$(SOURCE_BUNDLE_BASE)))

clean-source-dist:
$(gen_verbose) rm -rf -- $(SOURCE_DIST_BASE)-*
.PHONY: distclean-packages clean-unpacked-source-dist

distclean:: distclean-packages

Expand Down