ctf: generate CTF information for the kernel

This introduces a new tool, dwarf2ctf, which runs whenever 'make ctf' is
run, extracting information on the kernel's types and global variables
from the DWARF-format debug information in the kernel build tree,
deduplicating it, and emitting it in Sun's Compact Type Format into a
mmappable type archive named vmlinux.ctfa, which is installed at 'make
install' time into /lib/modules/$(uname -r)/kernel/.  Out-of-tree
modules cannot participate in this mechanism since the file is already
written: CTF information for such modules is instead linked into such
modules at build time as new sections named .SUNW_ctf (as a result, most
of the build-time machinery for this is in scripts/Makefile.modpost).
Care should be taken not to strip such sections into debug RPMs (they
are small enough that this should not be a problem).

Within the ctfa file, the type information is divided into a shared
repository, containing all types used by more than one module, CTF for
the core kernel, and separate CTF for each module built, whether or not
this module has been compiled in or not: if a file *could* be built as a
module, it will be considered to be a module from the perspective of CTF
file emission (and kallmodsyms: see the next commit).  This ensures that
external consumers such as DTrace always find types for a given module
in the same place, regardless of the local kernel configuration, as long
as that module is present at all, assisting in portability of D scripts
between installations.  The ctf_ar tool in libdtrace-ctf can be used to
inspect ctfa files, and the ctf_dump tool can be used to look at the ctf
files they contain.

This process needs a pair of new files, objects.builtin (which lists all
object files that are unconditionally built into the kernel and cannot
be built as modules) and modules_thick.builtin, which maps from the thin
archives that make up built-in modules to their constituent object
files.  Taken together, these files let dwarf2ctf determine whether a
given object file linked into vmlinux.o is part of a module, and if so,
which one.

There is a single manually-maintained blacklist of structure members
dwarf2ctf cannot handle in scripts/dwarf2ctf/member.blacklist: this is
used to identify structure members which have different definitions in
different object files even though they are defined in the same location
in the same source file, usually due to preprocessor magic.  (Currently,
the only item in this list is present for example purposes only, since
the file in question was recently removed from the kernel: dwarf2ctf can
these days identify most members needing blacklisting automatically, and
will fail with an error if it needs more help.  It is quite possible that
dwarf2ctf will fail on make allyesconfig kernel configurations and other
extreme cases: I hope to track all such bugs down in time.)

The documentation for dwarf2ctf is currently somewhat outdated: an
update is planned.  It remains largely accurate except for some details
of the deduplication pass.

This introduces new kernel build-time dependencies on elfutils, zlib,
glib, and the new libdtrace-ctf package (shared with DTrace userspace).
No new runtime dependencies are introduced.

Orabug: 30544408
Signed-off-by: Nick Alcock <[email protected]>
Signed-off-by: Tomas Jedlicka <[email protected]>
Reviewed-by: Kris Van Hees <[email protected]>

Author: nickalcock

.gitignore

@@ -48,6 +48,7 @@ Module.symvers
 modules.builtin
 modules.order
 modules_thick.builtin
+objects.builtin
 
 #
 # Top-level generic files
@@ -57,6 +58,7 @@ modules_thick.builtin
 /linux
 /vmlinux
 /vmlinux.32
+/vmlinux.ctfa
 /vmlinux-gdb.py
 /vmlinuz
 /System.map

Documentation/dontdiff

@@ -179,6 +179,7 @@ mkutf8data
 modpost
 modules.builtin
 modules.builtin.modinfo
+modules_thick.builtin
 modules.order
 modules_thick.builtin
 modversions.h*

Documentation/dwarf2ctf

@@ -55,9 +55,14 @@ iptables               1.4.2            iptables -V
 openssl & libcrypto    1.0.0            openssl version
 bc                     1.06.95          bc --version
 Sphinx\ [#f1]_	       1.3		sphinx-build --version
+elfutils\ [#f2]_       0.156		eu-readelf --version
+pkg-config\ [#f2]_     0.16		pkg-config --version
+glib\ [#f2]_           2.x		pkg-config --exists glib-2.0 && echo present
+libdtrace-ctf\ [#f2]_  1.1
 ====================== ===============  ========================================
 
 .. [#f1] Sphinx is needed only to build the Kernel documentation
+.. [#f2] This is needed at build-time when CTF or DTrace are enabled
 
 Kernel compilation
 ******************
@@ -84,7 +89,8 @@ pkg-config
 The build system, as of 4.18, requires pkg-config to check for installed
 kconfig tools and to determine flags settings for use in
 'make {g,x}config'.  Previously pkg-config was being used but not
-verified or documented.
+verified or documented.  dwarf2ctf also relies on it during 'make ctf' and
+while building out-of-tree modules with CONFIG_CTF enabled.
 
 Flex
 ----
@@ -356,6 +362,21 @@ OpenSSL
 
 - <https://www.openssl.org/>
 
+elfutils
+--------
+
+- <https://fedorahosted.org/elfutils/>
+
+glib 2.x
+--------
+
+- <http://www.gtk.org/>
+
+libdtrace-ctf
+-------------
+
+- <https://oss.oracle.com/git/?p=libdtrace-ctf.git>
+
 System utilities
 ****************
 

Documentation/process/changes.rst

@@ -1295,6 +1295,45 @@ modules: $(if $(KBUILD_BUILTIN),vmlinux) modules.order modules.builtin
 modules.order: descend
 	$(Q)$(AWK) '!x[$$0]++' $(addsuffix /$@, $(build-dirs)) > $@
 
+ifneq (CONFIG_CTF@,'@')
+
+# We need to force everything to be built, since we need the .o files below.
+KBUILD_BUILTIN := 1
+
+# This contains all the object files that are built directly into the
+# kernel (including built-in modules), for consumption by dwarf2ctf in
+# Makefile.modpost.
+# This is made doubly annoying by the presence of '.o' files which are actually
+# thin ar archives, and the need to support file(1) versions too old to
+# recognize them as archives at all.  (So we assume that everything that is not
+# an ELF object is an archive.)
+ifeq ($(SRCARCH),x86)
+objects.builtin: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),bzImage) FORCE
+else
+objects.builtin: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),vmlinux) FORCE
+endif
+	@echo $(KBUILD_VMLINUX_OBJS) | \
+		tr " " "\n" | grep "\.o$$" | xargs -r file | \
+		grep ELF | cut -d: -f1 > objects.builtin
+	@for archive in $$(echo $(KBUILD_VMLINUX_OBJS) |\
+		tr " " "\n" | xargs -r file | grep -v ELF | cut -d: -f1); do \
+		$(AR) t "$$archive" >> objects.builtin; \
+	done
+
+ctf: vmlinux.ctfa
+PHONY += ctf
+
+# Making CTF needs the builtin files unless out-of-tree.
+ifeq ($(KBUILD_EXTMOD),)
+vmlinux.ctfa: modules_thick.builtin objects.builtin
+endif
+vmlinux.ctfa:
+	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal vmlinux.ctfa
+else
+PHONY += objects.builtin
+objects.builtin:
+endif
+
 # Target to prepare building external modules
 PHONY += modules_prepare
 modules_prepare: prepare
@@ -1316,6 +1355,9 @@ _modinst_:
 	@sed 's:^:kernel/:' modules.order > $(MODLIB)/modules.order
 	@sed 's:^:kernel/:' modules.builtin > $(MODLIB)/modules.builtin
 	@cp -f $(objtree)/modules.builtin.modinfo $(MODLIB)/
+	@if [ -f $(objtree)/vmlinux.ctfa ] ; then \
+		cp -f $(objtree)/vmlinux.ctfa $(MODLIB)/kernel ; \
+	fi
 	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modinst
 
 # This depmod is only for convenience to give the initial
@@ -1382,8 +1424,8 @@ $(modbuiltin-thick-dirs): include/config/tristate.conf
 # make distclean Remove editor backup files, patch leftover files and the like
 
 # Directories & files removed with 'make clean'
-CLEAN_DIRS  += include/ksym
-CLEAN_FILES += modules.builtin.modinfo
+CLEAN_DIRS  += include/ksym .ctf
+CLEAN_FILES += modules.builtin.modinfo .ctf.filelist .ctf.filelist.raw
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_DIRS  += include/config include/generated          \
@@ -1484,6 +1526,8 @@ help:
 	@echo  '                    (requires a recent binutils and recent build (System.map))'
 	@echo  '  dir/file.ko     - Build module including final link'
 	@echo  '  modules_prepare - Set up for building external modules'
+	@echo  '  ctf             - Generate CTF type information for DTrace, installed by '
+	@echo  '                    make modules_install'
 	@echo  '  tags/TAGS	  - Generate tags file for editors'
 	@echo  '  cscope	  - Generate cscope index'
 	@echo  '  gtags           - Generate GNU GLOBAL index'
@@ -1693,9 +1737,9 @@ clean: $(clean-dirs)
 		-o -name '*.asn1.[ch]' \
 		-o -name '*.symtypes' -o -name 'modules.order' \
 		-o -name modules.builtin -o -name '.tmp_*.o.*' \
-		-o -name modules_thick.builtin \
+		-o -name modules_thick.builtin -o -name objects.builtin \
 		-o -name '*.c.[012]*.*' \
-		-o -name '*.ll' \
+		-o -name '*.ll' -o -name '*.ctfa' \
 		-o -name '*.gcno' \) -type f -print | xargs rm -f
 
 # Generate tags for editors

Makefile

@@ -565,6 +565,18 @@ config DIMLIB
 #
 config LIBFDT
 	bool
+#
+# CTF support is select'ed if needed
+#
+config CTF
+        bool "Compact Type Format generation"
+        default y
+        select STRIP_ASM_SYMS
+        depends on DEBUG_INFO && !DEBUG_INFO_REDUCED && !DEBUG_INFO_SPLIT && !DEBUG_INFO_DWARF4
+        help
+          Emit a compact, compressed description of the kernel's datatypes and
+          global variables into the vmlinux.ctfa archive (for in-tree modules)
+          or into .ctf sections in kernel modules (for out-of-tree modules).
 
 config OID_REGISTRY
 	tristate

lib/Kconfig

@@ -33,6 +33,7 @@ hostprogs-y += unifdef
 
 subdir-$(CONFIG_GCC_PLUGINS) += gcc-plugins
 subdir-$(CONFIG_MODVERSIONS) += genksyms
+subdir-$(CONFIG_CTF)         += dwarf2ctf
 subdir-$(CONFIG_SECURITY_SELINUX) += selinux
 
 # Let clean descend into subdirs

scripts/Makefile

@@ -1,11 +1,21 @@
 # SPDX-License-Identifier: GPL-2.0-only
 # ===========================================================================
-# Module final link
+# Module final link and CTF generation
 # ===========================================================================
+# 1) compile all <module>.mod.c files
+# 2) for external modules, generate CTF for the module (there is an extra,
+#    externally-invoked target that does this for the entire kernel but does
+#    not invoke the rst of the module-building process)
+# 3) final link of the module to a <module.ko> file
+
+# We need secondary expansion for 'module-ctfs-modular-prereq', below.
+
+.SECONDEXPANSION:
 
 PHONY := __modfinal
 __modfinal:
 
+include include/config/auto.conf
 include $(srctree)/scripts/Kbuild.include
 
 # for c_flags
@@ -27,17 +37,142 @@ quiet_cmd_cc_o_c = CC [M]  $@
 %.mod.o: %.mod.c FORCE
 	$(call if_changed_dep,cc_o_c)
 
+# Generate CTF for the entire kernel, or for the module alone if this is a
+# build of an external module.
+
+# These are overridden below for standalone modules only.
+module-ctfs-modular-prereq =
+module-ctfs-modular =
+module-ctf-flags =
+cmd_touch_ctf =
+ctf-dir = ///.nonexistent
+cmd-touch-ctf = @:
+
+ifdef CONFIG_CTF
+
+# This is quite tricky.  If called for non-external-modules, dwarf2ctf needs to
+# be told about all the built-in objects as well as all the external modules --
+# but Makefile.modpost only knows about the latter.  So the toplevel makefile
+# emits the names of the built-in objects into a temporary file, which is
+# then catted and its contents used as prerequisites by this rule.
+#
+# We write the names of the object files to be scanned for CTF content into a
+# file, then use that, to avoid hitting command-line length limits.
+
+ifeq ($(KBUILD_EXTMOD),)
+ctf-dir-mk :=
+quiet_cmd_ctf = CTFA
+      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf vmlinux.ctfa $(srctree) objects.builtin modules_thick.builtin $(srctree)/scripts/dwarf2ctf/member.blacklist $(ctf-filelist)
+ctf-builtins := objects.builtin
+ctf-builtins-prereq := $(ctf-builtins)
+ ctf-modules := $(shell find . -name '*.ko' -print)
+ctf-filelist := .ctf.filelist
+ctf-filelist-raw := .ctf.filelist.raw
+ctf-stamp :=
+
+else
+ctf-dir := $(KBUILD_EXTMOD)/.ctf
+ctf-dir-mk := $(ctf-dir)
+quiet_cmd_ctf = CTF
+      cmd_ctf = scripts/dwarf2ctf/dwarf2ctf $(ctf-dir) -e $(ctf-filelist)
+ctf-builtins := ////.no-builtins
+ctf-builtins-prereq :=
+ctf-modules := $(modules:.ko=.o)
+ctf-filelist := $(ctf-dir)/$(notdir $(M)-extmod).ctf.filelist
+ctf-filelist-raw := $(ctf-dir)/$(notdir $(M)-extmod).ctf.filelist.raw
+ctf-stamp = $(ctf-dir)/$(notdir $(M)-extmod).stamp
+
+# All the modules' CTF depends on the stamp file.
+
+all-module-ctfs = $(addprefix $(ctf-dir)/,$(notdir $(modules:.ko=.mod.ctf)))
+$(all-module-ctfs): $(ctf-stamp)
+
+endif
+
+# Split a list up like shell xargs does.
+define xargs =
+$(1) $(wordlist 1,1024,$(2))
+$(if $(word 1025,$(2)),$(call xargs,$(1),$(wordlist 1025,$(words $(2)),$(2))))
+endef
+
+$(ctf-filelist-raw): $(ctf-builtins-prereq) $(ctf-modules)
+	@rm -f $(ctf-filelist-raw);
+	@if [[ -n "$(ctf-dir-mk)" ]]; then \
+		mkdir -p "$(ctf-dir-mk)"; \
+	fi
+	$(call xargs,@printf "%s\n" >> $(ctf-filelist-raw),$^)
+	@touch $(ctf-filelist-raw)
+
+$(ctf-filelist): $(ctf-filelist-raw)
+	@rm -f $(ctf-filelist);
+	@cat $(ctf-filelist-raw) | while read -r obj; do \
+		case $$obj in \
+		$(ctf-builtins)) cat $$obj >> $(ctf-filelist);; \
+		*.a) ar t $$obj > $(ctf-filelist);; \
+		*.builtin) cat $$obj >> $(ctf-filelist);; \
+		*) echo "$$obj" >> $(ctf-filelist);; \
+		esac; \
+	done
+	@touch $(ctf-filelist)
+
+ifeq ($(KBUILD_EXTMOD),)
+# The CTF depends on the output CTF file list, and that depends
+# on the .ko files for the modules.
+vmlinux.ctfa: $(ctf-filelist)
+	$(call if_changed,ctf)
+else
+
+# The CTF depends on the output CTF file list, and that depends
+# on the .o files for the modules
+$(ctf-stamp): $(ctf-filelist)
+	$(call if_changed,ctf)
+	@shopt -s nullglob; \
+	for name in $(ctf-dir)/*.ctf.new; do \
+		$(srctree)/scripts/move-if-change $$name $${name%.new}; \
+	done; \
+	touch $(ctf-stamp)
+
+# Expands to the names of the CTF files to be incorporated into this module.
+# The former is used in prerequisite lists, thanks to secondary expansion.
+
+module-ctfs-modular-prereq = $$(addprefix $(ctf-dir)/,$$(notdir $$*.mod.ctf))
+module-ctfs-modular = $(addprefix $(ctf-dir)/,$(notdir $*.mod.ctf))
+
+# Expands to the name of a CTF file, given a target of a module name given to
+# one of the link rules below.
+
+ctf-module-name = $(addprefix $(ctf-dir)/,$(notdir $(basename $@)).mod.ctf)
+
+# An objcopy --add-section argument to add the CTF section to a standalone
+# module.
+
+module-ctf-flags = --add-section .ctf=$(ctf-module-name)
+
+# We have to put content in our dummy no-CTF files because --add-section
+# in binutils 2.20 silently fails if asked to add an empty file as a section.
+
+cmd_touch_ctf = @for name in $(filter $(ctf-dir)/%,$(module-ctfs-modular)); do \
+		    test -f $$name || dd if=/dev/zero of=$$name bs=1 count=1 2>/dev/null; \
+		done
+
+endif		# KBUILD_EXTMOD
+
+endif		# !CONFIG_CTF
+
 ARCH_POSTLINK := $(wildcard $(srctree)/arch/$(SRCARCH)/Makefile.postlink)
 
 quiet_cmd_ld_ko_o = LD [M]  $@
       cmd_ld_ko_o =                                                     \
 	$(LD) -r $(KBUILD_LDFLAGS)					\
 		$(KBUILD_LDFLAGS_MODULE) $(LDFLAGS_MODULE)		\
 		$(addprefix -T , $(KBUILD_LDS_MODULE))			\
-		-o $@ $(filter %.o, $^);				\
+                 $(LDFLAGS_$(modname)) -o [email protected]            		\
+		-o [email protected] $(patsubst $(ctf-dir)/%,,$(filter %.o, $^)) && \
+        $(OBJCOPY) $(module-ctf-flags) [email protected] $@ && rm -f [email protected] ;	\
 	$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true)
 
-$(modules): %.ko: %.o %.mod.o $(KBUILD_LDS_MODULE) FORCE
+$(modules): %.ko: %.o %.mod.o $(KBUILD_LDS_MODULE) $(module-ctfs-modular-prereq) FORCE
+	$(call cmd_touch_ctf)
 	+$(call if_changed,ld_ko_o)
 
 targets += $(modules) $(modules:.ko=.mod.o)

scripts/Makefile.modfinal

@@ -0,0 +1 @@
+dwarf2ctf

scripts/dwarf2ctf/.gitignore

@@ -0,0 +1,11 @@
+ifdef CONFIG_CTF
+hostprogs-y	:= dwarf2ctf
+always		:= $(hostprogs-y)
+
+dwarf2ctf-objs  := dwarf2ctf.o eu_simple.o
+
+HOSTCFLAGS_eu_simple.o := -I$(srctree)/scripts
+HOSTCFLAGS_dwarf2ctf.o := $(shell pkg-config --cflags glib-2.0) -I$(srctree)/scripts
+
+HOSTLDLIBS_dwarf2ctf := -ldtrace-ctf -lelf -ldw $(shell pkg-config --libs glib-2.0) -lz
+endif