[IPO] Optimise variadic functions

clang/test/CodeGen/aarch64-ABI-align-packed.c

@@ -1,5 +1,6 @@
 // REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -triple aarch64 -target-feature +neon -emit-llvm -O2 -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +neon -emit-llvm -O2 -o - %s -mllvm -expand-variadics-override=disable | FileCheck %s
+
 #include <stdarg.h>
 #include <arm_neon.h>
 

clang/test/CodeGenCXX/inline-then-fold-variadics.cpp

@@ -0,0 +1,128 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature
+
+// Simple calls to known variadic functions that are completely elided when optimisations are on
+// This is a functional check that the expand-variadic pass is consistent with clang's va_arg handling
+
+// -Wno-varargs avoids warning second argument to 'va_start' is not the last named parameter
+
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -Wno-varargs -O1 -emit-llvm -o - %s |  FileCheck %s
+
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -Wno-varargs -O1 -emit-llvm -o - %s |  FileCheck %s
+
+// x64 needs O2 to remove the extra SROA layer
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -Wno-varargs -O2 -emit-llvm -o - %s |  FileCheck %s
+
+
+#include <stdarg.h>
+#include <stdint.h>
+
+template <typename X, typename Y>
+static X first(...) {
+  va_list va;
+  __builtin_va_start(va, 0);
+  X r = va_arg(va, X);
+  va_end(va);
+  return r;
+}
+
+template <typename X, typename Y>
+static Y second(...) {
+  va_list va;
+  __builtin_va_start(va, 0);
+  va_arg(va, X);
+  Y r = va_arg(va, Y);
+  va_end(va);
+  return r;
+}
+
+
+extern "C"
+{
+
+// CHECK-LABEL: define {{[^@]+}}@first_pair_i32
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], i32 noundef [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_pair_i32(int x, int y)
+{
+  return first<int,int>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@second_pair_i32
+// CHECK-SAME: (i32 noundef [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_pair_i32(int x, int y)
+{
+  return second<int,int>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@first_pair_f64
+// CHECK-SAME: (double noundef returned [[X:%.*]], double noundef [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret double [[X]]
+//
+double first_pair_f64(double x, double y)
+{
+  return first<double,double>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@second_pair_f64
+// CHECK-SAME: (double noundef [[X:%.*]], double noundef returned [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret double [[Y]]
+//
+double second_pair_f64(double x, double y)
+{
+  return second<double,double>(x, y);
+}
+
+}
+
+
+
+extern "C"
+{
+// CHECK-LABEL: define {{[^@]+}}@first_i32_f64
+// CHECK-SAME: (i32 noundef returned [[X:%.*]], double noundef [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret i32 [[X]]
+//
+int first_i32_f64(int x, double y)
+{
+  return first<int,double>(x, y);
+}
+
+
+// CHECK-LABEL: define {{[^@]+}}@second_i32_f64
+// CHECK-SAME: (i32 noundef [[X:%.*]], double noundef returned [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret double [[Y]]
+//
+double second_i32_f64(int x, double y)
+{
+  return second<int,double>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@first_f64_i32
+// CHECK-SAME: (double noundef returned [[X:%.*]], i32 noundef [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret double [[X]]
+//
+double first_f64_i32(double x, int y)
+{
+  return first<double,int>(x, y);
+}
+
+// CHECK-LABEL: define {{[^@]+}}@second_f64_i32
+// CHECK-SAME: (double noundef [[X:%.*]], i32 noundef returned [[Y:%.*]])
+// CHECK-LABEL:{{.}}:
+// CHECK-NEXT:    ret i32 [[Y]]
+//
+int second_f64_i32(double x, int y)
+{
+  return second<double,int>(x, y);
+}
+}

llvm/include/llvm/InitializePasses.h

@@ -106,6 +106,7 @@ void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&);
 void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
 void initializeExpandPostRAPass(PassRegistry&);
 void initializeExpandReductionsPass(PassRegistry&);
+void initializeExpandVariadicsPass(PassRegistry &);
 void initializeExpandVectorPredicationPass(PassRegistry &);
 void initializeExternalAAWrapperPassPass(PassRegistry&);
 void initializeFEntryInserterPass(PassRegistry&);

llvm/include/llvm/Transforms/IPO/ExpandVariadics.h

@@ -0,0 +1,43 @@
+//===- ExpandVariadics.h - expand variadic functions ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H
+#define LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Module;
+class ModulePass;
+class OptimizationLevel;
+
+enum class ExpandVariadicsMode {
+  Unspecified, // Use the implementation defaults
+  Disable,     // Disable the pass entirely
+  Optimize,    // Optimise without changing ABI
+  Lowering,    // Change variadic calling convention
+};
+
+class ExpandVariadicsPass : public PassInfoMixin<ExpandVariadicsPass> {
+  const ExpandVariadicsMode Mode;
+
+public:
+  // Operates under passed mode unless overridden on commandline
+  ExpandVariadicsPass(ExpandVariadicsMode Mode);
+
+  // Chooses disable or optimize based on optimization level
+  ExpandVariadicsPass(OptimizationLevel Level);
+
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+ModulePass *createExpandVariadicsPass(ExpandVariadicsMode);
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_EXPANDVARIADICS_H

llvm/lib/Passes/PassBuilder.cpp

@@ -137,6 +137,7 @@
 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
 #include "llvm/Transforms/IPO/EmbedBitcodePass.h"
+#include "llvm/Transforms/IPO/ExpandVariadics.h"
 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
 #include "llvm/Transforms/IPO/FunctionAttrs.h"
 #include "llvm/Transforms/IPO/FunctionImport.h"

llvm/lib/Passes/PassBuilderPipelines.cpp

@@ -48,6 +48,7 @@
 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
 #include "llvm/Transforms/IPO/EmbedBitcodePass.h"
+#include "llvm/Transforms/IPO/ExpandVariadics.h"
 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
 #include "llvm/Transforms/IPO/FunctionAttrs.h"
 #include "llvm/Transforms/IPO/GlobalDCE.h"
@@ -1195,6 +1196,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   if (EnablePGOForceFunctionAttrs && PGOOpt)
     MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
 
+  // ExpandVariadics interacts well with the function inliner.
+  MPM.addPass(ExpandVariadicsPass(Level));
+
   MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
 
   if (EnableModuleInliner)

llvm/lib/Passes/PassRegistry.def

@@ -59,6 +59,7 @@ MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass())
 MODULE_PASS("dxil-upgrade", DXILUpgradePass())
 MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass())
 MODULE_PASS("extract-blocks", BlockExtractorPass({}, false))
+MODULE_PASS("expand-variadics", ExpandVariadicsPass(OptimizationLevel::O0))
 MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("globalopt", GlobalOptPass())

llvm/lib/Transforms/IPO/CMakeLists.txt

@@ -12,6 +12,7 @@ add_llvm_component_library(LLVMipo
   DeadArgumentElimination.cpp
   ElimAvailExtern.cpp
   EmbedBitcodePass.cpp
+  ExpandVariadics.cpp
   ExtractGV.cpp
   ForceFunctionAttrs.cpp
   FunctionAttrs.cpp