[lipo] Support creating Universal 64 bit Mach-O files. (#67737)

drodriguez · web-flow · commit efb46515ce9b · 2023-09-30T15:25:41.000-07:00
Xcode `lipo` seems to support a non-documented `-fat64` option that creates Universal Mach-O archives using 64 bit versions of the `fat_arch` header, which allows offsets larger than 32 bits to be specified. Modify `llvm-lipo` to support the same flag, and use the value of the flag to use either 32 bits or 64 bits Mach-O headers. The Mach-O universal writer allows specifying a new option to write these 64 bits headers. The default is still using 32 bits. `dsymutil` implemented support for a similar flag in https://reviews.llvm.org/D146879.
diff --git a/llvm/include/llvm/Object/MachOUniversalWriter.h b/llvm/include/llvm/Object/MachOUniversalWriter.h
@@ -97,9 +97,14 @@ class Slice {
   }
 };
 
-Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName);
+enum class FatHeaderType { FatHeader, Fat64Header };
 
-Error writeUniversalBinaryToStream(ArrayRef<Slice> Slices, raw_ostream &Out);
+Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName,
+                           FatHeaderType FatHeader = FatHeaderType::FatHeader);
+
+Error writeUniversalBinaryToStream(
+    ArrayRef<Slice> Slices, raw_ostream &Out,
+    FatHeaderType FatHeader = FatHeaderType::FatHeader);
 
 } // end namespace object
 
diff --git a/llvm/lib/Object/MachOUniversalWriter.cpp b/llvm/lib/Object/MachOUniversalWriter.cpp
@@ -240,25 +240,48 @@ Expected<Slice> Slice::create(const IRObjectFile &IRO, uint32_t Align) {
   return Slice{IRO, CPUType, CPUSubType, std::move(ArchName), Align};
 }
 
-static Expected<SmallVector<MachO::fat_arch, 2>>
+template <typename FatArchTy> struct FatArchTraits {
+  static const uint64_t OffsetLimit;
+  static const std::string StructName;
+  static const uint8_t BitCount;
+};
+
+template <> struct FatArchTraits<MachO::fat_arch> {
+  static const uint64_t OffsetLimit = UINT32_MAX;
+  static const std::string StructName;
+  static const uint8_t BitCount = 32;
+};
+const std::string FatArchTraits<MachO::fat_arch>::StructName = "fat_arch";
+
+template <> struct FatArchTraits<MachO::fat_arch_64> {
+  static const uint64_t OffsetLimit = UINT64_MAX;
+  static const std::string StructName;
+  static const uint8_t BitCount = 64;
+};
+const std::string FatArchTraits<MachO::fat_arch_64>::StructName = "fat_arch_64";
+
+template <typename FatArchTy>
+static Expected<SmallVector<FatArchTy, 2>>
 buildFatArchList(ArrayRef<Slice> Slices) {
-  SmallVector<MachO::fat_arch, 2> FatArchList;
+  SmallVector<FatArchTy, 2> FatArchList;
   uint64_t Offset =
-      sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
+      sizeof(MachO::fat_header) + Slices.size() * sizeof(FatArchTy);
 
   for (const auto &S : Slices) {
     Offset = alignTo(Offset, 1ull << S.getP2Alignment());
-    if (Offset > UINT32_MAX)
+    if (Offset > FatArchTraits<FatArchTy>::OffsetLimit)
       return createStringError(
           std::errc::invalid_argument,
-          ("fat file too large to be created because the offset "
-           "field in struct fat_arch is only 32-bits and the offset " +
+          ("fat file too large to be created because the offset field in the "
+           "struct " +
+           Twine(FatArchTraits<FatArchTy>::StructName) + " is only " +
+           Twine(FatArchTraits<FatArchTy>::BitCount) + "-bits and the offset " +
            Twine(Offset) + " for " + S.getBinary()->getFileName() +
            " for architecture " + S.getArchString() + "exceeds that.")
               .str()
               .c_str());
 
-    MachO::fat_arch FatArch;
+    FatArchTy FatArch;
     FatArch.cputype = S.getCPUType();
     FatArch.cpusubtype = S.getCPUSubType();
     FatArch.offset = Offset;
@@ -270,35 +293,33 @@ buildFatArchList(ArrayRef<Slice> Slices) {
   return FatArchList;
 }
 
-Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
-                                           raw_ostream &Out) {
-  MachO::fat_header FatHeader;
-  FatHeader.magic = MachO::FAT_MAGIC;
-  FatHeader.nfat_arch = Slices.size();
-
-  Expected<SmallVector<MachO::fat_arch, 2>> FatArchListOrErr =
-      buildFatArchList(Slices);
+template <typename FatArchTy>
+static Error writeUniversalArchsToStream(MachO::fat_header FatHeader,
+                                         ArrayRef<Slice> Slices,
+                                         raw_ostream &Out) {
+  Expected<SmallVector<FatArchTy, 2>> FatArchListOrErr =
+      buildFatArchList<FatArchTy>(Slices);
   if (!FatArchListOrErr)
     return FatArchListOrErr.takeError();
-  SmallVector<MachO::fat_arch, 2> FatArchList = *FatArchListOrErr;
+  SmallVector<FatArchTy, 2> FatArchList = *FatArchListOrErr;
 
   if (sys::IsLittleEndianHost)
     MachO::swapStruct(FatHeader);
   Out.write(reinterpret_cast<const char *>(&FatHeader),
             sizeof(MachO::fat_header));
 
   if (sys::IsLittleEndianHost)
-    for (MachO::fat_arch &FA : FatArchList)
+    for (FatArchTy &FA : FatArchList)
       MachO::swapStruct(FA);
   Out.write(reinterpret_cast<const char *>(FatArchList.data()),
-            sizeof(MachO::fat_arch) * FatArchList.size());
+            sizeof(FatArchTy) * FatArchList.size());
 
   if (sys::IsLittleEndianHost)
-    for (MachO::fat_arch &FA : FatArchList)
+    for (FatArchTy &FA : FatArchList)
       MachO::swapStruct(FA);
 
   size_t Offset =
-      sizeof(MachO::fat_header) + sizeof(MachO::fat_arch) * FatArchList.size();
+      sizeof(MachO::fat_header) + sizeof(FatArchTy) * FatArchList.size();
   for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
     MemoryBufferRef BufferRef = Slices[Index].getBinary()->getMemoryBufferRef();
     assert((Offset <= FatArchList[Index].offset) && "Incorrect slice offset");
@@ -311,8 +332,30 @@ Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
   return Error::success();
 }
 
+Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
+                                           raw_ostream &Out,
+                                           FatHeaderType HeaderType) {
+  MachO::fat_header FatHeader;
+  FatHeader.nfat_arch = Slices.size();
+
+  switch (HeaderType) {
+  case FatHeaderType::Fat64Header:
+    FatHeader.magic = MachO::FAT_MAGIC_64;
+    return writeUniversalArchsToStream<MachO::fat_arch_64>(FatHeader, Slices,
+                                                           Out);
+    break;
+  case FatHeaderType::FatHeader:
+    FatHeader.magic = MachO::FAT_MAGIC;
+    return writeUniversalArchsToStream<MachO::fat_arch>(FatHeader, Slices, Out);
+    break;
+  default:
+    llvm_unreachable("Invalid fat header type");
+  }
+}
+
 Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
-                                   StringRef OutputFileName) {
+                                   StringRef OutputFileName,
+                                   FatHeaderType HeaderType) {
   const bool IsExecutable = any_of(Slices, [](Slice S) {
     return sys::fs::can_execute(S.getBinary()->getFileName());
   });
@@ -324,7 +367,7 @@ Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
   if (!Temp)
     return Temp.takeError();
   raw_fd_ostream Out(Temp->FD, false);
-  if (Error E = writeUniversalBinaryToStream(Slices, Out)) {
+  if (Error E = writeUniversalBinaryToStream(Slices, Out, HeaderType)) {
     if (Error DiscardError = Temp->discard())
       return joinErrors(std::move(E), std::move(DiscardError));
     return E;
diff --git a/llvm/test/tools/llvm-lipo/create-fat64.test b/llvm/test/tools/llvm-lipo/create-fat64.test
@@ -0,0 +1,11 @@
+# RUN: yaml2obj %p/Inputs/i386-slice.yaml -o %t-i386.o
+# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml -o %t-x86_64.o
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal-32.o
+# RUN: llvm-objdump -m --universal-headers %t-universal-32.o | FileCheck %s -check-prefixes=FAT32
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -fat64 -output %t-universal-64.o
+# RUN: llvm-objdump -m --universal-headers %t-universal-64.o | FileCheck %s -check-prefixes=FAT64
+
+FAT32: fat_magic FAT_MAGIC
+FAT64: fat_magic FAT_MAGIC_64
diff --git a/llvm/tools/llvm-lipo/LipoOpts.td b/llvm/tools/llvm-lipo/LipoOpts.td
@@ -58,3 +58,6 @@ def replace
 def output : Option<["-", "--"], "output", KIND_SEPARATE>,
              HelpText<"Create output file with specified name">;
 def o : JoinedOrSeparate<["-"], "o">, Alias<output>;
+
+def fat64 : Option<["-", "--"], "fat64", KIND_FLAG>,
+            HelpText<"Use 64 bits Universal Mach-O format">;
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -116,6 +116,7 @@ struct Config {
   std::string ArchType;
   std::string OutputFile;
   LipoAction ActionToPerform;
+  bool UseFat64;
 };
 
 static Slice createSliceFromArchive(LLVMContext &LLVMCtx, const Archive &A) {
@@ -223,6 +224,8 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
                   Twine(AlignmentValue));
   }
 
+  C.UseFat64 = InputArgs.hasArg(LIPO_fat64);
+
   SmallVector<opt::Arg *, 1> ActionArgs(InputArgs.filtered(LIPO_action_group));
   if (ActionArgs.empty())
     reportError("at least one action should be specified");
@@ -596,9 +599,11 @@ buildSlices(LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
   return Slices;
 }
 
-[[noreturn]] static void createUniversalBinary(
-    LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
-    const StringMap<const uint32_t> &Alignments, StringRef OutputFileName) {
+[[noreturn]] static void
+createUniversalBinary(LLVMContext &LLVMCtx,
+                      ArrayRef<OwningBinary<Binary>> InputBinaries,
+                      const StringMap<const uint32_t> &Alignments,
+                      StringRef OutputFileName, FatHeaderType HeaderType) {
   assert(InputBinaries.size() >= 1 && "Incorrect number of input binaries");
   assert(!OutputFileName.empty() && "Create expects a single output file");
 
@@ -609,7 +614,7 @@ buildSlices(LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
   checkUnusedAlignments(Slices, Alignments);
 
   llvm::stable_sort(Slices);
-  if (Error E = writeUniversalBinary(Slices, OutputFileName))
+  if (Error E = writeUniversalBinary(Slices, OutputFileName, HeaderType))
     reportError(std::move(E));
 
   exit(EXIT_SUCCESS);
@@ -747,8 +752,9 @@ int llvm_lipo_main(int argc, char **argv, const llvm::ToolContext &) {
                  C.OutputFile);
     break;
   case LipoAction::CreateUniversal:
-    createUniversalBinary(LLVMCtx, InputBinaries, C.SegmentAlignments,
-                          C.OutputFile);
+    createUniversalBinary(
+        LLVMCtx, InputBinaries, C.SegmentAlignments, C.OutputFile,
+        C.UseFat64 ? FatHeaderType::Fat64Header : FatHeaderType::FatHeader);
     break;
   case LipoAction::ReplaceArch:
     replaceSlices(LLVMCtx, InputBinaries, C.SegmentAlignments, C.OutputFile,
