Add a corefile style option to process save-core; skinny corefiles

Add a new feature to process save-core on Darwin systems -- for
lldb to create a user process corefile with only the dirty (modified
memory) pages included.  All of the binaries that were used in the
corefile are assumed to still exist on the system for the duration
of the use of the corefile.  A new --style option to process save-core
is added, so a full corefile can be requested if portability across
systems, or across time, is needed for this corefile.

debugserver can now identify the dirty pages in a memory region
when queried with qMemoryRegionInfo, and the size of vm pages is
given in qHostInfo.

Create a new "all image infos" LC_NOTE for Mach-O which allows us
to describe all of the binaries that were loaded in the process --
load address, UUID, file path, segment load addresses, and optionally
whether code from the binary was executing on any thread.  The old
"read dyld_all_image_infos and then the in-memory Mach-O load
commands to get segment load addresses" no longer works when we
only have dirty memory.

rdar://69670807
Differential Revision: https://reviews.llvm.org/D88387

Author: jasonmolenda

lldb/bindings/interface/SBMemoryRegionInfo.i

@@ -46,6 +46,42 @@ public:
     const char *
     GetName ();
 
+    %feature("autodoc", "
+        GetRegionEnd(SBMemoryRegionInfo self) -> lldb::addr_t
+        Returns whether this memory region has a list of modified (dirty)
+        pages available or not.  When calling GetNumDirtyPages(), you will
+        have 0 returned for both \"dirty page list is not known\" and 
+        \"empty dirty page list\" (that is, no modified pages in this
+        memory region).  You must use this method to disambiguate.") HasDirtyMemoryPageList;
+    bool 
+    HasDirtyMemoryPageList();
+
+    %feature("autodoc", "
+        GetNumDirtyPages(SBMemoryRegionInfo self) -> uint32_t
+        Return the number of dirty (modified) memory pages in this
+        memory region, if available.  You must use the 
+        SBMemoryRegionInfo::HasDirtyMemoryPageList() method to
+        determine if a dirty memory list is available; it will depend
+        on the target system can provide this information.") GetNumDirtyPages;
+    uint32_t 
+    GetNumDirtyPages();
+
+    %feature("autodoc", "
+        GetDirtyPageAddressAtIndex(SBMemoryRegionInfo self, uint32_t idx) -> lldb::addr_t
+        Return the address of a modified, or dirty, page of memory.
+        If the provided index is out of range, or this memory region 
+        does not have dirty page information, LLDB_INVALID_ADDRESS 
+        is returned.") GetDirtyPageAddressAtIndex;
+    addr_t 
+    GetDirtyPageAddressAtIndex(uint32_t idx);
+
+    %feature("autodoc", "
+        GetPageSize(SBMemoryRegionInfo self) -> int
+        Return the size of pages in this memory region.  0 will be returned
+        if this information was unavailable.") GetPageSize();
+    int
+    GetPageSize();
+
     bool
     operator == (const lldb::SBMemoryRegionInfo &rhs) const;
 

lldb/docs/lldb-gdb-remote.txt

@@ -869,10 +869,14 @@ distribution_id: optional. For linux, specifies distribution id (e.g. ubuntu, fe
 osmajor: optional, specifies the major version number of the OS (e.g. for macOS 10.12.2, it would be 10)
 osminor: optional, specifies the minor version number of the OS (e.g. for macOS 10.12.2, it would be 12)
 ospatch: optional, specifies the patch level number of the OS (e.g. for macOS 10.12.2, it would be 2)
+vm-page-size: optional, specifies the target system VM page size, base 10.
+           Needed for the "dirty-pages:" list in the qMemoryRegionInfo
+           packet, where a list of dirty pages is sent from the remote 
+           stub.  This page size tells lldb how large each dirty page is.
 addressing_bits: optional, specifies how many bits in addresses are
 		 significant for addressing, base 10.  If bits 38..0
 		 in a 64-bit pointer are significant for addressing,
-		 then the value is 39.  This is needed on e.g. Aarch64
+		 then the value is 39.  This is needed on e.g. AArch64
 		 v8.3 ABIs that use pointer authentication, so lldb
 		 knows which bits to clear/set to get the actual
 		 addresses.
@@ -1174,6 +1178,18 @@ tuples to return are:
                                      // a hex encoded string value that
                                      // contains an error string
 
+    dirty-pages:[<hexaddr>][,<hexaddr]; // A list of memory pages within this
+                 // region that are "dirty" -- they have been modified.
+                 // Page addresses are in base16.  The size of a page can
+                 // be found from the qHostInfo's page-size key-value.
+                 //
+                 // If the stub supports identifying dirty pages within a
+                 // memory region, this key should always be present for all
+                 // qMemoryRegionInfo replies.  This key with no pages 
+                 // listed ("dirty-pages:;") indicates no dirty pages in 
+                 // this memory region.  The *absence* of this key means
+                 // that this stub cannot determine dirty pages.
+
 If the address requested is not in a mapped region (e.g. we've jumped through
 a NULL pointer and are at 0x0) currently lldb expects to get back the size
 of the unmapped region -- that is, the distance to the next valid region.

lldb/include/lldb/API/SBMemoryRegionInfo.h

@@ -73,6 +73,40 @@ class LLDB_API SBMemoryRegionInfo {
   ///     region. If no name can be determined the returns nullptr.
   const char *GetName();
 
+  /// Returns whether this memory region has a list of memory pages
+  /// that have been modified -- that are dirty.
+  ///
+  /// \return
+  ///     True if the dirty page list is available.
+  bool HasDirtyMemoryPageList();
+
+  /// Returns the number of modified pages -- dirty pages -- in this
+  /// memory region.
+  ///
+  /// \return
+  ///     The number of dirty page entries will be returned.  If
+  ///     there are no dirty pages in this memory region, 0 will
+  ///     be returned.  0 will also be returned if the dirty page
+  ///     list is not available for this memory region -- you must
+  ///     use HasDirtyMemoryPageList() to check for that.
+  uint32_t GetNumDirtyPages();
+
+  /// Returns the address of a memory page that has been modified in
+  /// this region.
+  ///
+  /// \return
+  ///     Returns the address for his dirty page in the list.
+  ///     If this memory region does not have a dirty page list,
+  ///     LLDB_INVALID_ADDRESS is returned.
+  addr_t GetDirtyPageAddressAtIndex(uint32_t idx);
+
+  /// Returns the size of a memory page in this region.
+  ///
+  /// \return
+  ///     Returns the size of the memory pages in this region,
+  ///     or 0 if this information is unavailable.
+  int GetPageSize();
+
   bool operator==(const lldb::SBMemoryRegionInfo &rhs) const;
 
   bool operator!=(const lldb::SBMemoryRegionInfo &rhs) const;

lldb/include/lldb/Core/PluginManager.h

@@ -191,7 +191,8 @@ class PluginManager {
   GetObjectFileCreateMemoryCallbackForPluginName(ConstString name);
 
   static Status SaveCore(const lldb::ProcessSP &process_sp,
-                         const FileSpec &outfile);
+                         const FileSpec &outfile,
+                         lldb::SaveCoreStyle &core_style);
 
   // ObjectContainer
   static bool

lldb/include/lldb/Symbol/ObjectFile.h

@@ -666,6 +666,22 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
   /// Creates a plugin-specific call frame info
   virtual std::unique_ptr<CallFrameInfo> CreateCallFrameInfo();
 
+  /// Load binaries listed in a corefile
+  ///
+  /// A corefile may have metadata listing binaries that can be loaded,
+  /// and the offsets at which they were loaded.  This method will try
+  /// to add them to the Target.  If any binaries were loaded,
+  ///
+  /// \param[in] process
+  ///     Process where to load binaries.
+  ///
+  /// \return
+  ///     Returns true if any binaries were loaded.
+
+  virtual bool LoadCoreFileImages(lldb_private::Process &process) {
+    return false;
+  }
+
 protected:
   // Member variables.
   FileSpec m_file;

lldb/include/lldb/Target/MemoryRegionInfo.h

@@ -10,8 +10,11 @@
 #ifndef LLDB_TARGET_MEMORYREGIONINFO_H
 #define LLDB_TARGET_MEMORYREGIONINFO_H
 
+#include <vector>
+
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/RangeMap.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Support/FormatProviders.h"
 
 namespace lldb_private {
@@ -32,10 +35,7 @@ class MemoryRegionInfo {
 
   RangeType &GetRange() { return m_range; }
 
-  void Clear() {
-    m_range.Clear();
-    m_read = m_write = m_execute = m_memory_tagged = eDontKnow;
-  }
+  void Clear() { *this = MemoryRegionInfo(); }
 
   const RangeType &GetRange() const { return m_range; }
 
@@ -97,11 +97,33 @@ class MemoryRegionInfo {
            m_write == rhs.m_write && m_execute == rhs.m_execute &&
            m_mapped == rhs.m_mapped && m_name == rhs.m_name &&
            m_flash == rhs.m_flash && m_blocksize == rhs.m_blocksize &&
-           m_memory_tagged == rhs.m_memory_tagged;
+           m_memory_tagged == rhs.m_memory_tagged &&
+           m_pagesize == rhs.m_pagesize;
   }
 
   bool operator!=(const MemoryRegionInfo &rhs) const { return !(*this == rhs); }
 
+  /// Get the target system's VM page size in bytes.
+  /// \return
+  ///     0 is returned if this information is unavailable.
+  int GetPageSize() { return m_pagesize; }
+
+  /// Get a vector of target VM pages that are dirty -- that have been
+  /// modified -- within this memory region.  This is an Optional return
+  /// value; it will only be available if the remote stub was able to
+  /// detail this.
+  llvm::Optional<std::vector<lldb::addr_t>> &GetDirtyPageList() {
+    return m_dirty_pages;
+  }
+
+  void SetPageSize(int pagesize) { m_pagesize = pagesize; }
+
+  void SetDirtyPageList(std::vector<lldb::addr_t> pagelist) {
+    if (m_dirty_pages.hasValue())
+      m_dirty_pages.getValue().clear();
+    m_dirty_pages = std::move(pagelist);
+  }
+
 protected:
   RangeType m_range;
   OptionalBool m_read = eDontKnow;
@@ -112,6 +134,8 @@ class MemoryRegionInfo {
   OptionalBool m_flash = eDontKnow;
   lldb::offset_t m_blocksize = 0;
   OptionalBool m_memory_tagged = eDontKnow;
+  int m_pagesize = 0;
+  llvm::Optional<std::vector<lldb::addr_t>> m_dirty_pages;
 };
 
 inline bool operator<(const MemoryRegionInfo &lhs,

lldb/include/lldb/lldb-enumerations.h

@@ -601,6 +601,7 @@ enum CommandArgumentType {
   eArgTypeCommand,
   eArgTypeColumnNum,
   eArgTypeModuleUUID,
+  eArgTypeSaveCoreStyle,
   eArgTypeLastArg // Always keep this entry as the last entry in this
                   // enumeration!!
 };
@@ -1111,6 +1112,14 @@ enum CommandInterpreterResult {
   /// Stopped because quit was requested.
   eCommandInterpreterResultQuitRequested,
 };
+
+// Style of core file to create when calling SaveCore.
+enum SaveCoreStyle {
+  eSaveCoreUnspecified = 0,
+  eSaveCoreFull = 1,
+  eSaveCoreDirtyOnly = 2,
+};
+
 } // namespace lldb
 
 #endif // LLDB_LLDB_ENUMERATIONS_H

lldb/include/lldb/lldb-private-interfaces.h

@@ -54,7 +54,9 @@ typedef ObjectFile *(*ObjectFileCreateMemoryInstance)(
     const lldb::ModuleSP &module_sp, lldb::DataBufferSP &data_sp,
     const lldb::ProcessSP &process_sp, lldb::addr_t offset);
 typedef bool (*ObjectFileSaveCore)(const lldb::ProcessSP &process_sp,
-                                   const FileSpec &outfile, Status &error);
+                                   const FileSpec &outfile,
+                                   lldb::SaveCoreStyle &core_style,
+                                   Status &error);
 typedef EmulateInstruction *(*EmulateInstructionCreateInstance)(
     const ArchSpec &arch, InstructionType inst_type);
 typedef OperatingSystem *(*OperatingSystemCreateInstance)(Process *process,

lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py

@@ -723,7 +723,8 @@ def parse_memory_region_packet(self, context):
                  "permissions",
                  "flags",
                  "name",
-                 "error"])
+                 "error",
+                 "dirty-pages"])
             self.assertIsNotNone(val)
 
         mem_region_dict["name"] = seven.unhexlify(mem_region_dict.get("name", ""))

lldb/source/API/SBMemoryRegionInfo.cpp

@@ -116,6 +116,42 @@ const char *SBMemoryRegionInfo::GetName() {
   return m_opaque_up->GetName().AsCString();
 }
 
+bool SBMemoryRegionInfo::HasDirtyMemoryPageList() {
+  LLDB_RECORD_METHOD_NO_ARGS(bool, SBMemoryRegionInfo, HasDirtyMemoryPageList);
+
+  return m_opaque_up->GetDirtyPageList().hasValue();
+}
+
+uint32_t SBMemoryRegionInfo::GetNumDirtyPages() {
+  LLDB_RECORD_METHOD_NO_ARGS(uint32_t, SBMemoryRegionInfo, GetNumDirtyPages);
+
+  uint32_t num_dirty_pages = 0;
+  llvm::Optional<std::vector<addr_t>> dirty_page_list =
+      m_opaque_up->GetDirtyPageList();
+  if (dirty_page_list.hasValue())
+    num_dirty_pages = dirty_page_list.getValue().size();
+
+  return num_dirty_pages;
+}
+
+addr_t SBMemoryRegionInfo::GetDirtyPageAddressAtIndex(uint32_t idx) {
+  LLDB_RECORD_METHOD(addr_t, SBMemoryRegionInfo, GetDirtyPageAddressAtIndex,
+                     (uint32_t), idx);
+
+  addr_t dirty_page_addr = LLDB_INVALID_ADDRESS;
+  const llvm::Optional<std::vector<addr_t>> &dirty_page_list =
+      m_opaque_up->GetDirtyPageList();
+  if (dirty_page_list.hasValue() && idx < dirty_page_list.getValue().size())
+    dirty_page_addr = dirty_page_list.getValue()[idx];
+
+  return dirty_page_addr;
+}
+
+int SBMemoryRegionInfo::GetPageSize() {
+  LLDB_RECORD_METHOD_NO_ARGS(int, SBMemoryRegionInfo, GetPageSize);
+  return m_opaque_up->GetPageSize();
+}
+
 bool SBMemoryRegionInfo::GetDescription(SBStream &description) {
   LLDB_RECORD_METHOD(bool, SBMemoryRegionInfo, GetDescription,
                      (lldb::SBStream &), description);

lldb/source/API/SBProcess.cpp

@@ -1227,7 +1227,8 @@ lldb::SBError SBProcess::SaveCore(const char *file_name) {
   }
 
   FileSpec core_file(file_name);
-  error.ref() = PluginManager::SaveCore(process_sp, core_file);
+  SaveCoreStyle core_style = SaveCoreStyle::eSaveCoreFull;
+  error.ref() = PluginManager::SaveCore(process_sp, core_file, core_style);
   return LLDB_RECORD_RESULT(error);
 }
 

lldb/source/Commands/CommandObjectMemory.cpp

@@ -1678,6 +1678,28 @@ class CommandObjectMemoryRegion : public CommandObjectParsed {
       if (memory_tagged == MemoryRegionInfo::OptionalBool::eYes)
         result.AppendMessage("memory tagging: enabled");
 
+      const llvm::Optional<std::vector<addr_t>> &dirty_page_list =
+          range_info.GetDirtyPageList();
+      if (dirty_page_list.hasValue()) {
+        const size_t page_count = dirty_page_list.getValue().size();
+        result.AppendMessageWithFormat(
+            "Modified memory (dirty) page list provided, %zu entries.\n",
+            page_count);
+        if (page_count > 0) {
+          bool print_comma = false;
+          result.AppendMessageWithFormat("Dirty pages: ");
+          for (size_t i = 0; i < page_count; i++) {
+            if (print_comma)
+              result.AppendMessageWithFormat(", ");
+            else
+              print_comma = true;
+            result.AppendMessageWithFormat("0x%" PRIx64,
+                                           dirty_page_list.getValue()[i]);
+          }
+          result.AppendMessageWithFormat(".\n");
+        }
+      }
+
       m_prev_end_addr = range_info.GetRange().GetRangeEnd();
       result.SetStatus(eReturnStatusSuccessFinishResult);
       return true;