[sourcekitd] Add support for transferring raw data

tools/SourceKit/tools/sourcekitd/bin/sourcekitd.exports

@@ -62,5 +62,7 @@ sourcekitd_variant_get_type
 sourcekitd_variant_json_description_copy
 sourcekitd_variant_string_get_length
 sourcekitd_variant_string_get_ptr
+sourcekitd_variant_data_get_size
+sourcekitd_variant_data_get_ptr
 sourcekitd_variant_int64_get_value
 sourcekitd_variant_uid_get_value

tools/SourceKit/tools/sourcekitd/include/sourcekitd/CompactArray.h

@@ -190,7 +190,9 @@ VariantFunctions CompactVariantFuncs<T>::Funcs = {
   nullptr/*Annot_string_get_length*/,
   nullptr/*Annot_string_get_ptr*/,
   nullptr/*Annot_int64_get_value*/,
-  nullptr/*Annot_uid_get_value*/
+  nullptr/*Annot_uid_get_value*/,
+  nullptr/*Annot_data_get_size*/,
+  nullptr/*Annot_data_get_ptr*/,
 };
 
 template <typename T>
@@ -235,7 +237,9 @@ VariantFunctions CompactArrayFuncs<T>::Funcs = {
   nullptr/*AnnotArray_string_get_length*/,
   nullptr/*AnnotArray_string_get_ptr*/,
   nullptr/*AnnotArray_int64_get_value*/,
-  nullptr/*AnnotArray_uid_get_value*/
+  nullptr/*AnnotArray_uid_get_value*/,
+  nullptr/*Annot_data_get_size*/,
+  nullptr/*Annot_data_get_ptr*/,
 };
 
 }

tools/SourceKit/tools/sourcekitd/include/sourcekitd/Internal.h

@@ -59,6 +59,7 @@ enum class CustomBufferKind {
   InheritedTypesArray,
   DocStructureElementArray,
   AttributesArray,
+  RawData
 };
 
 class ResponseBuilder {
@@ -219,6 +220,8 @@ struct VariantFunctions {
   const char *(*string_get_ptr)(sourcekitd_variant_t obj);
   int64_t (*int64_get_value)(sourcekitd_variant_t obj);
   sourcekitd_uid_t (*uid_get_value)(sourcekitd_variant_t obj);
+  size_t (*data_get_size)(sourcekitd_variant_t obj);
+  const void *(*data_get_ptr)(sourcekitd_variant_t obj);
 };
 
 }

tools/SourceKit/tools/sourcekitd/include/sourcekitd/RawData.h

@@ -0,0 +1,24 @@
+//===--- RawData.h - =-------------------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SOURCEKITD_RAW_DATA_H
+#define LLVM_SOURCEKITD_RAW_DATA_H
+
+#include "sourcekitd/Internal.h"
+
+namespace sourcekitd {
+
+VariantFunctions *getVariantFunctionsForRawData();
+
+} // end namespace sourcekitd
+
+#endif // LLVM_SOURCEKITD_RAW_DATA_H

tools/SourceKit/tools/sourcekitd/include/sourcekitd/RequestResponsePrinterBase.h

@@ -103,6 +103,11 @@ class RequestResponsePrinterBase {
       OSColor(OS, UIDColor) << UID;
     }
   }
+
+  void visitData(const void *Data, size_t Size) {
+    // FIXME: We should probably print the real data here
+    OS << "<data>";
+  }
 };
 
 }

tools/SourceKit/tools/sourcekitd/include/sourcekitd/sourcekitd.h

@@ -25,7 +25,7 @@
 /// The policy about the sourcekitd API is to keep it source and ABI compatible,
 /// thus SOURCEKITD_VERSION_MAJOR is expected to remain stable.
 #define SOURCEKITD_VERSION_MAJOR 0
-#define SOURCEKITD_VERSION_MINOR 3
+#define SOURCEKITD_VERSION_MINOR 4
 
 #define SOURCEKITD_VERSION_ENCODE(major, minor) ( \
       ((major) * 10000)                           \
@@ -309,7 +309,10 @@ typedef enum {
   SOURCEKITD_VARIANT_TYPE_INT64 = 3,
   SOURCEKITD_VARIANT_TYPE_STRING = 4,
   SOURCEKITD_VARIANT_TYPE_UID = 5,
-  SOURCEKITD_VARIANT_TYPE_BOOL = 6
+  SOURCEKITD_VARIANT_TYPE_BOOL = 6,
+  // Reserved for future addition
+  // SOURCEKITD_VARIANT_TYPE_DOUBLE = 7,
+  SOURCEKITD_VARIANT_TYPE_DATA = 8,
 } sourcekitd_variant_type_t;
 
 typedef enum {
@@ -517,6 +520,14 @@ SOURCEKITD_PUBLIC SOURCEKITD_WARN_RESULT
 const char *
 sourcekitd_variant_string_get_ptr(sourcekitd_variant_t obj);
 
+SOURCEKITD_PUBLIC SOURCEKITD_WARN_RESULT
+size_t
+sourcekitd_variant_data_get_size(sourcekitd_variant_t obj);
+
+SOURCEKITD_PUBLIC SOURCEKITD_WARN_RESULT
+const void *
+sourcekitd_variant_data_get_ptr(sourcekitd_variant_t obj);
+
 SOURCEKITD_PUBLIC SOURCEKITD_WARN_RESULT
 sourcekitd_uid_t
 sourcekitd_variant_uid_get_value(sourcekitd_variant_t obj);

tools/SourceKit/tools/sourcekitd/lib/API/CMakeLists.txt

@@ -8,6 +8,7 @@ set(sourcekitdAPI_sources
   CompactArray.cpp
   DocStructureArray.cpp
   DocSupportAnnotationArray.cpp
+  RawData.cpp
   Requests.cpp
   sourcekitdAPI-Common.cpp
   TokenAnnotationsArray.cpp

tools/SourceKit/tools/sourcekitd/lib/API/DocStructureArray.cpp

@@ -579,7 +579,9 @@ VariantFunctions DocStructureArrayFuncs::funcs = {
     nullptr /*AnnotArray_string_get_length*/,
     nullptr /*AnnotArray_string_get_ptr*/,
     nullptr /*AnnotArray_int64_get_value*/,
-    nullptr /*AnnotArray_uid_get_value*/
+    nullptr /*AnnotArray_uid_get_value*/,
+    nullptr /*Annot_data_get_size*/,
+    nullptr /*Annot_data_get_ptr*/,
 };
 
 VariantFunctions *sourcekitd::getVariantFunctionsForDocStructureElementArray() {

tools/SourceKit/tools/sourcekitd/lib/API/RawData.cpp

@@ -0,0 +1,59 @@
+//===--- RawData.cpp ------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "sourcekitd/RawData.h"
+
+using namespace sourcekitd;
+
+struct RawDataFuncs {
+  static sourcekitd_variant_type_t get_type(sourcekitd_variant_t var) {
+    return SOURCEKITD_VARIANT_TYPE_DATA;
+  }
+
+  static size_t data_get_size(sourcekitd_variant_t variant) {
+    return variant.data[2];
+  }
+
+  static const void *data_get_ptr(sourcekitd_variant_t variant) {
+    return reinterpret_cast<void *>(variant.data[1]);
+  }
+
+  static VariantFunctions Funcs;
+};
+
+VariantFunctions RawDataFuncs::Funcs = {
+  get_type,
+  nullptr /*AnnotArray_array_apply*/,
+  nullptr /*AnnotArray_array_get_bool*/,
+  nullptr /*Annot_array_array_get_count*/,
+  nullptr /*AnnotArray_array_get_int64*/,
+  nullptr /*AnnotArray_array_get_string*/,
+  nullptr /*AnnotArray_array_get_uid*/,
+  nullptr /*AnnotArray_array_get_value*/,
+  nullptr /*AnnotArray_bool_get_value*/,
+  nullptr /*AnnotArray_dictionary_apply*/,
+  nullptr /*AnnotArray_dictionary_get_bool*/,
+  nullptr /*AnnotArray_dictionary_get_int64*/,
+  nullptr /*AnnotArray_dictionary_get_string*/,
+  nullptr /*AnnotArray_dictionary_get_value*/,
+  nullptr /*AnnotArray_dictionary_get_uid*/,
+  nullptr /*AnnotArray_string_get_length*/,
+  nullptr /*AnnotArray_string_get_ptr*/,
+  nullptr /*AnnotArray_int64_get_value*/,
+  nullptr /*AnnotArray_uid_get_value*/,
+  data_get_size,
+  data_get_ptr,
+};
+
+VariantFunctions *sourcekitd::getVariantFunctionsForRawData() {
+  return &RawDataFuncs::Funcs;
+}

tools/SourceKit/tools/sourcekitd/lib/API/TokenAnnotationsArray.cpp

@@ -218,7 +218,9 @@ VariantFunctions CompactVariantFuncs<TokenAnnotationsArray>::Funcs = {
   nullptr/*Annot_string_get_length*/,
   nullptr/*Annot_string_get_ptr*/,
   nullptr/*Annot_int64_get_value*/,
-  nullptr/*Annot_uid_get_value*/
+  nullptr/*Annot_uid_get_value*/,
+  nullptr/*Annot_data_get_size*/,
+  nullptr/*Annot_data_get_ptr*/,
 };
 
 } // namespace sourcekitd

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-Common.cpp

@@ -115,6 +115,11 @@ class VariantVisitor {
       const char *Ptr = sourcekitd_uid_get_string_ptr(UID);
       return static_cast<ImplClass*>(this)->visitUID(StringRef(Ptr, Len));
     }
+    case SOURCEKITD_VARIANT_TYPE_DATA: {
+      const void *Data = sourcekitd_variant_data_get_ptr(Obj);
+      size_t Size = sourcekitd_variant_data_get_size(Obj);
+      return static_cast<ImplClass*>(this)->visitData(Data, Size);
+    }
     }
   }
 };
@@ -566,6 +571,26 @@ sourcekitd_variant_string_get_ptr(sourcekitd_variant_t obj) {
   return (const char *)obj.data[1];
 }
 
+size_t
+sourcekitd_variant_data_get_size(sourcekitd_variant_t obj) {
+  if (auto fn = VAR_FN(obj, data_get_size))
+    return fn(obj);
+
+  // Default implementation:
+  // We store the byte's length in data[2] and its data in data[1]
+  return obj.data[2];
+}
+
+const void *
+sourcekitd_variant_data_get_ptr(sourcekitd_variant_t obj) {
+  if (auto fn = VAR_FN(obj, data_get_ptr))
+    return fn(obj);
+
+  // Default implementation:
+  // We store the byte's length in data[2] and its data in data[1]
+  return reinterpret_cast<void *>(obj.data[1]);
+}
+
 sourcekitd_uid_t
 sourcekitd_variant_uid_get_value(sourcekitd_variant_t obj) {
   if (auto fn = VAR_FN(obj, uid_get_value))

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-InProc.cpp

@@ -16,6 +16,7 @@
 #include "sourcekitd/CodeCompletionResultsArray.h"
 #include "sourcekitd/DocStructureArray.h"
 #include "sourcekitd/DocSupportAnnotationArray.h"
+#include "sourcekitd/RawData.h"
 #include "sourcekitd/TokenAnnotationsArray.h"
 #include "sourcekitd/Logging.h"
 #include "SourceKit/Core/LLVM.h"
@@ -74,6 +75,8 @@ class SKDObject : public ThreadSafeRefCountedBaseVPTR {
   virtual Optional<StringRef> getString() const { return None; }
   virtual const char *getCString() const { return nullptr; }
   virtual bool getBool() const { return false; }
+  virtual const void *getDataPtr() const { return nullptr; }
+  virtual size_t getDataSize() const { return 0; }
 };
 
 class SKDDictionary: public SKDObject {
@@ -255,6 +258,8 @@ class SKDCustomData: public SKDObject {
       case CustomBufferKind::DocStructureElementArray:
       case CustomBufferKind::AttributesArray:
         return SOURCEKITD_VARIANT_TYPE_ARRAY;
+      case CustomBufferKind::RawData:
+        return SOURCEKITD_VARIANT_TYPE_DATA;
     }
     llvm::report_fatal_error("sourcekitd object did not resolve to a known type");
   }
@@ -263,10 +268,14 @@ class SKDCustomData: public SKDObject {
     return BufferKind;
   }
 
-  const void *getDataPtr() const {
+  const void *getDataPtr() const override {
     return BufferPtr->getBuffer().data();
   }
 
+  size_t getDataSize() const override {
+    return BufferPtr->getBuffer().size();
+  }
+
   static bool classof(const SKDObject *O) {
     return O->getKind() == ObjectKind::CustomData;
   }
@@ -915,6 +924,13 @@ static sourcekitd_uid_t SKDVar_uid_get_value(sourcekitd_variant_t obj) {
   return SKD_OBJ(obj)->getUID();
 }
 
+static const void *SKDVar_data_get_ptr(sourcekitd_variant_t obj) {
+  return SKD_OBJ(obj)->getDataPtr();
+}
+
+static size_t SKDVar_data_get_size(sourcekitd_variant_t obj) {
+  return SKD_OBJ(obj)->getDataSize();
+}
 
 static VariantFunctions SKDVariantFuncs = {
   SKDVar_get_type,
@@ -935,7 +951,9 @@ static VariantFunctions SKDVariantFuncs = {
   SKDVar_string_get_length,
   SKDVar_string_get_ptr,
   SKDVar_int64_get_value,
-  SKDVar_uid_get_value
+  SKDVar_uid_get_value,
+  SKDVar_data_get_size,
+  SKDVar_data_get_ptr,
 };
 
 static sourcekitd_variant_t variantFromSKDObject(SKDObjectRef Object) {
@@ -965,6 +983,10 @@ static sourcekitd_variant_t variantFromSKDObject(SKDObjectRef Object) {
       case CustomBufferKind::AttributesArray:
         return {{ (uintptr_t)getVariantFunctionsForAttributesArray(),
           (uintptr_t)DataObject->getDataPtr(), 0 }};
+      case CustomBufferKind::RawData:
+        return {{ (uintptr_t)getVariantFunctionsForRawData(),
+                  (uintptr_t)DataObject->getDataPtr(),
+                  (uintptr_t)DataObject->getDataSize() }};
     }
   }
 

tools/SourceKit/tools/sourcekitd/lib/API/sourcekitdAPI-XPC.cpp

@@ -15,6 +15,7 @@
 #include "sourcekitd/DocStructureArray.h"
 #include "sourcekitd/DocSupportAnnotationArray.h"
 #include "sourcekitd/TokenAnnotationsArray.h"
+#include "sourcekitd/RawData.h"
 #include "sourcekitd/RequestResponsePrinterBase.h"
 #include "SourceKit/Support/UIdent.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -586,6 +587,7 @@ sourcekitd_response_get_value(sourcekitd_response_t resp) {
   ((CustomBufferKind)*(const uint64_t*)xpc_data_get_bytes_ptr(xobj))
 #define CUSTOM_BUF_START(xobj) \
   ((const void*)(((const uint64_t*)xpc_data_get_bytes_ptr(xobj))+1))
+#define CUSTOM_BUF_SIZE(xobj) (xpc_data_get_length(xobj) - sizeof(uint64_t))
 
 static sourcekitd_variant_type_t XPCVar_get_type(sourcekitd_variant_t var) {
   xpc_object_t obj = XPC_OBJ(var);
@@ -620,6 +622,8 @@ static sourcekitd_variant_type_t XPCVar_get_type(sourcekitd_variant_t var) {
     case CustomBufferKind::DocStructureElementArray:
     case CustomBufferKind::AttributesArray:
       return SOURCEKITD_VARIANT_TYPE_ARRAY;
+    case CustomBufferKind::RawData:
+      return SOURCEKITD_VARIANT_TYPE_DATA;
     }
   }
 
@@ -709,6 +713,14 @@ static const char *XPCVar_string_get_ptr(sourcekitd_variant_t obj) {
   return xpc_string_get_string_ptr(XPC_OBJ(obj));
 }
 
+static size_t XPCVar_data_get_size(sourcekitd_variant_t obj) {
+  return xpc_data_get_length(XPC_OBJ(obj));
+}
+
+static const void *XPCVar_data_get_ptr(sourcekitd_variant_t obj) {
+  return xpc_data_get_bytes_ptr(XPC_OBJ(obj));
+}
+
 static int64_t XPCVar_int64_get_value(sourcekitd_variant_t obj) {
   return xpc_int64_get_value(XPC_OBJ(obj));
 }
@@ -737,7 +749,9 @@ static VariantFunctions XPCVariantFuncs = {
   XPCVar_string_get_length,
   XPCVar_string_get_ptr,
   XPCVar_int64_get_value,
-  XPCVar_uid_get_value
+  XPCVar_uid_get_value,
+  XPCVar_data_get_size,
+  XPCVar_data_get_ptr,
 };
 
 static sourcekitd_variant_t variantFromXPCObject(xpc_object_t obj) {
@@ -767,6 +781,10 @@ static sourcekitd_variant_t variantFromXPCObject(xpc_object_t obj) {
     case CustomBufferKind::AttributesArray:
       return {{ (uintptr_t)getVariantFunctionsForAttributesArray(),
                 (uintptr_t)CUSTOM_BUF_START(obj), 0 }};
+    case sourcekitd::CustomBufferKind::RawData:
+      return {{ (uintptr_t)getVariantFunctionsForRawData(),
+                (uintptr_t)CUSTOM_BUF_START(obj),
+                (uintptr_t)CUSTOM_BUF_SIZE(obj) }};
     }
   }