Skip to content

Update the Import Script to Include Windows Headers #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Update the Import Script to Include Windows Headers #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 193 additions & 56 deletions Utilities/import-llvm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
#!/usr/bin/env python

# This is a helper script for importing the things we use from LLVM.
#
# NOTE: We have slightly modified copies of some of the files, to reduce the
# dependency surface area, so if you run this to update the sources, please
# examine the diffs to remove things that don't make sense.

import errno
import optparse
Expand All @@ -13,52 +9,139 @@ import re
import shutil
import sys
import subprocess
import platform

# ADT types.
ADT_imports = ['EpochTracker', 'iterator', 'iterator_range', 'Hashing', 'None', 'Optional',
'Statistic', 'StringExtras', 'STLExtras', 'AllocatorList', 'Triple']

# ADT basic data structures.
ADT_imports += ['APFloat', 'APInt', 'APSInt', 'ArrayRef', 'PointerIntPair', 'SetVector', 'StringRef', 'StringSwitch',
'Twine', 'IntrusiveRefCntPtr', 'ilist', 'ilist_base', 'ilist_node', 'ilist_node_base',
'ilist_node_options', 'ilist_iterator', 'simple_ilist', 'OptionSet', 'PointerUnion']

# ADT Mapping structures.
ADT_imports += ['DenseMap', 'DenseMapInfo', 'DenseSet', 'FoldingSet', 'StringMap', 'StringSet']

# ADT "Small" structures.
ADT_imports += ['SmallPtrSet', 'SmallSet', 'SmallString', 'SmallVector']

# ADT Algorithms.
ADT_imports += ['edit_distance']
ADT_imports = [
'APFloat',
'APInt',
'APSInt',
'AllocatorList',
'ArrayRef',
'Bit',
'DenseMap',
'DenseMapInfo',
'DenseSet',
'EpochTracker',
'FoldingSet',
'Hashing',
'IntrusiveRefCntPtr',
'None',
'Optional',
'PointerIntPair',
'PointerUnion',
'STLExtras',
'SmallPtrSet',
'SmallSet',
'SmallString',
'SmallVector',
'Statistic',
'StringExtras',
'StringMap',
'StringRef',
'StringSet',
'StringSwitch',
'Triple',
'Twine',
'edit_distance',
'ilist',
'ilist_base',
'ilist_iterator',
'ilist_node',
'ilist_node_base',
'ilist_node_options',
'iterator',
'iterator_range',
'simple_ilist'
]

# Support types and infrastructure.
Support_imports = [
'AlignOf', 'Allocator', 'Atomic', 'CBindingWrapping', 'Casting', 'Capacity', 'CommandLine', 'Compiler',
'Endian', 'Errno', 'ErrorHandling', 'Errc', 'ErrorOr', 'Error', 'Format',
'ManagedStatic', 'MathExtras', 'Mutex', 'MutexGuard', 'Memory',
'MemoryBuffer', 'PointerLikeTypeTraits', 'Recycler', 'SwapByteOrder',
'Timer', 'TimeValue', 'Threading', 'Unicode', 'UniqueLock', 'Unix', 'WindowsError',
'Valgrind', 'circular_raw_ostream', 'raw_ostream', 'type_traits', 'JSON']

# Stuff we don't want, but have to pull in.
Support_imports += [
'COFF', 'ConvertUTF', 'ConvertUTFWrapper', 'Debug', 'FileSystem',
'FileUtilities', 'Host', 'Locale', 'MachO', 'Path', 'Process', 'Program', 'SMLoc',
'SourceMgr', 'Signals', 'StringSaver', 'ToolOutputFile', 'TrailingObjects', 'Unicode', 'UnicodeCharRanges',
'MemAlloc', 'Chrono', 'FormatProviders', 'FormatVariadic', 'FormatCommon',
'FormatVariadicDetails', 'NativeFormatting', 'DJB', 'ReverseIteration', 'MD5',
'SmallVectorMemoryBuffer', 'WithColor', 'Options', 'PrettyStackTrace', 'Watchdog',
'TargetParser', 'ARMBuildAttributes', 'ARMTargetParser.def', 'AArch64TargetParser.def', 'X86TargetParser.def', 'LineIterator']
'AArch64TargetParser',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Praise be M-x sort-lines

'APFloat',
'ARMBuildAttributes',
'ARMTargetParser',
'AlignOf',
'Allocator',
'Atomic',
'CBindingWrapping',
'Casting',
'Chrono',
'CommandLine',
'Compiler',
'ConvertUTF',
'ConvertUTFWrapper',
'DJB',
'DataTypes',
'Debug',
'Endian',
'Errc',
'Errno',
'Error',
'ErrorHandling',
'ErrorOr',
'FileSystem',
'FileUtilities',
'Format',
'FormatAdapters',
'FormatCommon',
'FormatProviders',
'FormatVariadic',
'FormatVariadicDetails',
'Host',
'LineIterator',
'Locale',
'MD5',
'ManagedStatic',
'MathExtras',
'MemAlloc',
'Memory',
'MemoryBuffer',
'Mutex',
'MutexGuard',
'NativeFormatting',
'Options',
'Path',
'PointerLikeTypeTraits',
'Process',
'Program',
'Recycler',
'ReverseIteration',
'SMLoc',
'Signals',
'SmallVectorMemoryBuffer',
'SourceMgr',
'StringSaver',
'Support',
'SwapByteOrder',
'TargetParser',
'Threading',
'Timer',
'Types',
'Unicode',
'Unicode',
'UnicodeCaseFold',
'UnicodeCharRanges',
'UniqueLock',
'Unix',
'Valgrind',
'VersionTuple',
'WindowsError',
'WindowsSupport',
'WithColor',
'X86TargetParser',
'circular_raw_ostream',
'raw_ostream',
'type_traits'
]

# Dependencies from llvm-c needed by Support.
C_imports = ['Types', 'DataTypes', 'Support', 'ErrorHandling']

# Support data structures.
Support_imports += ['YAMLParser', 'YAMLTraits']

# Source files to exclude.
Support_source_excludes = set(['Host'])
C_imports = [
'DataTypes',
'ErrorHandling',
'Support',
'Types'
]

llvm_srcroot = None
sourcekit_srcroot = None
Expand All @@ -80,6 +163,10 @@ def copyfile(src, dst):
note("cp %r %r" % (src, dst))
shutil.copyfile(src, dst)

def movefile(src, dst):
note("mv %r %r" % (src, dst))
shutil.movefile(src, dst)

def includes_prefix(source, include_dir):
defs = subprocess.check_output([
'clang', '-std=c++14', '-dM', '-E',
Expand Down Expand Up @@ -137,12 +224,32 @@ def main():
llvm_srcroot, = args
sourcekit_srcroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Remove the old llvm import
sourcekit_llvm_include = os.path.join(sourcekit_srcroot, 'include', 'llvm')
sourcekit_llvm_lib = os.path.join(sourcekit_srcroot, 'lib', 'LLVMSupport')
if os.path.exists(sourcekit_llvm_include):
shutil.rmtree(sourcekit_llvm_include)
if os.path.exists(sourcekit_llvm_lib):
shutil.rmtree(sourcekit_llvm_lib)

# Add the platform specific config files
#
# NOTE: If you're updating to a newer llvm, you may have to manually merge
# these with the newer version produced by the llvm build
sourcekit_llvm_config = os.path.join(sourcekit_srcroot, 'Utilities', 'import-llvm.d', 'include', 'llvm', 'Config')
shutil.copytree(sourcekit_llvm_config, os.path.join(sourcekit_llvm_include, 'Config'))

def import_header(dir, name):
src = os.path.join(llvm_srcroot, 'include', 'llvm', dir, name)
c_src = os.path.join(llvm_srcroot, 'include', 'llvm-c', name)
if os.path.exists(src):
dst = os.path.join(sourcekit_srcroot, 'include', 'llvm', dir, name)
mkdir_p(os.path.dirname(dst))
copyfile(src, dst)
if os.path.exists(c_src):
dst = os.path.join(sourcekit_srcroot, 'include', 'llvm-c', name)
mkdir_p(os.path.dirname(dst))
copyfile(c_src, dst)

def import_c_header(name):
src = os.path.join(llvm_srcroot, 'include', 'llvm-c', name)
Expand All @@ -162,23 +269,32 @@ def main():

for name in ADT_imports:
import_header('ADT', name+'.h')
if name not in Support_source_excludes:
import_source('Support', name+'.c')
import_source('Support', name+'.cpp')
import_source('Support', name+'.c')
import_source('Support', name+'.cpp')
for name in Support_imports:
if name.endswith('.def'):
import_header('Support', name)
continue
import_header('Support', name+'.h')
if name not in Support_source_excludes:
import_source('Support', name+'.c')
import_source('Support', name+'.cpp')
import_source('Support', os.path.join('Unix', name+'.h'))
import_source('Support', os.path.join('Unix', name+'.inc'))

import_header('Support', name+'.def')
import_source('Support', name+'.c')
import_source('Support', name+'.cpp')
import_source('Support', name+'.inc')
import_source('Support', name+'.h')
import_source('Support', os.path.join('Unix', name+'.h'))
import_source('Support', os.path.join('Unix', name+'.inc'))
import_source('Support', os.path.join('Windows', name+'.h'))
import_source('Support', os.path.join('Windows', name+'.inc'))
for name in C_imports:
import_c_header(name + '.h')

# Create symlinks so SwiftPM can find headers
os.makedirs(os.path.join(sourcekit_llvm_lib, 'include'))
os.symlink(os.path.join('..', '..', '..', 'include', 'llvm'),
os.path.join(sourcekit_llvm_lib, 'include', 'llvm'))
os.symlink(os.path.join('..', '..', '..', 'include', 'llvm-c'),
os.path.join(sourcekit_llvm_lib, 'include', 'llvm-c'))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could these symlinks be done by looping over the directories instead of listing them out manually? It's really hard to tell if these are correct by inspection.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cleaned these up a bit, I didn't need as many as I thought I did.


copyfile(os.path.join(sourcekit_srcroot, 'Utilities', 'import-llvm.d', 'lib', 'LLVMSupport', 'include', 'module.modulemap'),
os.path.join(sourcekit_srcroot, 'lib', 'LLVMSupport', 'include', 'module.modulemap'))

print ""
print "Adding prefix header includes"

Expand All @@ -189,11 +305,32 @@ def main():
]

include_dir = os.path.join(sourcekit_srcroot, 'include')

for base in base_dirs:
for root, dirs, files in os.walk(base):
for file in files:
maybe_add_prefix(os.path.join(root, file), include_dir)

# Finish off with a bunch of really gross hacks. These all offend my better
# sensibilities and what have you.

# TODO: Indexstoredb uses an out of tree/non merged OptionSet.h We should
# remove uses of this or get it merged upstream.
copyfile(os.path.join(sourcekit_srcroot, 'Utilities', 'import-llvm.d', 'include', 'llvm', 'ADT', 'OptionSet.h'),
os.path.join(sourcekit_srcroot, 'include', 'llvm', 'ADT', 'OptionSet.h'))

# AArch64TargetParser uses a relative path to find the .def file, so we
# copy this one to the expected location :(
def_location = os.path.join(sourcekit_srcroot, 'lib', 'include', 'llvm', 'Support')
if os.path.exists(def_location):
shutil.rmtree(def_location)
os.makedirs(def_location)
copyfile(os.path.join(sourcekit_srcroot, 'include', 'llvm', 'Support', 'AArch64TargetParser.def'),
os.path.join(def_location, 'AArch64TargetParser.def'))

# Patch out requirements for Demangle and YAML/Regex from the LLVM support we imported
patch_file = os.path.join(sourcekit_srcroot, 'Utilities', 'import-llvm.d', 'RemoveLLVMImports.patch')
with open(patch_file, 'r') as f:
subprocess.Popen(['patch', '-p1'], stdin=f, cwd=sourcekit_srcroot)

if __name__ == '__main__':
main()
67 changes: 67 additions & 0 deletions Utilities/import-llvm.d/RemoveLLVMImports.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
diff --git a/lib/LLVMSupport/Support/Statistic.cpp b/lib/LLVMSupport/Support/Statistic.cpp
index d57300a..86e6dc1 100644
--- a/lib/LLVMSupport/Support/Statistic.cpp
+++ b/lib/LLVMSupport/Support/Statistic.cpp
@@ -30,7 +30,6 @@
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/Timer.h"
-#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstring>
@@ -211,10 +210,6 @@ void llvm::PrintStatisticsJSON(raw_ostream &OS) {
const char *delim = "";
for (const Statistic *Stat : Stats.Stats) {
OS << delim;
- assert(yaml::needsQuotes(Stat->getDebugType()) == yaml::QuotingType::None &&
- "Statistic group/type name is simple.");
- assert(yaml::needsQuotes(Stat->getName()) == yaml::QuotingType::None &&
- "Statistic name is simple");
OS << "\t\"" << Stat->getDebugType() << '.' << Stat->getName() << "\": "
<< Stat->getValue();
delim = ",\n";
diff --git a/lib/LLVMSupport/Support/Timer.cpp b/lib/LLVMSupport/Support/Timer.cpp
index 82f5810..3ec1801 100644
--- a/lib/LLVMSupport/Support/Timer.cpp
+++ b/lib/LLVMSupport/Support/Timer.cpp
@@ -20,7 +20,6 @@
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/Process.h"
-#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
#include <limits>

@@ -388,10 +387,6 @@ void TimerGroup::clearAll() {

void TimerGroup::printJSONValue(raw_ostream &OS, const PrintRecord &R,
const char *suffix, double Value) {
- assert(yaml::needsQuotes(Name) == yaml::QuotingType::None &&
- "TimerGroup name should not need quotes");
- assert(yaml::needsQuotes(R.Name) == yaml::QuotingType::None &&
- "Timer name should not need quotes");
constexpr auto max_digits10 = std::numeric_limits<double>::max_digits10;
OS << "\t\"time." << Name << '.' << R.Name << suffix
<< "\": " << format("%.*e", max_digits10 - 1, Value);
diff --git a/lib/LLVMSupport/Support/Unix/Signals.inc b/lib/LLVMSupport/Support/Unix/Signals.inc
index ad88d5e..7ed843d 100644
--- a/lib/LLVMSupport/Support/Unix/Signals.inc
+++ b/lib/LLVMSupport/Support/Unix/Signals.inc
@@ -36,7 +36,6 @@
#include "Unix.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Config/config.h"
-#include "llvm/Demangle/Demangle.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/Format.h"
@@ -535,7 +534,7 @@ void llvm::sys::PrintStackTrace(raw_ostream &OS) {
if (dlinfo.dli_sname != nullptr) {
OS << ' ';
int res;
- char* d = itaniumDemangle(dlinfo.dli_sname, nullptr, nullptr, &res);
+ char* d = NULL;
if (!d) OS << dlinfo.dli_sname;
else OS << d;
free(d);
Loading