Skip to content

Commit 4ea4563

Browse files
maksleventalmakslevental
committed
types working
This fixes test breakage on clang bots. See comment llvm/llvm-project#121081 (comment)
1 parent bd163cb commit 4ea4563

File tree

13 files changed

+256
-6140
lines changed

13 files changed

+256
-6140
lines changed

‎projects/eudsl-py/CMakeLists.txt

Lines changed: 62 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,11 @@ add_eudslpygen(eudslpy-gen EUDSLPY
107107
src/eudslpy-gen.cpp
108108
)
109109

110-
eudslpygen(EUDSLGenacc
111-
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
112-
NAMESPACES mlir::acc mlir::acc::detail
113-
)
110+
# too big
111+
# eudslpygen(EUDSLGenacc
112+
# ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
113+
# NAMESPACES mlir::acc mlir::acc::detail
114+
# )
114115

115116
eudslpygen(EUDSLGenaffine
116117
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
@@ -140,10 +141,11 @@ eudslpygen(EUDSLGenarm_neon
140141
NAMESPACES mlir::arm_neon mlir::arm_neon::detail
141142
)
142143

143-
eudslpygen(EUDSLGenarm_sme
144-
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
145-
NAMESPACES mlir::arm_sme mlir::arm_sme::detail
146-
)
144+
# too big
145+
# eudslpygen(EUDSLGenarm_sme
146+
# ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
147+
# NAMESPACES mlir::arm_sme mlir::arm_sme::detail
148+
# )
147149

148150
eudslpygen(EUDSLGenarm_sve
149151
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
@@ -180,10 +182,10 @@ eudslpygen(EUDSLGenDLTIDialect
180182
NAMESPACES mlir::DLTIDialect mlir::DLTIDialect::detail
181183
)
182184

183-
#eudslpygen(EUDSLGenemitc
184-
# ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
185-
# NAMESPACES mlir::emitc mlir::emitc::detail
186-
#)
185+
eudslpygen(EUDSLGenemitc
186+
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
187+
NAMESPACES mlir::emitc mlir::emitc::detail
188+
)
187189

188190
eudslpygen(EUDSLGenfunc
189191
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
@@ -216,10 +218,10 @@ eudslpygen(EUDSLGenlinalg
216218
NAMESPACES mlir::linalg mlir::linalg::detail
217219
)
218220

219-
#eudslpygen(EUDSLGenLLVM
220-
# ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
221-
# NAMESPACES mlir::LLVM mlir::LLVM::detail
222-
#)
221+
eudslpygen(EUDSLGenLLVM
222+
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
223+
NAMESPACES mlir::LLVM mlir::LLVM::detail
224+
)
223225

224226
eudslpygen(EUDSLGenmath
225227
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
@@ -318,12 +320,14 @@ eudslpygen(EUDSLGensparse_tensor
318320
NAMESPACES mlir::sparse_tensor mlir::sparse_tensor::detail
319321
)
320322

321-
eudslpygen(EUDSLGenspirv
322-
${MLIR_INCLUDE_DIR}/mlir/Dialect/SPIRV/IR/SPIRVOps.h
323-
NAMESPACES mlir::spirv mlir::spirv::detail
324-
EXTRA_INCLUDES
325-
mlir/Dialect/SPIRV/IR/SPIRVOps.h
326-
)
323+
# nb::detail::nb_func_new("get_vce_triple_attr_name"): mismatched static/instance method flags in function overloads!
324+
# eudslpygen(EUDSLGenspirv
325+
# ${MLIR_INCLUDE_DIR}/mlir/Dialect/SPIRV/IR/SPIRVOps.h
326+
# NAMESPACES mlir::spirv mlir::spirv::detail
327+
# EXTRA_INCLUDES
328+
# mlir/Dialect/SPIRV/IR/SPIRVOps.h
329+
# mlir/IR/PatternMatch.h
330+
# )
327331

328332
eudslpygen(EUDSLGentensor
329333
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
@@ -355,107 +359,65 @@ eudslpygen(EUDSLGenx86vector
355359
NAMESPACES mlir::x86vector mlir::x86vector::detail
356360
)
357361

358-
#eudslpygen(EUDSLGenxegpu
359-
# ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
360-
# NAMESPACES mlir::xegpu mlir::xegpu::detail
361-
#)
362+
eudslpygen(EUDSLGenxegpu
363+
${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h
364+
NAMESPACES mlir::xegpu mlir::xegpu::detail
365+
)
362366

363367
nanobind_add_module(eudslpy_ext
364368
NB_STATIC
365369
NB_DOMAIN eudsl
366370
src/eudslpy_ext.cpp
367-
${EUDSLGenacc_GENERATED_SHARDS}
368-
${EUDSLGenaffine_GENERATED_SHARDS}
369-
${EUDSLGenamdgpu_GENERATED_SHARDS}
370-
${EUDSLGenamx_GENERATED_SHARDS}
371-
${EUDSLGenarith_GENERATED_SHARDS}
372-
${EUDSLGenarm_neon_GENERATED_SHARDS}
373-
${EUDSLGenarm_sme_GENERATED_SHARDS}
374-
${EUDSLGenarm_sve_GENERATED_SHARDS}
375-
${EUDSLGenasync_GENERATED_SHARDS}
376-
${EUDSLGenbufferization_GENERATED_SHARDS}
377-
${EUDSLGencf_GENERATED_SHARDS}
378-
${EUDSLGencomplex_GENERATED_SHARDS}
379-
${EUDSLGenDLTIDialect_GENERATED_SHARDS}
380-
# ${EUDSLGenemitc_GENERATED_SHARDS}
381-
${EUDSLGenfunc_GENERATED_SHARDS}
382-
${EUDSLGengpu_GENERATED_SHARDS}
383-
${EUDSLGenindex_GENERATED_SHARDS}
384-
# ${EUDSLGenirdl_GENERATED_SHARDS}
385-
# ${EUDSLGenlinalg_GENERATED_SHARDS}
386-
# ${EUDSLGenLLVM_GENERATED_SHARDS}
387-
${EUDSLGenmath_GENERATED_SHARDS}
388-
${EUDSLGenmemref_GENERATED_SHARDS}
389-
${EUDSLGenmesh_GENERATED_SHARDS}
390-
${EUDSLGenml_program_GENERATED_SHARDS}
391-
${EUDSLGenmpi_GENERATED_SHARDS}
392-
${EUDSLGennvgpu_GENERATED_SHARDS}
393-
${EUDSLGenNVVM_GENERATED_SHARDS}
394-
# ${EUDSLGenomp_GENERATED_SHARDS}
395-
${EUDSLGenpdl_GENERATED_SHARDS}
396-
${EUDSLGenpdl_interp_GENERATED_SHARDS}
397-
${EUDSLGenpolynomial_GENERATED_SHARDS}
398-
${EUDSLGenptr_GENERATED_SHARDS}
399-
${EUDSLGenquant_GENERATED_SHARDS}
400-
${EUDSLGenROCDL_GENERATED_SHARDS}
401-
# ${EUDSLGenscf_GENERATED_SHARDS}
402-
# ${EUDSLGenshape_GENERATED_SHARDS}
403-
${EUDSLGensparse_tensor_GENERATED_SHARDS}
404-
# ${EUDSLGenspirv_GENERATED_SHARDS}
405-
${EUDSLGentensor_GENERATED_SHARDS}
406-
${EUDSLGentosa_GENERATED_SHARDS}
407-
${EUDSLGentransform_GENERATED_SHARDS}
408-
${EUDSLGenub_GENERATED_SHARDS}
409-
# ${EUDSLGenvector_GENERATED_SHARDS}
410-
${EUDSLGenx86vector_GENERATED_SHARDS}
411-
# ${EUDSLGenxegpu_GENERATED_SHARDS}
412-
)
413-
add_dependencies(eudslpy_ext
414-
EUDSLGenacc
371+
)
372+
target_link_libraries(eudslpy_ext PRIVATE
373+
# broken
374+
# EUDSLGenirdl
375+
# EUDSLGenomp
376+
# EUDSLGenvector
377+
# weight reduction
378+
# EUDSLGenacc
379+
# EUDSLGenamx
380+
# EUDSLGenarm_neon
381+
# EUDSLGenarm_sme
382+
# EUDSLGenarm_sve
383+
# EUDSLGenDLTIDialect
384+
# EUDSLGenmesh
385+
# EUDSLGenml_program
386+
# EUDSLGenmpi
387+
# EUDSLGenptr
388+
# EUDSLGenquant
389+
# EUDSLGensparse_tensor
390+
# EUDSLGenspirv
391+
# EUDSLGentransform
392+
# EUDSLGenub
393+
# EUDSLGenx86vector
394+
# EUDSLGenxegpu
395+
415396
EUDSLGenaffine
416397
EUDSLGenamdgpu
417-
EUDSLGenamx
418398
EUDSLGenarith
419-
EUDSLGenarm_neon
420-
EUDSLGenarm_sme
421-
EUDSLGenarm_sve
422399
EUDSLGenasync
423400
EUDSLGenbufferization
424401
EUDSLGencf
425402
EUDSLGencomplex
426-
EUDSLGenDLTIDialect
427-
# EUDSLGenemitc
403+
EUDSLGenemitc
428404
EUDSLGenfunc
429405
EUDSLGengpu
430406
EUDSLGenindex
431-
# EUDSLGenirdl
432-
# EUDSLGenlinalg
433-
# EUDSLGenLLVM
407+
EUDSLGenlinalg
408+
EUDSLGenLLVM
434409
EUDSLGenmath
435410
EUDSLGenmemref
436-
EUDSLGenmesh
437-
EUDSLGenml_program
438-
EUDSLGenmpi
439411
EUDSLGennvgpu
440412
EUDSLGenNVVM
441-
# EUDSLGenomp
442413
EUDSLGenpdl
443414
EUDSLGenpdl_interp
444415
EUDSLGenpolynomial
445-
EUDSLGenptr
446-
EUDSLGenquant
447416
EUDSLGenROCDL
448-
# EUDSLGenscf
449-
# EUDSLGenshape
450-
EUDSLGensparse_tensor
451-
# EUDSLGenspirv
417+
EUDSLGenscf
418+
EUDSLGenshape
452419
EUDSLGentensor
453420
EUDSLGentosa
454-
EUDSLGentransform
455-
EUDSLGenub
456-
# EUDSLGenvector
457-
EUDSLGenx86vector
458-
# EUDSLGenxegpu
459421
)
460422
set_target_properties(eudslpy_ext
461423
PROPERTIES
@@ -503,7 +465,8 @@ target_compile_options(eudslpy_ext PRIVATE ${nanobind_options})
503465
target_compile_options(nanobind-static PRIVATE ${nanobind_options})
504466

505467
# note WORKING_DIRECTORY
506-
set(NB_STUBGEN_CMD "${Python_EXECUTABLE}" "-m" "nanobind.stubgen" --module eudslpy_ext --recursive --include-private --output-dir .)
468+
set(NB_STUBGEN_CMD "${Python_EXECUTABLE}" "-m" "nanobind.stubgen"
469+
--module eudslpy_ext --recursive --include-private --output-dir .)
507470
set(NB_STUBGEN_OUTPUTS "__init__.pyi")
508471
add_custom_command(
509472
OUTPUT ${NB_STUBGEN_OUTPUTS}

‎projects/eudsl-py/cmake/EUDSLPYConfig.cmake

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
# copy-pasta from AddMLIR.cmake/AddLLVM.cmake/TableGen.cmake
22

3-
# Clear out any pre-existing compile_commands file before processing. This
4-
# allows for generating a clean compile_commands on each configure.
5-
file(REMOVE ${CMAKE_BINARY_DIR}/eudslpygen_compile_commands.yml)
6-
73
# no clue why but with LLVM_LINK_LLVM_DYLIB even static libs depend on LLVM
84
get_property(MLIR_ALL_LIBS GLOBAL PROPERTY MLIR_ALL_LIBS)
95
foreach(_lib ${MLIR_ALL_LIBS})
@@ -52,7 +48,7 @@ function(eudslpygen target inputFile)
5248
# Filter out empty items before prepending each entry with -I
5349
list(REMOVE_ITEM eudslpygen_includes "")
5450
list(TRANSFORM eudslpygen_includes PREPEND -I)
55-
51+
5652
set(_gen_target_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${target}")
5753
file(MAKE_DIRECTORY ${_gen_target_dir})
5854
set(fullGenFile "${_gen_target_dir}/${target}.cpp.gen")
@@ -63,12 +59,16 @@ function(eudslpygen target inputFile)
6359
# this could be smarter by asking people to list td targets or something but that's too onerous
6460
file(GLOB_RECURSE global_tds "${MLIR_INCLUDE_DIR}/mlir/*.td")
6561
# use cc -MM to collect all transitive headers
66-
set(clang_command ${CMAKE_CXX_COMPILER} -v -xc++ "-std=c++${CMAKE_CXX_STANDARD}"
62+
set(clang_command ${CMAKE_CXX_COMPILER}
63+
# -v
64+
-xc++ "-std=c++${CMAKE_CXX_STANDARD}"
6765
-MM ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
6866
-MT ${fullGenFile_rel}
6967
${eudslpygen_includes}
7068
-o ${_depfile})
71-
execute_process(COMMAND ${clang_command} RESULT_VARIABLE had_error COMMAND_ECHO STDERR)
69+
execute_process(COMMAND ${clang_command} RESULT_VARIABLE had_error
70+
# COMMAND_ECHO STDERR
71+
)
7272
if(had_error OR NOT EXISTS "${_depfile}")
7373
set(additional_cmdline -o "${fullGenFile_rel}")
7474
else()
@@ -126,36 +126,41 @@ function(eudslpygen target inputFile)
126126
DEPENDS ${ARG_DEPENDS} ${eudslpygen_depends} ${local_headers} ${global_tds}
127127
COMMENT "EUDSLPY: Generating ${fullGenFile}..."
128128
)
129+
# this is the specific thing connected the dependencies...
130+
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${fullGenFile})
131+
132+
# epic hack to specify all shards that will be generated even though we don't know them before hand
133+
# TODO(max): refactor eudslpy-gen into its own subproject so that we can do execute_process(CMAKE_COMMAND... )
134+
set(_byproducts)
135+
# lol spirv has 260 ops
136+
set(_max_num_shards 30)
137+
foreach(i RANGE ${_max_num_shards})
138+
list(APPEND _byproducts "${fullGenFile}.shard.${i}.cpp")
139+
endforeach()
129140

130141
add_custom_command(OUTPUT "${fullGenFile}.sharded.cpp"
131142
COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/make_generated_registration.py
132143
${fullGenFile} -t ${target} -I ${ARG_EXTRA_INCLUDES} ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
144+
-m ${_max_num_shards}
145+
BYPRODUCTS ${_byproducts}
133146
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
134147
DEPENDS ${fullGenFile}
135148
COMMENT "EUDSLPY: Generating ${fullGenFile}.sharded.cpp..."
136149
)
137-
138-
# this is the specific thing connected the dependencies...
139150
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${fullGenFile}.sharded.cpp")
140-
file(GLOB _generated_shards "${_gen_target_dir}/*.shard.*")
141-
list(APPEND _generated_shards "${fullGenFile}.sharded.cpp")
142-
set(${target}_GENERATED_SHARDS ${_generated_shards} PARENT_SCOPE)
151+
152+
add_library(${target} STATIC "${fullGenFile}.sharded.cpp" ${_byproducts})
153+
execute_process(
154+
COMMAND "${Python_EXECUTABLE}" -m nanobind --include_dir
155+
OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_include_dir)
156+
target_include_directories(${target} PRIVATE ${eudslpygen_includes}
157+
${Python_INCLUDE_DIRS} ${nanobind_include_dir})
143158

144159
# `make clean' must remove all those generated files:
145160
# TODO(max): clean up dep files
146-
set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${fullGenFile})
147-
set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${fullGenFile} PROPERTIES
148-
GENERATED 1)
149-
150-
# Append the includes used for this file to the pdll_compilation_commands
151-
# file.
152-
file(APPEND ${CMAKE_BINARY_DIR}/eudslpygen_compile_commands.yml
153-
"--- !FileInfo:\n"
154-
" filepath: \"${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}\"\n"
155-
" includes: \"${CMAKE_CURRENT_SOURCE_DIR};${eudslpygen_includes}\"\n"
156-
)
157-
158-
add_public_eudslpygen_target(${target} "${fullGenFile}.sharded.cpp;${_generated_shards}")
161+
set_property(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
162+
APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${_byproducts})
163+
set_source_files_properties(${_byproducts} PROPERTIES GENERATED 1)
159164
endfunction()
160165

161166
macro(add_eudslpygen target project)

‎projects/eudsl-py/cmake/make_generated_registration.py

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import argparse
22
import re
3+
import sys
34
from pathlib import Path
45
from textwrap import dedent
56

67

7-
def make_source_shards(filename: Path, target, extra_includes):
8+
def make_source_shards(filename: Path, target, extra_includes, max_num_shards):
89
assert filename.name.endswith("cpp.gen"), "expected .cpp.gen file"
910
with open(filename) as f:
1011
source = f.read()
@@ -33,6 +34,12 @@ def make_source_shards(filename: Path, target, extra_includes):
3334
print(shar, file=f)
3435
print("}", file=f)
3536

37+
if len(shards) > max_num_shards:
38+
raise RuntimeError("expected less than 20 shards")
39+
for i in range(len(shards), max_num_shards):
40+
with open(f"{filename}.shard.{i}.cpp", "w") as f:
41+
print(f"// dummy shard {i}", file=f)
42+
3643
with open(f"{filename}.sharded.cpp", "w") as f:
3744
print(
3845
dedent(
@@ -45,7 +52,10 @@ def make_source_shards(filename: Path, target, extra_includes):
4552
file=f,
4653
)
4754
for i in range(len(shards)):
48-
print(dedent(f"extern void populate{target}{i}Module(nb::module_ &m);"), file=f)
55+
print(
56+
dedent(f"extern void populate{target}{i}Module(nb::module_ &m);"),
57+
file=f,
58+
)
4959
for i in range(len(shards)):
5060
print(dedent(f"populate{target}{i}Module(m);"), file=f)
5161

@@ -57,5 +67,8 @@ def make_source_shards(filename: Path, target, extra_includes):
5767
parser.add_argument("filename")
5868
parser.add_argument("-t", "--target")
5969
parser.add_argument("-I", "--extra_includes", nargs="*")
70+
parser.add_argument("-m", "--max-num-shards", type=int, default=20)
6071
args = parser.parse_args()
61-
make_source_shards(Path(args.filename), args.target, args.extra_includes)
72+
make_source_shards(
73+
Path(args.filename), args.target, args.extra_includes, args.max_num_shards + 1
74+
)

0 commit comments

Comments
 (0)