Refactor the test suite.

shoumikhin · facebook-github-bot · commit 1abeccea86d3 · 2024-10-02T16:59:11.000-07:00
Summary: Factor out all the dynamism and resource files lookup to make that be reusable to other tests.

Reviewed By: kirklandsign

Differential Revision: D63801081

fbshipit-source-id: e0f59e2ded1fc7a4821858349cc9e205d0d6f75a
diff --git a/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj b/extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj
@@ -8,7 +8,9 @@
 
 /* Begin PBXBuildFile section */
 		03B2D3682C8A515A0046936E /* App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03B2D3672C8A515A0046936E /* App.swift */; };
-		03B2D37A2C8A515C0046936E /* Tests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 03B2D3792C8A515C0046936E /* Tests.mm */; };
+		03B0118E2CAC567900054791 /* DynamicTestCase.m in Sources */ = {isa = PBXBuildFile; fileRef = 03B0118C2CAC567900054791 /* DynamicTestCase.m */; };
+		03B011912CAD114E00054791 /* ResourceTestCase.m in Sources */ = {isa = PBXBuildFile; fileRef = 03B011902CAD114E00054791 /* ResourceTestCase.m */; };
+		03B2D37A2C8A515C0046936E /* GenericTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 03B2D3792C8A515C0046936E /* GenericTests.mm */; };
 		03C7FA382C8AA3EC00E6E9AE /* Models in Resources */ = {isa = PBXBuildFile; fileRef = 03C7FA322C8AA24200E6E9AE /* Models */; };
 		03DD00A92C8FE44600FE4619 /* backend_coreml.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03DD00992C8FE44600FE4619 /* backend_coreml.xcframework */; };
 		03DD00AA2C8FE44600FE4619 /* kernels_custom.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03DD009A2C8FE44600FE4619 /* kernels_custom.xcframework */; };
@@ -24,7 +26,6 @@
 		03ED6D152C8AAFFF00F2D6EE /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D142C8AAFFF00F2D6EE /* Metal.framework */; };
 		03ED6D172C8AB00500F2D6EE /* CoreML.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D162C8AB00500F2D6EE /* CoreML.framework */; };
 		03ED6D192C8AB00A00F2D6EE /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 03ED6D182C8AB00A00F2D6EE /* Accelerate.framework */; };
-		8493389C2C9918950071ABAD /* UIKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8493389B2C9918950071ABAD /* UIKit.framework */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -44,7 +45,11 @@
 		03B2D3672C8A515A0046936E /* App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App.swift; sourceTree = "<group>"; };
 		03B2D36D2C8A515B0046936E /* App.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = App.entitlements; sourceTree = "<group>"; };
 		03B2D3752C8A515C0046936E /* Tests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = Tests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
-		03B2D3792C8A515C0046936E /* Tests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = Tests.mm; sourceTree = "<group>"; };
+		03B0118B2CAC567900054791 /* DynamicTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DynamicTestCase.h; sourceTree = "<group>"; };
+		03B0118C2CAC567900054791 /* DynamicTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = DynamicTestCase.m; sourceTree = "<group>"; };
+		03B0118F2CAD114E00054791 /* ResourceTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ResourceTestCase.h; sourceTree = "<group>"; };
+		03B011902CAD114E00054791 /* ResourceTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ResourceTestCase.m; sourceTree = "<group>"; };
+		03B2D3792C8A515C0046936E /* GenericTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = GenericTests.mm; sourceTree = "<group>"; };
 		03C7FA322C8AA24200E6E9AE /* Models */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Models; sourceTree = SOURCE_ROOT; };
 		03DD00992C8FE44600FE4619 /* backend_coreml.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = backend_coreml.xcframework; path = Frameworks/backend_coreml.xcframework; sourceTree = "<group>"; };
 		03DD009A2C8FE44600FE4619 /* kernels_custom.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = kernels_custom.xcframework; path = Frameworks/kernels_custom.xcframework; sourceTree = "<group>"; };
@@ -79,7 +84,6 @@
 				03ED6D172C8AB00500F2D6EE /* CoreML.framework in Frameworks */,
 				03ED6D152C8AAFFF00F2D6EE /* Metal.framework in Frameworks */,
 				03ED6D132C8AAFF700F2D6EE /* MetalPerformanceShaders.framework in Frameworks */,
-				8493389C2C9918950071ABAD /* UIKit.framework in Frameworks */,
 				03ED6D112C8AAFF200F2D6EE /* MetalPerformanceShadersGraph.framework in Frameworks */,
 				03ED6D0F2C8AAFE900F2D6EE /* libsqlite3.0.tbd in Frameworks */,
 				03DD00A92C8FE44600FE4619 /* backend_coreml.xcframework in Frameworks */,
@@ -103,6 +107,7 @@
 				03ED6CEB2C8AAF5300F2D6EE /* Frameworks */,
 				03C7FA322C8AA24200E6E9AE /* Models */,
 				03B2D3782C8A515C0046936E /* Tests */,
+				03B0118D2CAC567900054791 /* TestUtils */,
 				03B2D3652C8A515A0046936E /* Products */,
 			);
 			sourceTree = "<group>";
@@ -128,13 +133,24 @@
 		03B2D3782C8A515C0046936E /* Tests */ = {
 			isa = PBXGroup;
 			children = (
-				03B2D3792C8A515C0046936E /* Tests.mm */,
+				03B2D3792C8A515C0046936E /* GenericTests.mm */,
 				03B019502C8A80D30044D558 /* Tests.xcconfig */,
 				037C96A02C8A570B00B3DF38 /* Tests.xctestplan */,
 			);
 			path = Tests;
 			sourceTree = SOURCE_ROOT;
 		};
+		03B0118D2CAC567900054791 /* TestUtils */ = {
+			isa = PBXGroup;
+			children = (
+				03B0118B2CAC567900054791 /* DynamicTestCase.h */,
+				03B0118C2CAC567900054791 /* DynamicTestCase.m */,
+				03B0118F2CAD114E00054791 /* ResourceTestCase.h */,
+				03B011902CAD114E00054791 /* ResourceTestCase.m */,
+			);
+			path = TestUtils;
+			sourceTree = "<group>";
+		};
 		03ED6CEB2C8AAF5300F2D6EE /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
@@ -264,7 +280,9 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				03B2D37A2C8A515C0046936E /* Tests.mm in Sources */,
+				03B0118E2CAC567900054791 /* DynamicTestCase.m in Sources */,
+				03B011912CAD114E00054791 /* ResourceTestCase.m in Sources */,
+				03B2D37A2C8A515C0046936E /* GenericTests.mm in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -414,6 +432,7 @@
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.Benchmark;
 				PRODUCT_NAME = Benchmark;
+				REGISTER_APP_GROUPS = NO;
 				SDKROOT = auto;
 				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
 				SUPPORTS_MACCATALYST = NO;
@@ -448,6 +467,7 @@
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.Benchmark;
 				PRODUCT_NAME = Benchmark;
+				REGISTER_APP_GROUPS = NO;
 				SDKROOT = auto;
 				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
 				SUPPORTS_MACCATALYST = NO;
@@ -472,6 +492,7 @@
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.BenchmarkTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";
+				REGISTER_APP_GROUPS = NO;
 				SDKROOT = auto;
 				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
 				SUPPORTS_MACCATALYST = NO;
@@ -497,6 +518,7 @@
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.pytorch.executorch.BenchmarkTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";
+				REGISTER_APP_GROUPS = NO;
 				SDKROOT = auto;
 				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
 				SUPPORTS_MACCATALYST = NO;
diff --git a/extension/apple/Benchmark/TestUtils/DynamicTestCase.h b/extension/apple/Benchmark/TestUtils/DynamicTestCase.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#import <XCTest/XCTest.h>
+
+/**
+ * DynamicTestCase is a subclass of XCTestCase that allows dynamic creation of
+ * test methods. Subclasses should override the `+dynamicTests` method to
+ * provide a dictionary of test names and corresponding test blocks.
+ */
+@interface DynamicTestCase : XCTestCase
+
+/**
+ * Returns a dictionary mapping test names to test blocks.
+ * Subclasses should override this method to provide dynamic tests.
+ */
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTests;
+
+@end
diff --git a/extension/apple/Benchmark/TestUtils/DynamicTestCase.m b/extension/apple/Benchmark/TestUtils/DynamicTestCase.m
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#import "DynamicTestCase.h"
+
+#import <objc/runtime.h>
+#import <sys/utsname.h>
+
+#if TARGET_OS_IOS
+#import <UIKit/UIDevice.h>
+#endif
+
+static NSString *deviceInfoString(void) {
+  static NSString *deviceInfo;
+  static dispatch_once_t onceToken;
+  dispatch_once(&onceToken, ^{
+    struct utsname systemInfo;
+    uname(&systemInfo);
+#if TARGET_OS_IOS
+    UIDevice *device = UIDevice.currentDevice;
+    deviceInfo = [NSString stringWithFormat:@"%@_%@_%@",
+                                            device.systemName,
+                                            device.systemVersion,
+                                            @(systemInfo.machine)];
+#elif TARGET_OS_MAC
+        NSOperatingSystemVersion version = NSProcessInfo.processInfo.operatingSystemVersion;
+        deviceInfo = [NSString stringWithFormat:@"macOS_%ld_%ld_%ld_%@",
+                      (long)version.majorVersion,
+                      (long)version.minorVersion, (long)version.patchVersion, @(systemInfo.machine)];
+#endif // TARGET_OS_IOS
+    deviceInfo = [[deviceInfo
+        componentsSeparatedByCharactersInSet:[NSCharacterSet
+                                                 punctuationCharacterSet]]
+        componentsJoinedByString:@"_"];
+  });
+  return deviceInfo;
+}
+
+@implementation DynamicTestCase
+
++ (void)initialize {
+  if (self != [DynamicTestCase class]) {
+    NSString *deviceInfo = deviceInfoString();
+    [[self dynamicTests]
+        enumerateKeysAndObjectsUsingBlock:^(NSString *testName,
+                                            void (^testCase)(XCTestCase *),
+                                            BOOL __unused *stop) {
+          NSString *methodName =
+              [NSString stringWithFormat:@"test_%@_%@", testName, deviceInfo];
+          class_addMethod(self,
+                          NSSelectorFromString(methodName),
+                          imp_implementationWithBlock(testCase),
+                          "v@:");
+        }];
+  }
+}
+
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTests {
+  return @{};
+}
+
+@end
diff --git a/extension/apple/Benchmark/TestUtils/ResourceTestCase.h b/extension/apple/Benchmark/TestUtils/ResourceTestCase.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#import "DynamicTestCase.h"
+
+/**
+ * ResourceTestCase is a subclass of DynamicTestCase that generates tests based
+ * on resources. Subclasses should override the bundle, directories, predicates,
+ * and dynamicTestsForResources methods.
+ */
+@interface ResourceTestCase : DynamicTestCase
+
+/**
+ * Returns an array of NSBundle objects to search for resources.
+ * By default, returns the main bundle and the bundle for the class.
+ */
++ (NSArray<NSBundle *> *)bundles;
+
+/**
+ * Returns an array of directory paths (relative to the bundles' resource paths)
+ * to search. Subclasses should override to specify directories containing
+ * resources.
+ */
++ (NSArray<NSString *> *)directories;
+
+/**
+ * Returns a dictionary mapping resource keys to predicates.
+ * Each predicate is a block that takes a filename and returns a BOOL indicating
+ * a match. Subclasses should override to specify predicates for matching
+ * resources.
+ */
++ (NSDictionary<NSString *, BOOL (^)(NSString *)> *)predicates;
+
+/**
+ * Returns a dictionary mapping test names to test blocks, given a dictionary of
+ * resources. Subclasses should override to provide tests for combinations of
+ * resources.
+ *
+ * @param resources A dictionary mapping resource keys to resource file paths.
+ * @return A dictionary mapping test names to test blocks.
+ */
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTestsForResources:
+    (NSDictionary<NSString *, NSString *> *)resources;
+
+@end
diff --git a/extension/apple/Benchmark/TestUtils/ResourceTestCase.m b/extension/apple/Benchmark/TestUtils/ResourceTestCase.m
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#import "ResourceTestCase.h"
+
+static void generateCombinations(
+    NSDictionary<NSString *, NSArray<NSString *> *> *matchesByResource,
+    NSArray<NSString *> *keys,
+    NSMutableDictionary<NSString *, NSString *> *result,
+    NSUInteger index,
+    NSMutableSet<NSDictionary<NSString *, NSString *> *> *combinations) {
+  if (index == keys.count) {
+    if (result.count == keys.count) {
+      [combinations addObject:[result copy]];
+    }
+    return;
+  }
+  NSString *key = keys[index];
+  NSArray<NSString *> *matches = matchesByResource[key] ?: @[];
+  if (!matches.count) {
+    generateCombinations(
+        matchesByResource, keys, result, index + 1, combinations);
+    return;
+  }
+  for (NSString *match in matches) {
+    result[key] = match;
+    generateCombinations(
+        matchesByResource, keys, result, index + 1, combinations);
+    [result removeObjectForKey:key];
+  }
+}
+
+@implementation ResourceTestCase
+
++ (NSArray<NSBundle *> *)bundles {
+  return @[ [NSBundle mainBundle], [NSBundle bundleForClass:self] ];
+}
+
++ (NSArray<NSString *> *)directories {
+  return @[];
+}
+
++ (NSDictionary<NSString *, BOOL (^)(NSString *)> *)predicates {
+  return @{};
+}
+
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTestsForResources:
+    (NSDictionary<NSString *, NSString *> *)resources {
+  return @{};
+}
+
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTests {
+  NSMutableDictionary<NSString *, void (^)(XCTestCase *)> *tests =
+      [NSMutableDictionary new];
+  NSMutableSet<NSDictionary<NSString *, NSString *> *> *combinations =
+      [NSMutableSet new];
+  NSDictionary<NSString *, BOOL (^)(NSString *)> *predicates =
+      [self predicates];
+  NSArray<NSString *> *sortedKeys =
+      [predicates.allKeys sortedArrayUsingSelector:@selector(compare:)];
+
+  if (predicates.count == 0)
+    return @{};
+
+  for (NSBundle *bundle in self.bundles) {
+    for (NSString *directory in self.directories) {
+      NSMutableDictionary<NSString *, NSMutableArray<NSString *> *>
+          *matchesByResource = [NSMutableDictionary new];
+      NSString *dirPath =
+          [bundle.resourcePath stringByAppendingPathComponent:directory];
+      NSArray<NSString *> *files =
+          [NSFileManager.defaultManager contentsOfDirectoryAtPath:dirPath
+                                                            error:nil]
+              ?: @[];
+      for (NSString *file in files) {
+        NSString *fullPath = [dirPath stringByAppendingPathComponent:file];
+        for (NSString *key in sortedKeys) {
+          if (predicates[key](file)) {
+            matchesByResource[key] =
+                matchesByResource[key] ?: [NSMutableArray new];
+            [matchesByResource[key] addObject:fullPath];
+          }
+        }
+      }
+      NSMutableDictionary<NSString *, NSString *> *result =
+          [NSMutableDictionary new];
+      generateCombinations(
+          matchesByResource, sortedKeys, result, 0, combinations);
+    }
+  }
+  for (NSDictionary<NSString *, NSString *> *resources in combinations) {
+    NSMutableString *resourceString = [NSMutableString new];
+    NSCharacterSet *punctuationSet = [NSCharacterSet punctuationCharacterSet];
+    for (NSString *key in sortedKeys) {
+      NSString *lastComponent = [resources[key] lastPathComponent];
+      NSString *cleanedComponent =
+          [[lastComponent componentsSeparatedByCharactersInSet:punctuationSet]
+              componentsJoinedByString:@"_"];
+      [resourceString appendFormat:@"_%@", cleanedComponent];
+    }
+    NSDictionary<NSString *, void (^)(XCTestCase *)> *resourceTests =
+        [self dynamicTestsForResources:resources];
+    [resourceTests
+        enumerateKeysAndObjectsUsingBlock:^(
+            NSString *testName, void (^testBlock)(XCTestCase *), BOOL *stop) {
+          tests[[testName stringByAppendingString:resourceString]] = testBlock;
+        }];
+  }
+  return tests;
+}
+
+@end
diff --git a/extension/apple/Benchmark/Tests/GenericTests.mm b/extension/apple/Benchmark/Tests/GenericTests.mm
diff --git a/extension/apple/Benchmark/Tests/Tests.mm b/extension/apple/Benchmark/Tests/Tests.mm
