Add llama tests. (#5874)

Summary:
Pull Request resolved: #5874

.

Reviewed By: kirklandsign

Differential Revision: D63901612

fbshipit-source-id: 7baaa0fd5a83c2b2a067a0e36754e776d40f73f0

Author: shoumikhin

extension/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj

@@ -1,4 +1,4 @@
 # Resources
 
-This directory and all files in it will be copied to the bundle’s resources directory, accessible via `NSBundle.resourcePath`.
-Place any resource files you want to access at runtime here.
+This directory and all files in it will be copied to the bundle’s root directory.
+Place here any resource files you want to access at runtime.

extension/apple/Benchmark/Resources/README.md

@@ -69,16 +69,18 @@ @implementation ResourceTestCase
 
   for (NSBundle *bundle in self.bundles) {
     for (NSString *directory in self.directories) {
+      NSArray<NSURL *> *resourceURLs =
+          [bundle URLsForResourcesWithExtension:nil subdirectory:directory];
+      if (!resourceURLs.count) {
+        continue;
+      };
       NSMutableDictionary<NSString *, NSMutableArray<NSString *> *>
           *matchesByResource = [NSMutableDictionary new];
-      NSString *dirPath =
-          [bundle.resourcePath stringByAppendingPathComponent:directory];
-      NSArray<NSString *> *files =
-          [NSFileManager.defaultManager contentsOfDirectoryAtPath:dirPath
-                                                            error:nil]
-              ?: @[];
-      for (NSString *file in files) {
-        NSString *fullPath = [dirPath stringByAppendingPathComponent:file];
+
+      for (NSURL *url in resourceURLs) {
+        NSString *file = url.lastPathComponent;
+        NSString *fullPath = url.path;
+
         for (NSString *key in sortedKeys) {
           if (predicates[key](file)) {
             matchesByResource[key] =

extension/apple/Benchmark/TestUtils/ResourceTestCase.m

@@ -30,7 +30,7 @@ @implementation GenericTests
 + (NSArray<NSString *> *)directories {
   return @[
     @"Resources",
-    @"aatp/data", // AWS Farm devides look at this subdirectory.
+    @"aatp/data", // AWS Farm devices look for resources here.
   ];
 }
 

extension/apple/Benchmark/Tests/GenericTests.mm

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#import "ResourceTestCase.h"
+
+#import <executorch/examples/models/llama2/runner/runner.h>
+
+using namespace ::executorch::extension;
+using namespace ::executorch::runtime;
+
+@interface TokensPerSecondMetric : NSObject<XCTMetric>
+
+@property(nonatomic, assign) NSUInteger tokenCount;
+
+@end
+
+@implementation TokensPerSecondMetric
+
+- (id)copyWithZone:(NSZone *)zone {
+  TokensPerSecondMetric *copy = [[[self class] allocWithZone:zone] init];
+  copy.tokenCount = self.tokenCount;
+  return copy;
+}
+
+- (NSArray<XCTPerformanceMeasurement *> *)
+    reportMeasurementsFromStartTime:
+        (XCTPerformanceMeasurementTimestamp *)startTime
+                          toEndTime:
+                              (XCTPerformanceMeasurementTimestamp *)endTime
+                              error:(NSError **)error {
+  double elapsedTime =
+      (endTime.absoluteTimeNanoSeconds - startTime.absoluteTimeNanoSeconds) /
+      (double)NSEC_PER_SEC;
+  return @[ [[XCTPerformanceMeasurement alloc]
+      initWithIdentifier:NSStringFromClass([self class])
+             displayName:@"Tokens Per Second"
+             doubleValue:(self.tokenCount / elapsedTime)
+              unitSymbol:@"t/s"] ];
+}
+
+@end
+
+@interface LLaMATests : ResourceTestCase
+@end
+
+@implementation LLaMATests
+
++ (NSArray<NSString *> *)directories {
+  return @[
+    @"Resources",
+    @"aatp/data", // AWS Farm devices look for resources here.
+  ];
+}
+
++ (NSDictionary<NSString *, BOOL (^)(NSString *)> *)predicates {
+  return @{@"model" : ^BOOL(NSString *filename){
+      return [filename hasSuffix:@".pte"] && [filename containsString:@"llama"];
+}
+, @"tokenizer" : ^BOOL(NSString *filename) {
+  return [filename isEqual:@"tokenizer.model"];
+},
+}
+;
+}
+
++ (NSDictionary<NSString *, void (^)(XCTestCase *)> *)dynamicTestsForResources:
+    (NSDictionary<NSString *, NSString *> *)resources {
+  NSString *modelPath = resources[@"model"];
+  NSString *tokenizerPath = resources[@"tokenizer"];
+  return @{@"generate" : ^(XCTestCase *testCase){
+      auto __block runner = std::make_unique<example::Runner>(
+          modelPath.UTF8String, tokenizerPath.UTF8String);
+  const auto status = runner->load();
+  if (status != Error::Ok) {
+    XCTFail("Load failed with error %i", status);
+    return;
+  }
+  TokensPerSecondMetric *tokensPerSecondMetric = [TokensPerSecondMetric new];
+  [testCase measureWithMetrics:@[ tokensPerSecondMetric, [XCTMemoryMetric new] ]
+                         block:^{
+                           tokensPerSecondMetric.tokenCount = 0;
+                           const auto status = runner->generate(
+                               "Once upon a time",
+                               128,
+                               [=](const std::string &token) {
+                                 tokensPerSecondMetric.tokenCount++;
+                               },
+                               nullptr,
+                               false);
+                           XCTAssertEqual(status, Error::Ok);
+                         }];
+}
+}
+;
+}
+
+@end

extension/apple/Benchmark/Tests/LLaMATests.mm

@@ -1,4 +1,12 @@
+HEADER_SEARCH_PATHS[sdk=iphonesimulator*] = $(inherited) \
+$(SRCROOT)/../../../.. \
+$(TEMP_DIR)/cmake/include
+
+LIBRARY_SEARCH_PATHS[sdk=iphonesimulator*] = $(inherited) \
+$(TEMP_DIR)/cmake/lib
+
 OTHER_LDFLAGS[sdk=iphonesimulator*] = $(inherited) \
+@$(TEMP_DIR)/cmake/linker_flags \
 -force_load $(BUILT_PRODUCTS_DIR)/libexecutorch-simulator-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_coreml-simulator-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_mps-simulator-release.a \
@@ -7,7 +15,15 @@ OTHER_LDFLAGS[sdk=iphonesimulator*] = $(inherited) \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_optimized-simulator-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_quantized-simulator-release.a
 
+HEADER_SEARCH_PATHS[sdk=iphoneos*] = $(inherited) \
+$(SRCROOT)/../../../.. \
+$(TEMP_DIR)/cmake/include
+
+LIBRARY_SEARCH_PATHS[sdk=iphoneos*] = $(inherited) \
+$(TEMP_DIR)/cmake/lib
+
 OTHER_LDFLAGS[sdk=iphoneos*] = $(inherited) \
+@$(TEMP_DIR)/cmake/linker_flags \
 -force_load $(BUILT_PRODUCTS_DIR)/libexecutorch-ios-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_coreml-ios-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_mps-ios-release.a \
@@ -16,7 +32,15 @@ OTHER_LDFLAGS[sdk=iphoneos*] = $(inherited) \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_optimized-ios-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_quantized-ios-release.a
 
+HEADER_SEARCH_PATHS[sdk=macos*] = $(inherited) \
+$(SRCROOT)/../../../.. \
+$(TEMP_DIR)/cmake/include
+
+LIBRARY_SEARCH_PATHS[sdk=macos*] = $(inherited) \
+$(TEMP_DIR)/cmake/lib
+
 OTHER_LDFLAGS[sdk=macos*] = $(inherited) \
+@$(TEMP_DIR)/cmake/linker_flags \
 -force_load $(BUILT_PRODUCTS_DIR)/libexecutorch-macos-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_coreml-macos-release.a \
 -force_load $(BUILT_PRODUCTS_DIR)/libbackend_mps-macos-release.a \