Skip to content

Commit 42bc306

Browse files
pytorchbotlucylq
pytorchbot
and
lucylq
authored
Use generated files in module test (#10597)
Instead of checking in the PTE/PTD files. This helps us keep up to date with aot changes, especially with incoming flat_tensor refactor. Main changes here are with the model. Instead of using AddModule from examples/portable, use ModuleAdd from test/models/export_program, like our other tests. There is a shape difference (1 tensor vs 2 tensors+alpha), so we have to change all the inputs. Differential Revision: [D73470865](https://our.internmc.facebook.com/intern/diff/D73470865/) This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: #10497 by @lucylq ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/lucylq/67/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/lucylq/67/head Merge bot PR base: https://github.com/pytorch/executorch/tree/main Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/lucylq/67/orig @diff-train-skip-merge Co-authored-by: lucylq <[email protected]>
1 parent 8f6de2c commit 42bc306

File tree

10 files changed

+135
-132
lines changed

10 files changed

+135
-132
lines changed

‎extension/android/executorch_android/android_test_setup.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ which "${PYTHON_EXECUTABLE}"
1515
BASEDIR=$(dirname "$(realpath $0)")
1616

1717
prepare_add() {
18-
cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources"
18+
python3 -m test.models.export_program --modules "ModuleAdd" --outdir "${BASEDIR}/src/androidTest/resources/"
1919
}
2020

2121
prepare_tinyllama() {

‎extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
/** Unit tests for {@link Module}. */
3939
@RunWith(AndroidJUnit4.class)
4040
public class ModuleInstrumentationTest {
41-
private static String TEST_FILE_NAME = "/add.pte";
41+
private static String TEST_FILE_NAME = "/ModuleAdd.pte";
4242
private static String MISSING_FILE_NAME = "/missing.pte";
4343
private static String NON_PTE_FILE_NAME = "/test.txt";
4444
private static String FORWARD_METHOD = "forward";

‎extension/module/test/CMakeLists.txt

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,32 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)
1919

2020
set(_test_srcs module_test.cpp)
2121

22+
add_custom_command(
23+
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
24+
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
25+
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
26+
COMMAND
27+
python3 -m test.models.export_program --modules "ModuleAdd"
28+
--outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
29+
COMMAND
30+
python3 -m test.models.export_program --modules "ModuleLinear"
31+
--external-constants --outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
32+
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
33+
)
34+
35+
add_custom_target(
36+
generated_module_test_files
37+
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
38+
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
39+
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
40+
)
41+
42+
set(test_env
43+
"ET_MODULE_ADD_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
44+
"ET_MODULE_LINEAR_PROGRAM_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
45+
"ET_MODULE_LINEAR_DATA_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
46+
)
47+
2248
et_cxx_test(
2349
extension_module_test
2450
SOURCES
@@ -31,7 +57,8 @@ et_cxx_test(
3157
portable_ops_lib
3258
)
3359

34-
set(test_env "RESOURCES_PATH=${EXECUTORCH_ROOT}/extension/module/test/resources")
60+
add_dependencies(extension_module_test generated_module_test_files)
61+
set_property(TEST extension_module_test PROPERTY ENVIRONMENT ${test_env})
3562

3663
set_property(
3764
TEST extension_module_test

‎extension/module/test/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ load(":targets.bzl", "define_common_targets")
55

66
oncall("executorch")
77

8-
define_common_targets()
8+
define_common_targets(is_fbcode = True)

‎extension/module/test/module_test.cpp

Lines changed: 72 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,17 @@
1515

1616
#include <executorch/extension/data_loader/file_data_loader.h>
1717
#include <executorch/extension/tensor/tensor.h>
18+
#include <executorch/runtime/core/exec_aten/testing_util/tensor_util.h>
1819

1920
using namespace ::executorch::extension;
2021
using namespace ::executorch::runtime;
2122

2223
class ModuleTest : public ::testing::Test {
2324
protected:
2425
static void SetUpTestSuite() {
25-
std::string resources_path;
26-
if (const char* env = std::getenv("RESOURCES_PATH")) {
27-
resources_path = env;
28-
}
29-
model_path_ = resources_path + "/add.pte";
30-
linear_path_ = resources_path + "/linear.pte";
31-
linear_data_path_ = resources_path + "/linear.ptd";
26+
model_path_ = std::getenv("ET_MODULE_ADD_PATH");
27+
linear_path_ = std::getenv("ET_MODULE_LINEAR_PROGRAM_PATH");
28+
linear_data_path_ = std::getenv("ET_MODULE_LINEAR_DATA_PATH");
3229
}
3330

3431
static inline std::string model_path_;
@@ -109,22 +106,29 @@ TEST_F(ModuleTest, TestMethodMeta) {
109106
const auto meta = module.method_meta("forward");
110107
EXPECT_EQ(meta.error(), Error::Ok);
111108
EXPECT_STREQ(meta->name(), "forward");
112-
EXPECT_EQ(meta->num_inputs(), 2);
109+
// tensor, tensor, alpha
110+
EXPECT_EQ(meta->num_inputs(), 3);
113111
EXPECT_EQ(*(meta->input_tag(0)), Tag::Tensor);
114112
EXPECT_EQ(meta->num_outputs(), 1);
115113
EXPECT_EQ(*(meta->output_tag(0)), Tag::Tensor);
116114

117115
const auto input_meta = meta->input_tensor_meta(0);
118116
EXPECT_EQ(input_meta.error(), Error::Ok);
119117
EXPECT_EQ(input_meta->scalar_type(), executorch::aten::ScalarType::Float);
120-
EXPECT_EQ(input_meta->sizes().size(), 1);
121-
EXPECT_EQ(input_meta->sizes()[0], 1);
118+
EXPECT_EQ(input_meta->sizes().size(), 2);
119+
EXPECT_EQ(input_meta->sizes()[0], 2);
120+
121+
const auto input_meta1 = meta->input_tensor_meta(1);
122+
EXPECT_EQ(input_meta1.error(), Error::Ok);
123+
EXPECT_EQ(input_meta1->scalar_type(), executorch::aten::ScalarType::Float);
124+
EXPECT_EQ(input_meta1->sizes().size(), 2);
125+
EXPECT_EQ(input_meta1->sizes()[0], 2);
122126

123127
const auto output_meta = meta->output_tensor_meta(0);
124128
EXPECT_EQ(output_meta.error(), Error::Ok);
125129
EXPECT_EQ(output_meta->scalar_type(), executorch::aten::ScalarType::Float);
126-
EXPECT_EQ(output_meta->sizes().size(), 1);
127-
EXPECT_EQ(output_meta->sizes()[0], 1);
130+
EXPECT_EQ(output_meta->sizes().size(), 2);
131+
EXPECT_EQ(output_meta->sizes()[0], 2);
128132
}
129133

130134
TEST_F(ModuleTest, TestNonExistentMethodMeta) {
@@ -136,17 +140,16 @@ TEST_F(ModuleTest, TestNonExistentMethodMeta) {
136140

137141
TEST_F(ModuleTest, TestExecute) {
138142
Module module(model_path_);
139-
auto tensor = make_tensor_ptr({1.f});
143+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
140144

141-
const auto result = module.execute("forward", {tensor, tensor});
145+
const auto result = module.execute("forward", {tensor, tensor, 1.0});
142146
EXPECT_EQ(result.error(), Error::Ok);
143147

144148
EXPECT_TRUE(module.is_loaded());
145149
EXPECT_TRUE(module.is_method_loaded("forward"));
146150

147-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
148-
149-
EXPECT_NEAR(data[0], 2, 1e-5);
151+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
152+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
150153
}
151154

152155
TEST_F(ModuleTest, TestExecutePreload) {
@@ -155,14 +158,13 @@ TEST_F(ModuleTest, TestExecutePreload) {
155158
const auto error = module.load();
156159
EXPECT_EQ(error, Error::Ok);
157160

158-
auto tensor = make_tensor_ptr({1.f});
161+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
159162

160-
const auto result = module.execute("forward", {tensor, tensor});
163+
const auto result = module.execute("forward", {tensor, tensor, 1.0});
161164
EXPECT_EQ(result.error(), Error::Ok);
162165

163-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
164-
165-
EXPECT_NEAR(data[0], 2, 1e-5);
166+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
167+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
166168
}
167169

168170
TEST_F(ModuleTest, TestExecutePreload_method) {
@@ -171,14 +173,13 @@ TEST_F(ModuleTest, TestExecutePreload_method) {
171173
const auto error = module.load_method("forward");
172174
EXPECT_EQ(error, Error::Ok);
173175

174-
auto tensor = make_tensor_ptr({1.f});
176+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
175177

176-
const auto result = module.execute("forward", {tensor, tensor});
178+
const auto result = module.execute("forward", {tensor, tensor, 1.0});
177179
EXPECT_EQ(result.error(), Error::Ok);
178180

179-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
180-
181-
EXPECT_NEAR(data[0], 2, 1e-5);
181+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
182+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
182183
}
183184

184185
TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
@@ -190,14 +191,13 @@ TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
190191
const auto load_method_error = module.load_method("forward");
191192
EXPECT_EQ(load_method_error, Error::Ok);
192193

193-
auto tensor = make_tensor_ptr({1.f});
194+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
194195

195-
const auto result = module.execute("forward", {tensor, tensor});
196+
const auto result = module.execute("forward", {tensor, tensor, 1.0});
196197
EXPECT_EQ(result.error(), Error::Ok);
197198

198-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
199-
200-
EXPECT_NEAR(data[0], 2, 1e-5);
199+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
200+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
201201
}
202202

203203
TEST_F(ModuleTest, TestExecuteOnNonExistent) {
@@ -219,32 +219,30 @@ TEST_F(ModuleTest, TestExecuteOnCurrupted) {
219219
TEST_F(ModuleTest, TestGet) {
220220
Module module(model_path_);
221221

222-
auto tensor = make_tensor_ptr({1.f});
222+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
223223

224-
const auto result = module.get("forward", {tensor, tensor});
224+
const auto result = module.get("forward", {tensor, tensor, 1.0});
225225
EXPECT_EQ(result.error(), Error::Ok);
226-
const auto data = result->toTensor().const_data_ptr<float>();
227-
EXPECT_NEAR(data[0], 2, 1e-5);
226+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
227+
EXPECT_TENSOR_CLOSE(result->toTensor(), *expected.get());
228228
}
229229

230230
TEST_F(ModuleTest, TestForward) {
231231
auto module = std::make_unique<Module>(model_path_);
232-
auto tensor = make_tensor_ptr({21.f});
232+
auto tensor = make_tensor_ptr({2, 2}, {21.f, 22.f, 23.f, 24.f});
233233

234-
const auto result = module->forward({tensor, tensor});
234+
const auto result = module->forward({tensor, tensor, 1.0});
235235
EXPECT_EQ(result.error(), Error::Ok);
236236

237-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
237+
const auto expected = make_tensor_ptr({2, 2}, {42.f, 44.f, 46.f, 48.f});
238+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
238239

239-
EXPECT_NEAR(data[0], 42, 1e-5);
240-
241-
auto tensor2 = make_tensor_ptr({2.f});
240+
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});
242241
const auto result2 = module->forward({tensor2, tensor2});
243242
EXPECT_EQ(result2.error(), Error::Ok);
244243

245-
const auto data2 = result->at(0).toTensor().const_data_ptr<float>();
246-
247-
EXPECT_NEAR(data2[0], 4, 1e-5);
244+
const auto expected2 = make_tensor_ptr({2, 2}, {4.f, 6.f, 8.f, 10.f});
245+
EXPECT_TENSOR_CLOSE(result2->at(0).toTensor(), *expected2.get());
248246
}
249247

250248
TEST_F(ModuleTest, TestForwardWithInvalidInputs) {
@@ -294,20 +292,20 @@ TEST_F(ModuleTest, TestProgramSharingAndDataLoaderManagement) {
294292
EXPECT_EQ(load_error, Error::Ok);
295293
EXPECT_TRUE(module1->is_loaded());
296294

297-
auto tensor = make_tensor_ptr({1.f});
295+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
298296

299-
const auto result1 = module1->execute("forward", {tensor, tensor});
297+
const auto result1 = module1->execute("forward", {tensor, tensor, 1.0});
300298
EXPECT_EQ(result1.error(), Error::Ok);
301299

302300
auto module2 = std::make_unique<Module>(module1->program());
303301

304-
const auto result2 = module2->execute("forward", {tensor, tensor});
302+
const auto result2 = module2->execute("forward", {tensor, tensor, 1.0});
305303
EXPECT_EQ(result2.error(), Error::Ok);
306304

307305
module1 = std::make_unique<Module>("/path/to/nonexistent/file.pte");
308306
EXPECT_FALSE(module1->is_loaded());
309307

310-
const auto result3 = module2->execute("forward", {tensor, tensor});
308+
const auto result3 = module2->execute("forward", {tensor, tensor, 1.0});
311309
EXPECT_EQ(result3.error(), Error::Ok);
312310
}
313311

@@ -339,14 +337,13 @@ TEST_F(ModuleTest, TestProgramPersistenceAndReuseAfterModuleDestruction) {
339337

340338
EXPECT_EQ(module.program(), shared_program);
341339

342-
auto tensor = make_tensor_ptr({1.f});
340+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
343341

344-
const auto result = module.execute("forward", {tensor, tensor});
342+
const auto result = module.execute("forward", {tensor, tensor, 1.0});
345343
EXPECT_EQ(result.error(), Error::Ok);
346344

347-
auto data = result->at(0).toTensor().const_data_ptr<float>();
348-
349-
EXPECT_NEAR(data[0], 2, 1e-5);
345+
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
346+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
350347
}
351348

352349
TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
@@ -364,22 +361,22 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
364361
EXPECT_TRUE(program != nullptr);
365362

366363
auto thread = [](std::shared_ptr<Program> program,
367-
const std::array<float, 1>& input) {
364+
const std::array<float, 4>& input) {
368365
Module module(program);
369-
auto tensor = from_blob((void*)input.data(), {1});
366+
auto tensor = from_blob((void*)input.data(), {2, 2});
370367

371-
const auto result = module.forward({tensor, tensor});
368+
const auto result = module.forward({tensor, tensor, 1.0});
372369
EXPECT_EQ(result.error(), Error::Ok);
373370

374371
const auto data = result->at(0).toTensor().const_data_ptr<float>();
375372
EXPECT_NEAR(data[0], (input[0] * 2), 1e-5);
376373
};
377374

378-
std::thread t1(thread, program, std::array<float, 1>{1});
379-
std::thread t2(thread, program, std::array<float, 1>{2});
380-
std::thread t3(thread, program, std::array<float, 1>{3});
381-
std::thread t4(thread, program, std::array<float, 1>{4});
382-
std::thread t5(thread, program, std::array<float, 1>{5});
375+
std::thread t1(thread, program, std::array<float, 4>{1, 2, 3, 4});
376+
std::thread t2(thread, program, std::array<float, 4>{2, 3, 4, 5});
377+
std::thread t3(thread, program, std::array<float, 4>{3, 4, 5, 6});
378+
std::thread t4(thread, program, std::array<float, 4>{4, 5, 6, 7});
379+
std::thread t5(thread, program, std::array<float, 4>{5, 6, 7, 8});
383380

384381
t1.join();
385382
t2.join();
@@ -391,37 +388,38 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
391388
TEST_F(ModuleTest, TestSetInputsBeforeExecute) {
392389
Module module(model_path_);
393390

394-
auto tensor1 = make_tensor_ptr({4.f});
395-
auto tensor2 = make_tensor_ptr({5.f});
391+
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
392+
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});
396393

397-
EXPECT_EQ(module.set_inputs({tensor1, tensor2}), Error::Ok);
394+
EXPECT_EQ(module.set_inputs({tensor1, tensor2, 1.0}), Error::Ok);
398395

399396
const auto result = module.forward();
400397
EXPECT_EQ(result.error(), Error::Ok);
401398

402-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
403-
EXPECT_NEAR(data[0], 9, 1e-5);
399+
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
400+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
404401
}
405402

406403
TEST_F(ModuleTest, TestSetInputCombinedWithExecute) {
407404
Module module(model_path_);
408405

409-
auto tensor1 = make_tensor_ptr({2.f});
410-
auto tensor2 = make_tensor_ptr({3.f});
406+
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
407+
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});
411408

412409
EXPECT_EQ(module.set_input(tensor2, 1), Error::Ok);
410+
EXPECT_EQ(module.set_input(1.0, 2), Error::Ok); // alpha
413411

414412
const auto result = module.forward(tensor1);
415413
EXPECT_EQ(result.error(), Error::Ok);
416414

417-
const auto data = result->at(0).toTensor().const_data_ptr<float>();
418-
EXPECT_NEAR(data[0], 5, 1e-5);
415+
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
416+
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
419417
}
420418

421419
TEST_F(ModuleTest, TestPartiallySetInputs) {
422420
Module module(model_path_);
423421

424-
auto tensor = make_tensor_ptr({1.f});
422+
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
425423

426424
EXPECT_EQ(module.set_input(tensor, 0), Error::Ok);
427425

@@ -455,8 +453,6 @@ TEST_F(ModuleTest, TestPTD) {
455453

456454
ASSERT_EQ(module.load_method("forward"), Error::Ok);
457455

458-
auto tensor1 =
459-
make_tensor_ptr({3, 3}, {2.f, 3.f, 4.f, 2.f, 3.f, 4.f, 2.f, 3.f, 4.f});
460-
461-
ASSERT_EQ(module.forward(tensor1).error(), Error::Ok);
456+
auto tensor = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 2.f});
457+
ASSERT_EQ(module.forward(tensor).error(), Error::Ok);
462458
}

0 commit comments

Comments
 (0)