Skip to content

Commit 862f462

Browse files
author
Jamie Smith
authored
Use a virtual environment for Python packages by default (ARMmbed#261)
* Use a virtual environment for Python packages by default * Install python3-venv in CI * Don't use sudo * Oops missed a "STATUS" * Fix second build dir not using venv
1 parent b219302 commit 862f462

File tree

6 files changed

+115
-30
lines changed

6 files changed

+115
-30
lines changed

‎.github/workflows/basic_checks.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,10 +214,10 @@ jobs:
214214
name: Checkout repo
215215
uses: actions/checkout@v3
216216

217-
218-
- name: Install Python packages
217+
- name: Install python3-venv
219218
run: |
220-
python3 -m pip install -r tools/requirements.txt
219+
apt-get update
220+
apt-get install -y python3-venv
221221
222222
-
223223
name: cmake build

‎.github/workflows/greentea_cmake.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,10 @@ jobs:
1010
- name: Checkout
1111
uses: actions/checkout@v3
1212

13-
- name: Install Python packages
13+
- name: Install python3-venv
1414
run: |
15-
python3 -m pip install -r tools/requirements.txt
15+
apt-get update
16+
apt-get install -y python3-venv
1617
1718
- name: Build NUCLEO_G031K8 with baremetal profile
1819
run: |

‎.github/workflows/test_building_multiple_executables.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,19 @@ jobs:
99
steps:
1010
- name: Checkout
1111
uses: actions/checkout@v3
12-
12+
1313
- name: Install Python packages
1414
run: |
15-
python3 -m pip install -r tools/requirements.txt
15+
python3 -m pip install -r tools/requirements.txt
1616
17+
# Note: For this CI job we use MBED_CREATE_PYTHON_VENV=FALSE so that we can make sure
18+
# this mode works.
1719
- name: Build the multiple_executables example
1820
run: |
1921
cd tools/cmake/tests/multiple_executables/
2022
mkdir cmake_build
2123
cd cmake_build
22-
cmake -DMBED_TARGET=ARM_MUSCA_S1 ..
24+
cmake -DMBED_TARGET=ARM_MUSCA_S1 -DMBED_CREATE_PYTHON_VENV=FALSE ..
2325
cmake --build .
2426
2527
- name: Verify the post-build command has run successfully on each image

‎tools/cmake/app.cmake

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,28 +11,7 @@ if(CCACHE)
1111
endif()
1212

1313
# Find Python (needed to generate configurations)
14-
find_package(Python3 REQUIRED COMPONENTS Interpreter)
15-
include(CheckPythonPackage)
16-
17-
# Check python packages
18-
set(PYTHON_PACKAGES_TO_CHECK intelhex prettytable future jinja2)
19-
foreach(PACKAGE_NAME ${PYTHON_PACKAGES_TO_CHECK})
20-
string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UCASE) # Ucase name needed for CMake variable
21-
string(TOLOWER ${PACKAGE_NAME} PACKAGE_NAME_LCASE) # Lcase name needed for import statement
22-
23-
check_python_package(${PACKAGE_NAME_LCASE} HAVE_PYTHON_${PACKAGE_NAME_UCASE})
24-
if(NOT HAVE_PYTHON_${PACKAGE_NAME_UCASE})
25-
message(WARNING "Missing Python dependency ${PACKAGE_NAME}")
26-
endif()
27-
endforeach()
28-
29-
# Check deps for memap
30-
if(Python3_FOUND AND HAVE_PYTHON_INTELHEX AND HAVE_PYTHON_PRETTYTABLE)
31-
set(HAVE_MEMAP_DEPS TRUE)
32-
else()
33-
set(HAVE_MEMAP_DEPS FALSE)
34-
message(STATUS "Missing Python dependencies (at least one of: python3, intelhex, prettytable) so the memory map cannot be printed")
35-
endif()
14+
include(mbed_python_interpreter)
3615

3716
include(mbed_generate_config_header)
3817
include(mbed_target_functions)

‎tools/cmake/mbed_generate_configuration.cmake

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ set(MBED_INTERNAL_LAST_MBED_TARGET "${MBED_TARGET}" CACHE INTERNAL "Previous mbe
3232
get_filename_component(MBED_APP_JSON_PATH "${MBED_APP_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
3333
get_filename_component(CUSTOM_TARGETS_JSON_PATH "${CUSTOM_TARGETS_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
3434

35+
# Make it so that if mbed_app.json or custom_targets.json are modified, CMake is rerun.
36+
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_APP_JSON_PATH})
37+
if(EXISTS "${CUSTOM_TARGETS_JSON_PATH}" AND (NOT IS_DIRECTORY "${CUSTOM_TARGETS_JSON_PATH}"))
38+
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CUSTOM_TARGETS_JSON_PATH})
39+
endif()
40+
3541
# Check if mbed_app.json was modified
3642
# Note: if the path is an empty string, get_filename_component(ABSOLUTE) will convert it to a directory,
3743
# so we have to verify that the path we have is a file, not a dir.
@@ -92,6 +98,7 @@ if(MBED_NEED_TO_RECONFIGURE)
9298

9399
set(MBEDTOOLS_CONFIGURE_COMMAND ${Python3_EXECUTABLE}
94100
-c "import mbed_tools.cli.main\; exit(mbed_tools.cli.main.cli())" # This is used instead of invoking mbed_tools as a script, because it might not be on the user's PATH.
101+
-v # without -v, warnings (e.g. "you have tried to override a nonexistent parameter") do not get printed
95102
configure
96103
-t GCC_ARM # GCC_ARM is currently the only supported toolchain
97104
-m "${MBED_TARGET}"

‎tools/cmake/mbed_python_interpreter.cmake

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Copyright (c) 2024 ARM Limited. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# CMake script to find the Python interpreter and either install or find
5+
# Mbed's dependencies.
6+
7+
option(MBED_CREATE_PYTHON_VENV "If true, Mbed OS will create its own virtual environment (venv) and install its Python packages there. This removes the need to manually install Python packages." TRUE)
8+
9+
if(MBED_CREATE_PYTHON_VENV)
10+
# Use the venv.
11+
12+
# Note: venv is stored in the source directory as it can be shared between all the build directories
13+
# (not target specific)
14+
set(MBED_VENV_LOCATION ${CMAKE_SOURCE_DIR}/mbed-os/venv)
15+
set(VENV_STAMP_FILE ${MBED_VENV_LOCATION}/mbed-venv.stamp)
16+
set(MBED_REQUIREMENTS_TXT_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../requirements.txt")
17+
18+
# Find Python3, using the venv if it already exists
19+
set (ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
20+
set (Python3_FIND_VIRTUALENV FIRST)
21+
find_package(Python3 REQUIRED COMPONENTS Interpreter)
22+
include(CheckPythonPackage)
23+
24+
set(NEED_TO_CREATE_VENV FALSE)
25+
set(NEED_TO_INSTALL_PACKAGES FALSE)
26+
if(NOT EXISTS "${VENV_STAMP_FILE}")
27+
set(NEED_TO_CREATE_VENV TRUE)
28+
set(NEED_TO_INSTALL_PACKAGES TRUE)
29+
elseif("${MBED_REQUIREMENTS_TXT_LOCATION}" IS_NEWER_THAN "${VENV_STAMP_FILE}")
30+
set(NEED_TO_INSTALL_PACKAGES TRUE)
31+
endif()
32+
33+
if(NEED_TO_CREATE_VENV)
34+
# Create venv.
35+
# Using approach from here: https://discourse.cmake.org/t/possible-to-create-a-python-virtual-env-from-cmake-and-then-find-it-with-findpython3/1132/2
36+
message(STATUS "Mbed: Creating virtual environment with Python interpreter ${Python3_EXECUTABLE}")
37+
execute_process(
38+
COMMAND ${Python3_EXECUTABLE} -m venv ${MBED_VENV_LOCATION}
39+
COMMAND_ERROR_IS_FATAL ANY
40+
)
41+
42+
## Reset FindPython3 cache variables so it will run again
43+
unset(Python3_EXECUTABLE)
44+
unset(_Python3_EXECUTABLE CACHE)
45+
unset(_Python3_INTERPRETER_PROPERTIES CACHE)
46+
unset(_Python3_INTERPRETER_SIGNATURE CACHE)
47+
## Launch a new search for Python3
48+
find_package (Python3 REQUIRED COMPONENTS Interpreter)
49+
endif()
50+
51+
if(NEED_TO_INSTALL_PACKAGES)
52+
message(STATUS "Mbed: Installing Python requirements for Mbed into venv")
53+
# Upgrade pip first in case it needs an upgrade, to prevent a warning being printed
54+
execute_process(
55+
COMMAND ${Python3_EXECUTABLE} -m pip install --upgrade pip
56+
COMMAND_ERROR_IS_FATAL ANY
57+
)
58+
execute_process(
59+
COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MBED_REQUIREMENTS_TXT_LOCATION}
60+
COMMAND_ERROR_IS_FATAL ANY
61+
)
62+
63+
message(STATUS "Mbed: venv created successfully")
64+
file(TOUCH ${VENV_STAMP_FILE})
65+
endif()
66+
67+
# We always have the memap deps with the venv
68+
set(HAVE_MEMAP_DEPS TRUE)
69+
70+
else()
71+
72+
find_package(Python3 REQUIRED COMPONENTS Interpreter)
73+
include(CheckPythonPackage)
74+
75+
76+
# Check python packages
77+
set(PYTHON_PACKAGES_TO_CHECK intelhex prettytable future jinja2)
78+
foreach(PACKAGE_NAME ${PYTHON_PACKAGES_TO_CHECK})
79+
string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UCASE) # Ucase name needed for CMake variable
80+
string(TOLOWER ${PACKAGE_NAME} PACKAGE_NAME_LCASE) # Lcase name needed for import statement
81+
82+
check_python_package(${PACKAGE_NAME_LCASE} HAVE_PYTHON_${PACKAGE_NAME_UCASE})
83+
if(NOT HAVE_PYTHON_${PACKAGE_NAME_UCASE})
84+
message(WARNING "Missing Python dependency ${PACKAGE_NAME}")
85+
endif()
86+
endforeach()
87+
88+
# Check deps for memap
89+
if(Python3_FOUND AND HAVE_PYTHON_INTELHEX AND HAVE_PYTHON_PRETTYTABLE)
90+
set(HAVE_MEMAP_DEPS TRUE)
91+
else()
92+
set(HAVE_MEMAP_DEPS FALSE)
93+
message(STATUS "Missing Python dependencies (at least one of: python3, intelhex, prettytable) so the memory map cannot be printed")
94+
endif()
95+
96+
endif()

0 commit comments

Comments
 (0)