Merge in MicroPython 1.15

.git-blame-ignore-revs

@@ -0,0 +1,32 @@
+# tests/run-tests.py: Reformat with Black.
+2a38d7103672580882fb621a5b76e8d26805d593
+
+# all: Update Python code to conform to latest black formatting.
+06659077a81b85882254cf0953c33b27614e018e
+
+# tools/uncrustify: Enable more opts to remove space between func and '('.
+77ed6f69ac35c1663a5633a8ee1d8a2446542204
+
+# tools/codeformat.py: Include extmod/{btstack,nimble} in code formatting.
+026fda605e03113d6e753290d65fed774418bc53
+
+# all: Format code to add space after C++-style comment start.
+84fa3312cfa7d2237d4b56952f2cd6e3591210c4
+
+# tests: Format all Python code with black, except tests in basics subdir.
+3dc324d3f1312e40d3a8ed87e7244966bb756f26
+
+# all: Remove spaces inside and around parenthesis.
+1a3e386c67e03a79eb768cb6e9f6777e002d6660
+
+# all: Remove spaces between nested paren and inside function arg paren.
+feb25775851ba0c04b8d1013716f442258879d9c
+
+# all: Reformat C and Python source code with tools/codeformat.py.
+69661f3343bedf86e514337cff63d96cc42f8859
+
+# stm32/usbdev: Convert files to unix line endings.
+abde0fa2267f9062b28c3c015d7662a550125cc6
+
+# all: Remove trailing spaces, per coding conventions.
+761e4c7ff62896c7d8f8c3dfc3cc98a4cc4f2f6f

.github/workflows/build.yml

@@ -73,16 +73,16 @@ jobs:
       run: |
         make -C ports/unix VARIANT=coverage -j2
     - name: Test all
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests -j1
+      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1
       working-directory: tests
     - name: Native Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests -j1 --emit native
+      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native
       working-directory: tests
     - name: mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests -j1 --mpy-cross-flags='-mcache-lookup-bc' --via-mpy -d basics float micropython
+      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --mpy-cross-flags='-mcache-lookup-bc' --via-mpy -d basics float micropython
       working-directory: tests
     - name: Native mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests -j1 --mpy-cross-flags='-mcache-lookup-bc' --via-mpy --emit native -d basics float micropython
+      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --mpy-cross-flags='-mcache-lookup-bc' --via-mpy --emit native -d basics float micropython
       working-directory: tests
     - name: Build mpy-cross.static-aarch64
       run: make -C mpy-cross -j2 -f Makefile.static-aarch64

docs/library/uasyncio.rst

@@ -40,6 +40,10 @@ Core functions
 
     Returns the corresponding `Task` object.
 
+.. function:: current_task()
+
+    Return the `Task` object associated with the currently running task.
+
 .. function:: run(coro)
 
     Create a new task from the given coroutine and run it until it completes.

extmod/extmod.cmake

@@ -0,0 +1,93 @@
+# CMake fragment for MicroPython extmod component
+
+set(MICROPY_EXTMOD_DIR "${MICROPY_DIR}/extmod")
+set(MICROPY_OOFATFS_DIR "${MICROPY_DIR}/lib/oofatfs")
+
+set(MICROPY_SOURCE_EXTMOD
+    ${MICROPY_DIR}/lib/embed/abort_.c
+    ${MICROPY_DIR}/lib/utils/printf.c
+    ${MICROPY_EXTMOD_DIR}/machine_i2c.c
+    ${MICROPY_EXTMOD_DIR}/machine_mem.c
+    ${MICROPY_EXTMOD_DIR}/machine_pulse.c
+    ${MICROPY_EXTMOD_DIR}/machine_signal.c
+    ${MICROPY_EXTMOD_DIR}/machine_spi.c
+    ${MICROPY_EXTMOD_DIR}/modbluetooth.c
+    ${MICROPY_EXTMOD_DIR}/modbtree.c
+    ${MICROPY_EXTMOD_DIR}/modframebuf.c
+    ${MICROPY_EXTMOD_DIR}/modonewire.c
+    ${MICROPY_EXTMOD_DIR}/moduasyncio.c
+    ${MICROPY_EXTMOD_DIR}/modubinascii.c
+    ${MICROPY_EXTMOD_DIR}/moducryptolib.c
+    ${MICROPY_EXTMOD_DIR}/moductypes.c
+    ${MICROPY_EXTMOD_DIR}/moduhashlib.c
+    ${MICROPY_EXTMOD_DIR}/moduheapq.c
+    ${MICROPY_EXTMOD_DIR}/modujson.c
+    ${MICROPY_EXTMOD_DIR}/modurandom.c
+    ${MICROPY_EXTMOD_DIR}/modure.c
+    ${MICROPY_EXTMOD_DIR}/moduselect.c
+    ${MICROPY_EXTMOD_DIR}/modussl_axtls.c
+    ${MICROPY_EXTMOD_DIR}/modussl_mbedtls.c
+    ${MICROPY_EXTMOD_DIR}/modutimeq.c
+    ${MICROPY_EXTMOD_DIR}/moduwebsocket.c
+    ${MICROPY_EXTMOD_DIR}/moduzlib.c
+    ${MICROPY_EXTMOD_DIR}/modwebrepl.c
+    ${MICROPY_EXTMOD_DIR}/uos_dupterm.c
+    ${MICROPY_EXTMOD_DIR}/utime_mphal.c
+    ${MICROPY_EXTMOD_DIR}/vfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_blockdev.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c
+    ${MICROPY_EXTMOD_DIR}/vfs_lfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_posix.c
+    ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c
+    ${MICROPY_EXTMOD_DIR}/vfs_reader.c
+    ${MICROPY_EXTMOD_DIR}/virtpin.c
+    ${MICROPY_EXTMOD_DIR}/nimble/modbluetooth_nimble.c
+)
+
+# Library for btree module and associated code
+
+set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
+
+if(EXISTS "${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c")
+    add_library(micropy_extmod_btree OBJECT
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
+        ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
+    )
+
+    target_include_directories(micropy_extmod_btree PRIVATE
+        ${MICROPY_LIB_BERKELEY_DIR}/PORT/include
+    )
+
+    target_compile_definitions(micropy_extmod_btree PRIVATE
+        __DBINTERFACE_PRIVATE=1
+        mpool_error=printf
+        abort=abort_
+        "virt_fd_t=void*"
+    )
+
+    # The include directories and compile definitions below are needed to build
+    # modbtree.c and should be added to the main MicroPython target.
+
+    list(APPEND MICROPY_INC_CORE
+        "${MICROPY_LIB_BERKELEY_DIR}/PORT/include"
+    )
+
+    list(APPEND MICROPY_DEF_CORE
+        __DBINTERFACE_PRIVATE=1
+        "virt_fd_t=void*"
+    )
+endif()

extmod/moduselect.c

@@ -128,7 +128,7 @@ STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) {
         // poll the objects
         mp_uint_t n_ready = poll_map_poll(&poll_map, rwx_len);
 
-        if (n_ready > 0 || (timeout != -1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
+        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
             // one or more objects are ready, or we had a timeout
             mp_obj_t list_array[3];
             list_array[0] = mp_obj_new_list(rwx_len[0], NULL);
@@ -230,7 +230,7 @@ STATIC mp_uint_t poll_poll_internal(uint n_args, const mp_obj_t *args) {
     for (;;) {
         // poll the objects
         n_ready = poll_map_poll(&self->poll_map, NULL);
-        if (n_ready > 0 || (timeout != -1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
+        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_tick >= timeout)) {
             break;
         }
         MICROPY_EVENT_POLL_HOOK

extmod/uasyncio/__init__.py

@@ -10,6 +10,7 @@
     "wait_for_ms": "funcs",
     "gather": "funcs",
     "Event": "event",
+    "ThreadSafeFlag": "event",
     "Lock": "lock",
     "open_connection": "stream",
     "start_server": "stream",

extmod/uasyncio/core.py

@@ -264,6 +264,10 @@ def get_event_loop(runq_len=0, waitq_len=0):
     return Loop
 
 
+def current_task():
+    return cur_task
+
+
 def new_event_loop():
     global _task_queue, _io_queue
     # TaskQueue of Task instances

extmod/uasyncio/event.py

@@ -14,6 +14,8 @@ def is_set(self):
 
     def set(self):
         # Event becomes set, schedule any tasks waiting on it
+        # Note: This must not be called from anything except the thread running
+        # the asyncio loop (i.e. neither hard or soft IRQ, or a different thread).
         while self.waiting.peek():
             core._task_queue.push_head(self.waiting.pop_head())
         self.state = True

extmod/vfs_posix_file.c

@@ -137,7 +137,11 @@ STATIC mp_uint_t vfs_posix_file_write(mp_obj_t o_in, const void *buf, mp_uint_t
 
 STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
     mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in);
-    check_fd_is_open(o);
+
+    if (request != MP_STREAM_CLOSE) {
+        check_fd_is_open(o);
+    }
+
     switch (request) {
         case MP_STREAM_FLUSH: {
             int ret;

lib/utils/gchelper.h

@@ -39,6 +39,8 @@ typedef uintptr_t gc_helper_regs_t[6];
 typedef uintptr_t gc_helper_regs_t[4];
 #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__)
 typedef uintptr_t gc_helper_regs_t[10];
+#elif defined(__aarch64__)
+typedef uintptr_t gc_helper_regs_t[11]; // x19-x29
 #endif
 
 #endif

lib/utils/gchelper_generic.c

@@ -123,6 +123,33 @@ STATIC void gc_helper_get_regs(gc_helper_regs_t arr) {
     arr[9] = r13;
 }
 
+#elif defined(__aarch64__)
+
+STATIC void gc_helper_get_regs(gc_helper_regs_t arr) {
+    const register long x19 asm ("x19");
+    const register long x20 asm ("x20");
+    const register long x21 asm ("x21");
+    const register long x22 asm ("x22");
+    const register long x23 asm ("x23");
+    const register long x24 asm ("x24");
+    const register long x25 asm ("x25");
+    const register long x26 asm ("x26");
+    const register long x27 asm ("x27");
+    const register long x28 asm ("x28");
+    const register long x29 asm ("x29");
+    arr[0] = x19;
+    arr[1] = x20;
+    arr[2] = x21;
+    arr[3] = x22;
+    arr[4] = x23;
+    arr[5] = x24;
+    arr[6] = x25;
+    arr[7] = x26;
+    arr[8] = x27;
+    arr[9] = x28;
+    arr[10] = x29;
+}
+
 #else
 
 #error "Architecture not supported for gc_helper_get_regs. Set MICROPY_GCREGS_SETJMP to use the fallback implementation."

mpy-cross/main.c

@@ -149,7 +149,7 @@ STATIC void pre_process_options(int argc, char **argv) {
                         heap_size *= 1024 * 1024;
                     }
                     if (word_adjust) {
-                        heap_size = heap_size * BYTES_PER_WORD / 4;
+                        heap_size = heap_size * MP_BYTES_PER_OBJ_WORD / 4;
                     }
                 } else {
                     exit(usage(argv));
@@ -161,7 +161,7 @@ STATIC void pre_process_options(int argc, char **argv) {
 }
 
 MP_NOINLINE int main_(int argc, char **argv) {
-    mp_stack_set_limit(40000 * (BYTES_PER_WORD / 4));
+    mp_stack_set_limit(40000 * (sizeof(void *) / 4));
 
     pre_process_options(argc, argv);
 

ports/unix/Makefile

@@ -226,26 +226,23 @@ include $(TOP)/py/mkrules.mk
 
 .PHONY: test test_full
 
-test: $(PROG) $(TOP)/tests/run-tests
+test: $(PROG) $(TOP)/tests/run-tests.py
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --auto-jobs
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --auto-jobs
 
-test_full: $(PROG) $(TOP)/tests/run-tests
+test_full: $(PROG) $(TOP)/tests/run-tests.py
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests -d thread
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --emit native
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) -d basics float micropython
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py -d thread
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --emit native
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) -d basics float micropython
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython
 	cat $(TOP)/tests/basics/0prelim.py | ./$(PROG) | grep -q 'abc'
 
 test_gcov: test_full
 	gcov -o $(BUILD)/py $(TOP)/py/*.c
 	gcov -o $(BUILD)/extmod $(TOP)/extmod/*.c
 
-coverage_clean:
-	$(MAKE) V=2 BUILD=build-coverage PROG=micropython_coverage clean
-
 # build an interpreter for fuzzing
 fuzz:
 	$(MAKE) \

ports/unix/main.c

@@ -340,6 +340,9 @@ STATIC int invalid_args(void) {
 STATIC void pre_process_options(int argc, char **argv) {
     for (int a = 1; a < argc; a++) {
         if (argv[a][0] == '-') {
+            if (strcmp(argv[a], "-c") == 0 || strcmp(argv[a], "-m") == 0) {
+                break; // Everything after this is a command/module and arguments for it
+            }
             if (strcmp(argv[a], "-h") == 0) {
                 print_help(argv);
                 exit(0);
@@ -386,7 +389,7 @@ STATIC void pre_process_options(int argc, char **argv) {
                         goto invalid_arg;
                     }
                     if (word_adjust) {
-                        heap_size = heap_size * BYTES_PER_WORD / 4;
+                        heap_size = heap_size * MP_BYTES_PER_OBJ_WORD / 4;
                     }
                     // If requested size too small, we'll crash anyway
                     if (heap_size < 700) {
@@ -399,6 +402,8 @@ STATIC void pre_process_options(int argc, char **argv) {
                 }
                 a++;
             }
+        } else {
+            break; // Not an option but a file
         }
     }
 }
@@ -445,7 +450,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
     signal(SIGPIPE, SIG_IGN);
     #endif
 
-    mp_stack_set_limit(40000 * (BYTES_PER_WORD / 4));
+    mp_stack_set_limit(40000 * (sizeof(void *) / 4));
 
     pre_process_options(argc, argv);
 
@@ -571,11 +576,10 @@ MP_NOINLINE int main_(int argc, char **argv) {
                 if (a + 1 >= argc) {
                     return invalid_args();
                 }
+                set_sys_argv(argv, a + 1, a); // The -c becomes first item of sys.argv, as in CPython
+                set_sys_argv(argv, argc, a + 2); // Then what comes after the command
                 ret = do_str(argv[a + 1]);
-                if (ret & FORCED_EXIT) {
-                    break;
-                }
-                a += 1;
+                break;
             } else if (strcmp(argv[a], "-m") == 0) {
                 if (a + 1 >= argc) {
                     return invalid_args();
@@ -595,7 +599,12 @@ MP_NOINLINE int main_(int argc, char **argv) {
 
                 mp_obj_t mod;
                 nlr_buf_t nlr;
-                bool subpkg_tried = false;
+
+                // Allocating subpkg_tried on the stack can lead to compiler warnings about this
+                // variable being clobbered when nlr is implemented using setjmp/longjmp.  Its
+                // value must be preserved across calls to setjmp/longjmp.
+                static bool subpkg_tried;
+                subpkg_tried = false;
 
             reimport:
                 if (nlr_push(&nlr) == 0) {

ports/unix/moduselect.c

@@ -29,6 +29,10 @@
 
 #if MICROPY_PY_USELECT_POSIX
 
+#if MICROPY_PY_USELECT
+#error "Can't have both MICROPY_PY_USELECT and MICROPY_PY_USELECT_POSIX."
+#endif
+
 #include <stdio.h>
 #include <errno.h>
 #include <poll.h>