Merge pull request #6999 from jepler/picow-ssl

pico_w: implement ssl with caveats

Author: dhalbert

.gitmodules

@@ -148,7 +148,7 @@
 	url = https://github.com/adafruit/esp-idf.git
 	branch = circuitpython8
 [submodule "ports/espressif/certificates/nina-fw"]
-	path = ports/espressif/certificates/nina-fw
+	path = lib/certificates/nina-fw
 	url = https://github.com/adafruit/nina-fw.git
 [submodule "frozen/Adafruit_CircuitPython_ST7789"]
 	path = frozen/Adafruit_CircuitPython_ST7789
@@ -316,3 +316,6 @@
 [submodule "ports/raspberrypi/lib/lwip"]
 	path = ports/raspberrypi/lib/lwip
 	url = https://github.com/lwip-tcpip/lwip.git
+[submodule "lib/mbedtls"]
+	path = lib/mbedtls
+	url = https://github.com/ARMmbed/mbedtls.git

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     -   id: end-of-file-fixer
         exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*|ports/espressif/esp-idf-config/.*|ports/espressif/boards/.*/sdkconfig)'
     -   id: trailing-whitespace
-        exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*)'
+        exclude: '^(tests/.*\.exp|tests/cmdline/.*|tests/.*/data/.*|lib/mbedtls_errors/.*)'
 -   repo: local
     hooks:
     -   id: translations

ports/espressif/certificates/README.md renamed to lib/certificates/README.md

@@ -0,0 +1,42 @@
+MBEDTLS Error Strings for MicroPython
+=====================================
+
+This directory contains source code and tools to rework the Mbedtls error strings for
+micropython to use less space. In short, instead of storing and printing something like
+"SSL - Our own certificate(s) is/are too large to send in an SSL message" it prints
+the name of the error #define, which would be "MBEDTLS_ERR_SSL_CERTIFICATE_TOO_LARGE" in
+this case, and only stores `SSL_CERTIFICATE_TOO_LARGE` in flash. The exact Mbedtls error
+defines are used because they're easy to search for to find more detailed information.
+
+Mbedtls defines a specific format for error value #defines and
+includes a Perl script to gather all `MBEDTLS_ERR` defines from includes files together with
+english error text. From that the Perl script generates `mbedtls_strerror()`. The files in this
+directory modify this process to produce a more space efficient error lookup table with
+shorter error strings.
+
+The files are as follows:
+- `generate_errors.diff` - diff for original mbedtls perl script
+- `error.fmt` - modified code template for MicroPython
+- `mp_mbedtls_errors.c` - source file with `mbedtls_strerror` this is built using the include
+  files in `../mbedtls`
+- `do-mp.sh` - shell script to produce `mp_mbedtls_errors.c`
+- `tester.c` - simple C main to test `mp_mbedtls_errors.c` locally on a dev box
+- `do-test.sh` - shell script to produce `mp_mbedtls_errors.c` and compile the `tester` app
+- `do-esp32.sh` - shell script to produce `esp32_mbedtls_errors.c` -- see below
+
+In order not to store multiple copies of `mbedtls_errors.c`
+([https://github.com/micropython/micropython/pull/5819#discussion_r445528006](see))
+it is assumed that all ports use the same version of mbedtls with the same error #defines.
+This is true as of MP v1.13, and ESP-IDF versions 3.3.2 and 4.0.1. If anything changes in the
+future the `do-esp32.sh` script can be used to generate an esp32-specific version.
+
+### How-to
+
+- To build MicroPython all that is needed is to include the `mp_mbedtls_errors.c` into the build
+  (the Makefiles do this automatically). Note that Perl is not needed for routine MicroPython
+  builds.
+- When a new version of Mbedtls is pulled-in the `do-mp.sh` script should be run to
+  re-generate `mp_mbedtls_errors.c`.
+- The `tester` app should be run if changes to the string handling in `error.fmt` are made:
+  it tests that there is not an off-by-one error in the string copying/appending, etc.
+- To include `mbedtls_strerror` error strings define `MBEDTLS_ERROR_C` in the build.

ports/espressif/certificates/nina-fw renamed to lib/certificates/nina-fw

@@ -0,0 +1,7 @@
+#! /bin/bash -e
+# Generate esp32_mbedtls_errors.c for use in the Esp32 port, with the ESP-IDF version of mbedtls
+# The IDF_PATH env var must be set to the top-level dir of ESPIDF
+echo "IDF_PATH=$IDF_PATH"
+MBEDTLS=$IDF_PATH/components/mbedtls/mbedtls
+patch -o esp32_generate_errors.pl $MBEDTLS/scripts/generate_errors.pl <generate_errors.diff
+perl ./esp32_generate_errors.pl $MBEDTLS/include/mbedtls . esp32_mbedtls_errors.c

lib/mbedtls

@@ -0,0 +1,4 @@
+#! /bin/bash -e
+# Generate mp_mbedtls_errors.c for inclusion in ports that use $MPY/lib/mbedtls
+patch -o mp_generate_errors.pl ../mbedtls/scripts/generate_errors.pl <generate_errors.diff
+perl ./mp_generate_errors.pl ../mbedtls/include/mbedtls . mp_mbedtls_errors.c

lib/mbedtls_errors/README.md

@@ -0,0 +1,4 @@
+#! /bin/bash -e
+# Generate mp_mbedtls_errors.c and build the tester app
+./do-mp.sh
+cc -o tester -I../mbedtls/include/ mp_mbedtls_errors.c tester.c

lib/mbedtls_errors/do-esp32.sh

@@ -0,0 +1,165 @@
+/*
+ *  Error message information
+ *
+ *  Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *  not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  This file is part of mbed TLS (https://tls.mbed.org)
+ */
+
+#if !defined(MBEDTLS_CONFIG_FILE)
+#include "mbedtls/config.h"
+#else
+#include MBEDTLS_CONFIG_FILE
+#endif
+
+#if defined(MBEDTLS_ERROR_C) || defined(MBEDTLS_ERROR_STRERROR_DUMMY)
+#include "mbedtls/error.h"
+#include <string.h>
+#endif
+
+#if defined(MBEDTLS_PLATFORM_C)
+#include "mbedtls/platform.h"
+#else
+#define mbedtls_snprintf snprintf
+#define mbedtls_time_t   time_t
+#endif
+
+#if defined(MBEDTLS_ERROR_C)
+
+#include <stdio.h>
+
+HEADER_INCLUDED
+
+// Error code table type
+struct ssl_errs {
+    int16_t errnum;
+    const char *errstr;
+};
+
+// Table of high level error codes
+static const struct ssl_errs mbedtls_high_level_error_tab[] = {
+// BEGIN generated code
+HIGH_LEVEL_CODE_CHECKS
+// END generated code
+};
+
+static const struct ssl_errs mbedtls_low_level_error_tab[] = {
+// Low level error codes
+//
+// BEGIN generated code
+LOW_LEVEL_CODE_CHECKS
+// END generated code
+};
+
+static const char *mbedtls_err_prefix = "MBEDTLS_ERR_";
+#define MBEDTLS_ERR_PREFIX_LEN ( sizeof("MBEDTLS_ERR_")-1 )
+
+// copy error text into buffer, ensure null termination, return strlen of result
+static size_t mbedtls_err_to_str(int err, const struct ssl_errs tab[], int tab_len, char *buf, size_t buflen) {
+    if (buflen == 0) return 0;
+
+    // prefix for all error names
+    strncpy(buf, mbedtls_err_prefix, buflen);
+    if (buflen <= MBEDTLS_ERR_PREFIX_LEN+1) {
+        buf[buflen-1] = 0;
+        return buflen-1;
+    }
+
+    // append error name from table
+    for (int i = 0; i < tab_len; i++) {
+        if (tab[i].errnum == err) {
+            strncpy(buf+MBEDTLS_ERR_PREFIX_LEN, tab[i].errstr, buflen-MBEDTLS_ERR_PREFIX_LEN);
+            buf[buflen-1] = 0;
+            return strlen(buf);
+        }
+    }
+
+    mbedtls_snprintf(buf+MBEDTLS_ERR_PREFIX_LEN, buflen-MBEDTLS_ERR_PREFIX_LEN, "UNKNOWN (0x%04X)",
+            err);
+    return strlen(buf);
+}
+
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+
+void mbedtls_strerror(int ret, char *buf, size_t buflen) {
+    int use_ret;
+
+    if (buflen == 0) return;
+
+    buf[buflen-1] = 0;
+
+    if (ret < 0) ret = -ret;
+
+    //
+    // High-level error codes
+    //
+    uint8_t got_hl = (ret & 0xFF80) != 0;
+    if (got_hl) {
+        use_ret = ret & 0xFF80;
+
+        // special case
+#if defined(MBEDTLS_SSL_TLS_C)
+        if (use_ret == -(MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE)) {
+            strncpy(buf, "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE", buflen);
+            buf[buflen-1] = 0;
+            return;
+        }
+#endif
+
+        size_t len = mbedtls_err_to_str(use_ret, mbedtls_high_level_error_tab,
+                ARRAY_SIZE(mbedtls_high_level_error_tab), buf, buflen);
+
+        buf += len;
+        buflen -= len;
+        if (buflen == 0) return;
+    }
+
+    //
+    // Low-level error codes
+    //
+    use_ret = ret & ~0xFF80;
+
+    if (use_ret == 0) return;
+
+    // If high level code is present, make a concatenation between both error strings.
+    if (got_hl) {
+        if (buflen < 2) return;
+        *buf++ = '+';
+        buflen--;
+    }
+
+    mbedtls_err_to_str(use_ret, mbedtls_low_level_error_tab,
+            ARRAY_SIZE(mbedtls_low_level_error_tab), buf, buflen);
+}
+
+#else /* MBEDTLS_ERROR_C */
+
+#if defined(MBEDTLS_ERROR_STRERROR_DUMMY)
+
+/*
+ * Provide an non-function in case MBEDTLS_ERROR_C is not defined
+ */
+void mbedtls_strerror( int ret, char *buf, size_t buflen )
+{
+    ((void) ret);
+
+    if( buflen > 0 )
+        buf[0] = '\0';
+}
+
+#endif /* MBEDTLS_ERROR_STRERROR_DUMMY */
+
+#endif /* MBEDTLS_ERROR_C */

lib/mbedtls_errors/do-mp.sh

@@ -0,0 +1,22 @@
+--- generate_errors_orig.pl	2020-06-20 08:40:38.819060379 -0700
++++ generate_errors.pl	2020-06-20 08:47:26.511163591 -0700
+@@ -162,16 +162,12 @@
+ 
+     if ($error_name eq "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE")
+     {
+-        ${$code_check} .= "${white_space}if( use_ret == -($error_name) )\n".
+-                          "${white_space}\{\n".
+-                          "${white_space}    mbedtls_snprintf( buf, buflen, \"$module_name - $description\" );\n".
+-                          "${white_space}    return;\n".
+-                          "${white_space}}\n"
++        # no-op, this case is hard-coded in error.fmt
+     }
+     else
+     {
+-        ${$code_check} .= "${white_space}if( use_ret == -($error_name) )\n".
+-                          "${white_space}    mbedtls_snprintf( buf, buflen, \"$module_name - $description\" );\n"
++         my $error_text = $error_name =~ s/^MBEDTLS_ERR_//r;
++         ${$code_check} .= "${white_space}{ -($error_name), \"$error_text\" },\n"
+     }
+ };
+