Merge pull request #6608 from tannewt/web_workflow_port

Allow for dynamic reconfigure including port

Author: tannewt

docs/environment.rst

@@ -39,6 +39,10 @@ CIRCUITPY_WEB_API_PASSWORD
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 Password required to make modifications to the board from the Web Workflow.
 
+CIRCUITPY_WEB_API_PORT
+~~~~~~~~~~~~~~~~~~~~~~
+TCP port number used for the web HTTP API. Defaults to 80 when omitted.
+
 CIRCUITPY_WIFI_PASSWORD
 ~~~~~~~~~~~~~~~~~~~~~~~
 Wi-Fi password used to auto connect to CIRCUITPY_WIFI_SSID.

docs/workflows.md

@@ -72,7 +72,7 @@ Read-only characteristic that returns the UTF-8 encoded version string.
 The web workflow is depends on adding Wi-Fi credentials into the `/.env` file. The keys are
 `CIRCUITPY_WIFI_SSID` and `CIRCUITPY_WIFI_PASSWORD`. Once these are defined, CircuitPython will
 automatically connect to the network and start the webserver used for the workflow. The webserver
-is on port 80. It also enables MDNS.
+is on port 80 unless overridden by `CIRCUITPY_WEB_API_PORT`. It also enables MDNS.
 
 Here is an example `/.env`:
 
@@ -83,6 +83,8 @@ CIRCUITPY_WIFI_PASSWORD='secretpassword'
 
 # To enable modifying files from the web. Change this too!
 CIRCUITPY_WEB_API_PASSWORD='passw0rd'
+
+CIRCUITPY_WEB_API_PORT=80
 ```
 
 MDNS is used to resolve [`circuitpython.local`](http://circuitpython.local) to a device specific

ports/espressif/common-hal/mdns/Server.c

@@ -204,5 +204,9 @@ mp_obj_t common_hal_mdns_server_find(mdns_server_obj_t *self, const char *servic
 }
 
 void common_hal_mdns_server_advertise_service(mdns_server_obj_t *self, const char *service_type, const char *protocol, mp_int_t port) {
-    mdns_service_add(NULL, service_type, protocol, port, NULL, 0);
+    if (mdns_service_exists(service_type, protocol, NULL)) {
+        mdns_service_port_set(service_type, protocol, port);
+    } else {
+        mdns_service_add(NULL, service_type, protocol, port, NULL, 0);
+    }
 }

shared-bindings/mdns/Server.c

@@ -161,6 +161,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mdns_server_find_obj, 1, _mdns_server_find);
 //|     def advertise_service(self, *,  service_type: str, protocol: str, port: int) -> None:
 //|         """Respond to queries for the given service with the given port.
 //|
+//|         ``service_type`` and ``protocol`` can only occur on one port. Any call after the first
+//|         will update the entry's port.
+//|
 //|         :param str service_type: The service type such as "_http"
 //|         :param str protocol: The service protocol such as "_tcp"
 //|         :param int port: The port used by the service"""

supervisor/shared/web_workflow/static/welcome.js

@@ -44,7 +44,7 @@ async function find_devices() {
         li.appendChild(a);
         var port = "";
         if (device.port != 80) {
-            port = ":" + version_info.port;
+            port = ":" + device.port;
         }
         var server;
         if (mdns_works) {

supervisor/shared/web_workflow/web_workflow.c

@@ -98,6 +98,7 @@ typedef struct {
 static wifi_radio_error_t wifi_status = WIFI_RADIO_ERROR_NONE;
 
 static mdns_server_obj_t mdns;
+static uint32_t web_api_port = 80;
 
 static socketpool_socketpool_obj_t pool;
 static socketpool_socket_obj_t listening;
@@ -189,6 +190,9 @@ void supervisor_web_workflow_status(void) {
             }
 
             mp_printf(&mp_plat_print, "%s", _our_ip_encoded);
+            if (web_api_port != 80) {
+                mp_printf(&mp_plat_print, ":%d", web_api_port);
+            }
             // TODO: Use these unicode to show signal strength: ▂▄▆█
         }
     } else {
@@ -199,11 +203,6 @@ void supervisor_web_workflow_status(void) {
 void supervisor_start_web_workflow(void) {
     #if CIRCUITPY_WEB_WORKFLOW && CIRCUITPY_WIFI
 
-    if (common_hal_wifi_radio_get_enabled(&common_hal_wifi_radio_obj) &&
-        wifi_radio_get_ipv4_address(&common_hal_wifi_radio_obj) != 0) {
-        // Already started.
-        return;
-    }
 
     char ssid[33];
     char password[64];
@@ -218,15 +217,20 @@ void supervisor_start_web_workflow(void) {
         password_len <= 0 || (size_t)password_len >= sizeof(password)) {
         return;
     }
-    common_hal_wifi_init(false);
-    common_hal_wifi_radio_set_enabled(&common_hal_wifi_radio_obj, true);
+    if (!common_hal_wifi_radio_get_enabled(&common_hal_wifi_radio_obj)) {
+        common_hal_wifi_init(false);
+        common_hal_wifi_radio_set_enabled(&common_hal_wifi_radio_obj, true);
+    }
 
     // TODO: Do our own scan so that we can find the channel we want before calling connect.
     // Otherwise, connect will do a full slow scan to pick the best AP.
 
     // NUL terminate the strings because dotenv doesn't.
     ssid[ssid_len] = '\0';
     password[password_len] = '\0';
+    // We can all connect again because it will return early if we're already connected to the
+    // network. If we are connected to a different network, then it will disconnect before
+    // attempting to connect to the given network.
     wifi_status = common_hal_wifi_radio_connect(
         &common_hal_wifi_radio_obj, (uint8_t *)ssid, ssid_len, (uint8_t *)password, password_len,
         0, 0.1, NULL, 0);
@@ -236,21 +240,47 @@ void supervisor_start_web_workflow(void) {
         return;
     }
 
-    mdns_server_construct(&mdns, true);
-    common_hal_mdns_server_set_instance_name(&mdns, MICROPY_HW_BOARD_NAME);
-    common_hal_mdns_server_advertise_service(&mdns, "_circuitpython", "_tcp", 80);
+    char port_encoded[6];
+    size_t port_len = 0;
+    size_t new_port = web_api_port;
+    #if CIRCUITPY_DOTENV
+    port_len = dotenv_get_key("/.env", "CIRCUITPY_WEB_API_PORT", port_encoded, sizeof(port_encoded) - 1);
+    #endif
+    if (0 < port_len && port_len < sizeof(port_encoded)) {
+        port_encoded[port_len] = '\0';
+        new_port = strtoul(port_encoded, NULL, 10);
+    }
+
+    bool first_start = mdns.base.type != &mdns_server_type;
+    bool port_changed = new_port != web_api_port;
 
-    pool.base.type = &socketpool_socketpool_type;
-    common_hal_socketpool_socketpool_construct(&pool, &common_hal_wifi_radio_obj);
+    if (first_start) {
+        ESP_LOGI(TAG, "Starting web workflow");
+        mdns_server_construct(&mdns, true);
+        mdns.base.type = &mdns_server_type;
+        common_hal_mdns_server_set_instance_name(&mdns, MICROPY_HW_BOARD_NAME);
+        pool.base.type = &socketpool_socketpool_type;
+        common_hal_socketpool_socketpool_construct(&pool, &common_hal_wifi_radio_obj);
 
-    ESP_LOGI(TAG, "Starting web workflow");
-    listening.base.type = &socketpool_socket_type;
-    socketpool_socket(&pool, SOCKETPOOL_AF_INET, SOCKETPOOL_SOCK_STREAM, &listening);
-    common_hal_socketpool_socket_settimeout(&listening, 0);
-    // Bind to any ip.
-    // TODO: Make this port .env configurable.
-    common_hal_socketpool_socket_bind(&listening, "", 0, 80);
-    common_hal_socketpool_socket_listen(&listening, 1);
+        listening.base.type = &socketpool_socket_type;
+        active.base.type = &socketpool_socket_type;
+        active.num = -1;
+        active.connected = false;
+
+        websocket_init();
+    }
+    if (port_changed) {
+        common_hal_socketpool_socket_close(&listening);
+    }
+    if (first_start || port_changed) {
+        web_api_port = new_port;
+        common_hal_mdns_server_advertise_service(&mdns, "_circuitpython", "_tcp", web_api_port);
+        socketpool_socket(&pool, SOCKETPOOL_AF_INET, SOCKETPOOL_SOCK_STREAM, &listening);
+        common_hal_socketpool_socket_settimeout(&listening, 0);
+        // Bind to any ip.
+        common_hal_socketpool_socket_bind(&listening, "", 0, web_api_port);
+        common_hal_socketpool_socket_listen(&listening, 1);
+    }
 
     mp_int_t api_password_len = dotenv_get_key("/.env", "CIRCUITPY_WEB_API_PASSWORD", _api_password + 1, sizeof(_api_password) - 2);
     if (api_password_len > 0) {
@@ -259,12 +289,6 @@ void supervisor_start_web_workflow(void) {
         _base64_in_place(_api_password, api_password_len + 1, sizeof(_api_password));
     }
 
-    active.base.type = &socketpool_socket_type;
-    active.num = -1;
-    active.connected = false;
-
-    websocket_init();
-
     // TODO:
     // GET /cp/serial.txt
     //   - Most recent 1k of serial output.
@@ -283,6 +307,10 @@ static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int l
     }
 }
 
+STATIC void _print_raw(void *env, const char *str, size_t len) {
+    _send_raw((socketpool_socket_obj_t *)env, (const uint8_t *)str, (size_t)len);
+}
+
 static void _send_str(socketpool_socket_obj_t *socket, const char *str) {
     _send_raw(socket, (const uint8_t *)str, strlen(str));
 }
@@ -301,14 +329,19 @@ static void _send_strs(socketpool_socket_obj_t *socket, ...) {
 }
 
 static void _send_chunk(socketpool_socket_obj_t *socket, const char *chunk) {
-    char encoded_len[sizeof(size_t) * 2 + 1];
-    int len = snprintf(encoded_len, sizeof(encoded_len), "%X", strlen(chunk));
-    _send_raw(socket, (const uint8_t *)encoded_len, len);
-    _send_raw(socket, (const uint8_t *)"\r\n", 2);
+    mp_print_t _socket_print = {socket, _print_raw};
+    mp_printf(&_socket_print, "%X\r\n", strlen(chunk));
     _send_raw(socket, (const uint8_t *)chunk, strlen(chunk));
     _send_raw(socket, (const uint8_t *)"\r\n", 2);
 }
 
+STATIC void _print_chunk(void *env, const char *str, size_t len) {
+    mp_print_t _socket_print = {env, _print_raw};
+    mp_printf(&_socket_print, "%X\r\n", len);
+    _send_raw((socketpool_socket_obj_t *)env, (const uint8_t *)str, len);
+    _send_raw((socketpool_socket_obj_t *)env, (const uint8_t *)"\r\n", 2);
+}
+
 // A bit of a misnomer because it sends all arguments as one chunk.
 // The last argument must be NULL! Otherwise, it won't stop.
 static void _send_chunks(socketpool_socket_obj_t *socket, ...) {
@@ -326,9 +359,9 @@ static void _send_chunks(socketpool_socket_obj_t *socket, ...) {
     }
     va_end(strs_to_count);
 
-    char encoded_len[sizeof(size_t) * 2 + 1];
-    snprintf(encoded_len, sizeof(encoded_len), "%X", chunk_len);
-    _send_strs(socket, encoded_len, "\r\n", NULL);
+
+    mp_print_t _socket_print = {socket, _print_raw};
+    mp_printf(&_socket_print, "%X\r\n", chunk_len);
 
     str = va_arg(strs_to_send, const char *);
     while (str != NULL) {
@@ -531,7 +564,12 @@ static void _reply_redirect(socketpool_socket_obj_t *socket, _request *request,
         _send_str(socket, "http");
     }
 
-    _send_strs(socket, "://", hostname, ".local", path, "\r\n", NULL);
+    _send_strs(socket, "://", hostname, ".local", NULL);
+    if (web_api_port != 80) {
+        mp_print_t _socket_print = {socket, _print_raw};
+        mp_printf(&_socket_print, ":%d", web_api_port);
+    }
+    _send_strs(socket, path, "\r\n", NULL);
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
 }
@@ -540,6 +578,7 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req
     socketpool_socket_send(socket, (const uint8_t *)OK_JSON, strlen(OK_JSON));
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
+    mp_print_t _socket_print = {socket, _print_chunk};
     _send_chunk(socket, "[");
     bool first = true;
 
@@ -560,7 +599,7 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req
         }
         // We use nanoseconds past Jan 1, 1970 for consistency with BLE API and
         // LittleFS.
-        _send_chunk(socket, ", \"modified_ns\": ");
+        _send_chunk(socket, ", ");
 
         uint64_t truncated_time = timeutils_mktime(1980 + (file_info.fdate >> 9),
             (file_info.fdate >> 5) & 0xf,
@@ -569,15 +608,17 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req
             (file_info.ftime >> 5) & 0x1f,
             (file_info.ftime & 0x1f) * 2) * 1000000000ULL;
 
-        char encoded_number[32];
-        snprintf(encoded_number, sizeof(encoded_number), "%lld", truncated_time);
-        _send_chunks(socket, encoded_number, ", \"file_size\": ", NULL);
+        // Use snprintf because mp_printf doesn't support 64 bit numbers by
+        // default.
+        char encoded_time[32];
+        snprintf(encoded_time, sizeof(encoded_time), "%llu", truncated_time);
+        mp_printf(&_socket_print, "\"modified_ns\": %s, ", encoded_time);
         size_t file_size = 0;
         if ((file_info.fattrib & AM_DIR) == 0) {
             file_size = file_info.fsize;
         }
-        snprintf(encoded_number, sizeof(encoded_number), "%d", file_size);
-        _send_chunks(socket, encoded_number, "}", NULL);
+        mp_printf(&_socket_print, "\"file_size\": %d }", file_size);
+
         first = false;
         res = f_readdir(dir, &file_info);
     }
@@ -587,12 +628,10 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req
 
 static void _reply_with_file(socketpool_socket_obj_t *socket, _request *request, const char *filename, FIL *active_file) {
     uint32_t total_length = f_size(active_file);
-    char encoded_len[10];
-    snprintf(encoded_len, sizeof(encoded_len), "%d", total_length);
 
-    _send_strs(socket,
-        "HTTP/1.1 200 OK\r\n",
-        "Content-Length: ", encoded_len, "\r\n", NULL);
+    _send_str(socket, "HTTP/1.1 200 OK\r\n");
+    mp_print_t _socket_print = {socket, _print_raw};
+    mp_printf(&_socket_print, "Content-Length: %d\r\n", total_length);
     // TODO: Make this a table to save space.
     if (_endswith(filename, ".txt") || _endswith(filename, ".py")) {
         _send_str(socket, "Content-Type: text/plain\r\n");
@@ -640,27 +679,23 @@ static void _reply_with_devices_json(socketpool_socket_obj_t *socket, _request *
     socketpool_socket_send(socket, (const uint8_t *)OK_JSON, strlen(OK_JSON));
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
-    char total_encoded[4];
-    snprintf(total_encoded, sizeof(total_encoded), "%d", total_results);
-    _send_chunks(socket, "{\"total\": ", total_encoded, ", \"devices\": [", NULL);
+    mp_print_t _socket_print = {socket, _print_chunk};
+
+    mp_printf(&_socket_print, "{\"total\": %d, \"devices\": [", total_results);
     for (size_t i = 0; i < count; i++) {
         if (i > 0) {
             _send_chunk(socket, ",");
         }
         const char *hostname = common_hal_mdns_remoteservice_get_hostname(&found_devices[i]);
         const char *instance_name = common_hal_mdns_remoteservice_get_instance_name(&found_devices[i]);
-        char port_encoded[4];
         int port = common_hal_mdns_remoteservice_get_port(&found_devices[i]);
-        snprintf(port_encoded, sizeof(port_encoded), "%d", port);
-        char ip_encoded[4 * 4];
         uint32_t ipv4_address = mdns_remoteservice_get_ipv4_address(&found_devices[i]);
         uint8_t *octets = (uint8_t *)&ipv4_address;
-        snprintf(ip_encoded, sizeof(ip_encoded), "%d.%d.%d.%d", octets[0], octets[1], octets[2], octets[3]);
-        _send_chunks(socket,
-            "{\"hostname\": \"", hostname, "\", ",
-            "\"instance_name\": \"", instance_name, "\", ",
-            "\"port\": ", port_encoded, ", ",
-            "\"ip\": \"", ip_encoded, "\"}", NULL);
+        mp_printf(&_socket_print,
+            "{\"hostname\": \"%s\", "
+            "\"instance_name\": \"%s\", "
+            "\"port\": %d, "
+            "\"ip\": \"%d.%d.%d.%d\"}", hostname, instance_name, port, octets[0], octets[1], octets[2], octets[3]);
         common_hal_mdns_remoteservice_deinit(&found_devices[i]);
     }
     _send_chunk(socket, "]}");
@@ -672,24 +707,22 @@ static void _reply_with_version_json(socketpool_socket_obj_t *socket, _request *
     _send_str(socket, OK_JSON);
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
-    char encoded_creator_id[11]; // 2 ** 32 is 10 decimal digits plus one for \0
-    snprintf(encoded_creator_id, sizeof(encoded_creator_id), "%u", CIRCUITPY_CREATOR_ID);
-    char encoded_creation_id[11]; // 2 ** 32 is 10 decimal digits plus one for \0
-    snprintf(encoded_creation_id, sizeof(encoded_creation_id), "%u", CIRCUITPY_CREATION_ID);
+    mp_print_t _socket_print = {socket, _print_chunk};
+
     const char *hostname = common_hal_mdns_server_get_hostname(&mdns);
-    _send_chunks(socket,
-        "{\"web_api_version\": 1, ",
-        "\"version\": \"", MICROPY_GIT_TAG, "\", ",
-        "\"build_date\": \"", MICROPY_BUILD_DATE, "\", ",
-        "\"board_name\": \"", MICROPY_HW_BOARD_NAME, "\", ",
-        "\"mcu_name\": \"", MICROPY_HW_MCU_NAME, "\", ",
-        "\"board_id\": \"", CIRCUITPY_BOARD_ID, "\", ",
-        "\"creator_id\": ", encoded_creator_id, ", ",
-        "\"creation_id\": ", encoded_creation_id, ", ",
-        "\"hostname\": \"", hostname, "\", ",
-        "\"port\": 80, ",
-        "\"ip\": \"", _our_ip_encoded,
-        "\"}", NULL);
+    // Note: this leverages the fact that C concats consecutive string literals together.
+    mp_printf(&_socket_print,
+        "{\"web_api_version\": 1, "
+        "\"version\": \"" MICROPY_GIT_TAG "\", "
+        "\"build_date\": \"" MICROPY_BUILD_DATE "\", "
+        "\"board_name\": \"" MICROPY_HW_BOARD_NAME "\", "
+        "\"mcu_name\": \"" MICROPY_HW_MCU_NAME "\", "
+        "\"board_id\": \"" CIRCUITPY_BOARD_ID "\", "
+        "\"creator_id\": %u, "
+        "\"creation_id\": %u, "
+        "\"hostname\": \"%s\", "
+        "\"port\": %d, "
+        "\"ip\": \"%s\"}", CIRCUITPY_CREATOR_ID, CIRCUITPY_CREATION_ID, hostname, web_api_port, _our_ip_encoded);
     // Empty chunk signals the end of the response.
     _send_chunk(socket, "");
 }
@@ -1179,7 +1212,12 @@ static void _process_request(socketpool_socket_obj_t *socket, _request *request)
                         request->authenticated = strncmp(request->header_value, prefix, strlen(prefix)) == 0 &&
                             strcmp(_api_password, request->header_value + strlen(prefix)) == 0;
                     } else if (strcasecmp(request->header_key, "Host") == 0) {
-                        request->redirect = strcmp(request->header_value, "circuitpython.local") == 0;
+                        // Do a prefix check so that port is ignored. Length must be the same or the
+                        // header ends in :.
+                        const char *cp_local = "circuitpython.local";
+                        request->redirect = strncmp(request->header_value, cp_local, strlen(cp_local)) == 0 &&
+                            (strlen(request->header_value) == strlen(cp_local) ||
+                                request->header_value[strlen(cp_local)] == ':');
                     } else if (strcasecmp(request->header_key, "Content-Length") == 0) {
                         request->content_length = strtoul(request->header_value, NULL, 10);
                     } else if (strcasecmp(request->header_key, "Expect") == 0) {