Refactor upload methods to streamline the debugging experience (ARMmbed#129)

* Change all debug configurations to not reset and exit on error.  Initial tasks.json implementation.

* st-util working on command line and VS Code!

* Convert STM32CUBE, fix spaces in GDB server task command

* pyocd working!

* Convert OpenOCD

* Convert JLINK

* Convert REDLINK upload method

* Fix CMake typo

* Try and fix some variable scope issues

* Initial CLion implementation.  Need to test with hardware.

* Add a separate task that kicks off a build before each debug session

Author: Jamie Smith

tools/cmake/UploadMethodManager.cmake

@@ -14,6 +14,14 @@ set(UPLOAD_METHOD "${UPLOAD_METHOD_DEFAULT}" CACHE STRING "Method for uploading
 # use a higher numbered port to allow use without root on Linux/Mac
 set(GDB_PORT 23331 CACHE STRING "Port that the GDB server will be started on.")
 
+# Load the upload method.  This is expected to set the following variables:
+# UPLOAD_${UPLOAD_METHOD}_FOUND - True iff the dependencies for this upload method were found
+# UPLOAD_SUPPORTS_DEBUG - True iff this upload method supports debugging
+# UPLOAD_GDBSERVER_DEBUG_COMMAND - Command to start a new GDB server
+# UPLOAD_WANTS_EXTENDED_REMOTE - True iff GDB should use "target extended-remote" to connect to the GDB server
+# UPLOAD_LAUNCH_COMMANDS - List of GDB commands to run after launching GDB.
+# UPLOAD_RESTART_COMMANDS - List of commands to run when the "restart chip" function is used.
+# See here for more info: https://github.com/mbed-ce/mbed-os/wiki/Debugger-Commands-and-State-in-Upload-Methods
 include(UploadMethod${UPLOAD_METHOD})
 
 if(NOT "${UPLOAD_${UPLOAD_METHOD}_FOUND}")
@@ -24,46 +32,27 @@ if(NOT (("${UPLOAD_METHOD}" STREQUAL NONE) OR ("${${UPLOAD_METHOD}_UPLOAD_ENABLE
 	message(FATAL_ERROR "The upload method ${UPLOAD_METHOD} is not enabled in the config code for this target -- set ${UPLOAD_METHOD}_UPLOAD_ENABLED to TRUE to enable it.")
 endif()
 
-message(STATUS "Board upload method set to ${UPLOAD_METHOD}")
-
-# ----------------------------------------------
-# Generate gdbinit if needed
-
 if(UPLOAD_SUPPORTS_DEBUG)
-	# create init file for GDB client
-	if(UPLOAD_WANTS_EXTENDED_REMOTE)
-		set(UPLOAD_GDB_REMOTE_KEYWORD "extended-remote")
-	else()
-		set(UPLOAD_GDB_REMOTE_KEYWORD "remote")
-	endif()
-
-	file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/mbed-cmake.gdbinit CONTENT
-"# connect to GDB server
-target ${UPLOAD_GDB_REMOTE_KEYWORD} localhost:${GDB_PORT}
-")
+	message(STATUS "Mbed: Code upload and debugging enabled via upload method ${UPLOAD_METHOD}")
+elseif(NOT "${UPLOAD_METHOD}" STREQUAL "NONE")
+	message(STATUS "Mbed: Code upload enabled via upload method ${UPLOAD_METHOD}")
 endif()
 
-# UPLOAD_SUPPORTS_DEBUG needs to be made into a cache variable so that it can
+# These variables need to be made into a cache variables so that they can
 # be seen by higher level directories when they call mbed_generate_upload_debug_targets()
 set(MBED_UPLOAD_SUPPORTS_DEBUG ${UPLOAD_SUPPORTS_DEBUG} CACHE INTERNAL "" FORCE)
+set(MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND ${UPLOAD_GDBSERVER_DEBUG_COMMAND} CACHE INTERNAL "" FORCE)
+set(MBED_UPLOAD_WANTS_EXTENDED_REMOTE ${UPLOAD_WANTS_EXTENDED_REMOTE} CACHE INTERNAL "" FORCE)
+set(MBED_UPLOAD_LAUNCH_COMMANDS ${UPLOAD_LAUNCH_COMMANDS} CACHE INTERNAL "" FORCE)
+set(MBED_UPLOAD_RESTART_COMMANDS ${UPLOAD_RESTART_COMMANDS} CACHE INTERNAL "" FORCE)
 
 # ----------------------------------------------
 # Function for creating targets
 
-function(mbed_generate_upload_debug_targets target)
+function(mbed_generate_upload_target target)
 	# add upload target
 	gen_upload_target(${target}
 		${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_BASE_NAME:${target}>.bin
 		${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_BASE_NAME:${target}>.hex
 	)
-
-	# add debug target
-	if(MBED_UPLOAD_SUPPORTS_DEBUG AND MBED_GDB_FOUND)
-		add_custom_target(debug-${target}
-			COMMENT "starting GDB to debug ${target}..."
-			COMMAND ${MBED_GDB}
-			--command=${CMAKE_BINARY_DIR}/mbed-cmake.gdbinit
-			$<TARGET_FILE:${target}>
-			USES_TERMINAL)
-	endif()
 endfunction()

tools/cmake/mbed_ide_debug_cfg_generator.cmake

@@ -7,13 +7,21 @@
 # -------------------------------------------------------------
 
 if($ENV{CLION_IDE})
-	message(STATUS "Detected CLion IDE, will generate CLion debug configurations")
+	message(STATUS "Mbed: Detected CLion IDE, will generate CLion debug configurations")
 	set(MBED_GENERATE_CLION_DEBUG_CFGS TRUE)
 
+	set(MBED_CLION_PROFILE_NAME "" CACHE STRING "Name of the Clion build profile (Settings > Build, Execution, Deployment > CMake > Name textbox")
+
+	if(MBED_CLION_PROFILE_NAME STREQUAL "")
+		message(FATAL_ERROR "In order to generate CLion configuration files, Mbed CE needs to know the name of the current CLion build profile.  This name is the string typed into the Name textbox under Settings > Build, Execution, Deployment > CMake.  Pass this name with '-DMBED_CLION_PROFILE_NAME=<name>'.")
+	endif()
+
 elseif(CMAKE_EXPORT_COMPILE_COMMANDS) # TODO: Is this actually a reliable way of detecting VS Code? Not sure if it will create false positives.
-	message(STATUS "Detected VS Code IDE, will generate VS Code debug configurations")
+	message(STATUS "Mbed: Detected VS Code IDE, will generate VS Code debug configurations")
 	set(MBED_GENERATE_VS_CODE_DEBUG_CFGS TRUE)
 
+elseif(MBED_UPLOAD_SUPPORTS_DEBUG)
+	message(STATUS "Mbed: No IDE detected, will generate configurations for command-line debugging (e.g. ninja gdbserver, then ninja debug-SomeProgram)")
 endif()
 
 # CLion generator
@@ -31,11 +39,43 @@ if(MBED_GENERATE_CLION_DEBUG_CFGS)
 	    set(CONFIG_NAME "GDB ${CMAKE_TARGET} ${MBED_TARGET} ${CMAKE_BUILD_TYPE}")
 	    set(RUN_CONF_PATH "${CLION_RUN_CONF_DIR}/${CONFIG_NAME}.xml")
 
+		# Convert the CMake list into the correct format for the run configuration XML
+		list(GET MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND 0 GDBSERVER_EXECUTABLE)
+		list(SUBLIST MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND 1 -1 GDBSERVER_ARGS)
+		set(GDBSERVER_ARGS_STR "")
+		set(IS_FIRST_ARG TRUE)
+		foreach(ELEMENT ${GDBSERVER_ARGS})
+
+			if(IS_FIRST_ARG)
+				set(IS_FIRST_ARG FALSE)
+			else()
+				string(APPEND GDBSERVER_ARGS_STR " ")
+			endif()
+
+			# Escape quotes and ampersands
+			string(REPLACE "\"" "&quot;" ELEMENT "${ELEMENT}")
+			string(REPLACE "&" "&amp;" ELEMENT "${ELEMENT}")
+
+			if("${ELEMENT}" MATCHES " ")
+				string(APPEND GDBSERVER_ARGS_STR "&quot;${ELEMENT}&quot;")
+			else()
+				string(APPEND GDBSERVER_ARGS_STR "${ELEMENT}")
+			endif()
+		endforeach()
+
+		# Generate run configuration XML file.
+		# This file is based on a generic Embedded GDB Server run configuration generated by CLion,
+		# with constant strings replaced by placeholders for CMake.
 	    file(GENERATE OUTPUT ${RUN_CONF_PATH} CONTENT
 "<!-- Autogenerated by Mbed OS.  Do not edit! -->
 <component name=\"ProjectRunConfigurationManager\">
-  <configuration default=\"false\" name=\"${CONFIG_NAME}\" type=\"CLion_Remote\" remoteCommand=\"localhost:${GDB_PORT}\" symbolFile=\"$<TARGET_FILE:${CMAKE_TARGET}>\" sysroot=\"\">
-    <method v=\"2\" />
+  <configuration default=\"false\" name=\"${CONFIG_NAME}\" type=\"com.jetbrains.cidr.embedded.customgdbserver.type\" PROGRAM_PARAMS=\"${GDBSERVER_ARGS_STR}\" REDIRECT_INPUT=\"false\" ELEVATE=\"false\" USE_EXTERNAL_CONSOLE=\"false\" PASS_PARENT_ENVS_2=\"true\" PROJECT_NAME=\"${PROJECT_NAME}\" TARGET_NAME=\"${CMAKE_TARGET}\" CONFIG_NAME=\"${MBED_CLION_PROFILE_NAME}\" version=\"1\" RUN_TARGET_PROJECT_NAME=\"${PROJECT_NAME}\" RUN_TARGET_NAME=\"${CMAKE_TARGET}\">
+    <custom-gdb-server version=\"1\" gdb-connect=\"localhost:${GDB_PORT}\" executable=\"${GDBSERVER_EXECUTABLE}\" warmup-ms=\"0\" download-type=\"UPDATED_ONLY\" reset-cmd=\"monitor reset\" reset-type=\"AFTER_DOWNLOAD\">
+      <debugger kind=\"GDB\" isBundled=\"true\" />
+    </custom-gdb-server>
+    <method v=\"2\">
+      <option name=\"CLION.COMPOUND.BUILD\" enabled=\"true\" />
+    </method>
   </configuration>
 </component>
 ")
@@ -55,9 +95,16 @@ elseif(MBED_GENERATE_VS_CODE_DEBUG_CFGS)
 	# Start building up json file.  Needs to be a global property so we can append to it from anywhere.
 	# Note: Cannot use a cache variable for this because cache variables aren't allowed to contain newlines.
 	set_property(GLOBAL PROPERTY VSCODE_LAUNCH_JSON_CONTENT
-"{
+"// Auto-generated by Mbed CE.  Edits will be erased when CMake is rerun.
+{
     \"configurations\": [")
 
+	set_property(GLOBAL PROPERTY VSCODE_TASKS_JSON_CONTENT
+"// Auto-generated by Mbed CE.  Edits will be erased when CMake is rerun.
+{
+	\"version\": \"2.0.0\",
+	\"tasks\": [")
+
 	# Find objdump as the extension uses it.  In a sane world it should be in the compiler bin dir.
 	find_program(MBED_OBJDUMP
 		NAMES arm-none-eabi-objdump objdump
@@ -68,10 +115,15 @@ elseif(MBED_GENERATE_VS_CODE_DEBUG_CFGS)
 	function(mbed_generate_ide_debug_configuration CMAKE_TARGET)
 
 	    # Create name (combine target name, Mbed target, and build config to generate a unique string)
-	    set(CONFIG_NAME "Connect to GDB ${CMAKE_TARGET} ${MBED_TARGET} ${CMAKE_BUILD_TYPE}")
+	    set(CONFIG_NAME "Debug ${CMAKE_TARGET} ${MBED_TARGET} ${CMAKE_BUILD_TYPE}")
+
+		# Convert CMake lists to json
+		list(JOIN MBED_UPLOAD_LAUNCH_COMMANDS "\", \"" UPLOAD_LAUNCH_COMMANDS_FOR_JSON)
+		list(JOIN MBED_UPLOAD_RESTART_COMMANDS "\", \"" UPLOAD_RESTART_COMMANDS_FOR_JSON)
 
 	    # property list here: https://github.com/Marus/cortex-debug/blob/master/debug_attributes.md
 	    set_property(GLOBAL APPEND_STRING PROPERTY VSCODE_LAUNCH_JSON_CONTENT "
+	// Debug launch for target ${CMAKE_TARGET}.
 	{
 	    \"type\": \"cortex-debug\",
 	    \"name\": \"${CONFIG_NAME}\",
@@ -81,32 +133,155 @@ elseif(MBED_GENERATE_VS_CODE_DEBUG_CFGS)
 		\"objdumpPath\": \"${MBED_OBJDUMP}\",
 		\"servertype\": \"external\",
 		\"gdbTarget\": \"localhost:${GDB_PORT}\",
-		\"request\": \"attach\"
+		\"request\": \"launch\",
+		\"preLaunchTask\": \"Build ${CMAKE_TARGET} and start GDB server\",
+		// Override the command sequences used by VS Code to be correct for this GDB server
+		\"overrideLaunchCommands\": [\"${UPLOAD_LAUNCH_COMMANDS_FOR_JSON}\"],
+		\"overrideRestartCommands\": [\"${UPLOAD_RESTART_COMMANDS_FOR_JSON}\"],
 	},")
 
+		# Add tasks to both build only, and build and start the GDB server.
+		# Schema for tasks.json can be seen here https://code.visualstudio.com/docs/editor/tasks-appendix
+		set_property(GLOBAL APPEND_STRING PROPERTY VSCODE_TASKS_JSON_CONTENT "
+		// Build for target ${CMAKE_TARGET}
+		{
+			\"label\": \"Build ${CMAKE_TARGET}\",
+			\"type\": \"shell\",
+			\"command\": \"${CMAKE_COMMAND}\",
+			\"args\": [\"--build\", \"${CMAKE_BINARY_DIR}\", \"--target\", \"${CMAKE_TARGET}\"],
+		},
+		// Build ${CMAKE_TARGET} and run the GDB server
+		{
+			\"label\": \"Build ${CMAKE_TARGET} and start GDB server\",
+			\"dependsOn\": [\"Build ${CMAKE_TARGET}\", \"GDB Server\"],
+			\"dependsOrder\": \"sequence\",
+		},")
+
 	endfunction(mbed_generate_ide_debug_configuration)
 
 	# Take all generated debug configurations and write them to launch.json.
 	function(mbed_finalize_ide_debug_configurations)
+
+
 		# Add footer
 	    set_property(GLOBAL APPEND_STRING PROPERTY VSCODE_LAUNCH_JSON_CONTENT "
     ]
 }")
 
 		get_property(VSCODE_LAUNCH_JSON_CONTENT GLOBAL PROPERTY VSCODE_LAUNCH_JSON_CONTENT)
 	    file(GENERATE OUTPUT ${VSCODE_LAUNCH_JSON_PATH} CONTENT ${VSCODE_LAUNCH_JSON_CONTENT})
+
+	
+		# Convert the CMake list into the correct format for tasks.json
+		list(GET MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND 0 GDBSERVER_EXECUTABLE)
+		list(SUBLIST MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND 1 -1 GDBSERVER_ARGS)
+		set(GDBSERVER_ARGS_STR "")
+		set(IS_FIRST_ARG TRUE)
+		foreach(ELEMENT ${GDBSERVER_ARGS})
+		
+			if(IS_FIRST_ARG)
+				set(IS_FIRST_ARG FALSE)
+			else()
+				string(APPEND GDBSERVER_ARGS_STR ", ")
+			endif()
+
+			# Escape any quotes in the element
+			string(REPLACE "\"" "\\\"" ELEMENT "${ELEMENT}")
+
+			string(APPEND GDBSERVER_ARGS_STR "\"${ELEMENT}\"")
+		endforeach()
+
+		set_property(GLOBAL APPEND_STRING PROPERTY VSCODE_TASKS_JSON_CONTENT "
+		{
+			\"label\": \"GDB Server\",
+			\"type\": \"shell\",
+			\"command\": \"${GDBSERVER_EXECUTABLE}\",
+			\"args\": [${GDBSERVER_ARGS_STR}],
+			\"isBackground\": true,
+			// This task is run to start the GDB server, so that the launch configuration can connect to it.
+      		// Problem is, it's a GDB server, and since it never exits, VSCode
+     	 	// will never start the debug session. All this is needed so VSCode just lets it run.
+			\"problemMatcher\": [
+				{
+					\"pattern\": [
+						{
+							\"regexp\": \"________________\",
+							\"file\": 1,
+							\"location\": 2,
+							\"message\": 3
+						}
+					],
+					\"background\": {
+						\"activeOnStart\": true,
+						\"beginsPattern\": \".*\",
+						\"endsPattern\": \".*\",
+					}
+				}
+			],
+		}
+	]
+
+}
+	")
+
+		# Write out tasks.json
+		set(VSCODE_TASKS_JSON_PATH ${CMAKE_SOURCE_DIR}/.vscode/tasks.json)
+		get_property(VSCODE_TASKS_JSON_CONTENT GLOBAL PROPERTY VSCODE_TASKS_JSON_CONTENT)
+	    file(GENERATE OUTPUT ${VSCODE_TASKS_JSON_PATH} CONTENT ${VSCODE_TASKS_JSON_CONTENT})
+
 	endfunction(mbed_finalize_ide_debug_configurations)
 
-# No-op generator
+# Command-line generator
 # -------------------------------------------------------------
-else()
+elseif(MBED_UPLOAD_SUPPORTS_DEBUG)
 
 	function(mbed_generate_ide_debug_configuration CMAKE_TARGET)
-		# Empty
+
+			# add debug target
+			if(MBED_UPLOAD_SUPPORTS_DEBUG AND MBED_GDB_FOUND)
+			add_custom_target(debug-${target}
+				COMMENT "Starting GDB to debug ${target}..."
+				COMMAND ${MBED_GDB}
+				--command=${CMAKE_BINARY_DIR}/mbed-cmake.gdbinit
+				$<TARGET_FILE:${target}>
+				USES_TERMINAL)
+			endif()
+
 	endfunction(mbed_generate_ide_debug_configuration)
 
 	function(mbed_finalize_ide_debug_configurations)
-		# Empty
+
+		# create init file for GDB client
+		if(MBED_UPLOAD_WANTS_EXTENDED_REMOTE)
+			set(UPLOAD_GDB_REMOTE_KEYWORD "extended-remote")
+		else()
+			set(UPLOAD_GDB_REMOTE_KEYWORD "remote")
+		endif()
+
+		list(JOIN MBED_UPLOAD_LAUNCH_COMMANDS "\n" MBED_UPLOAD_LAUNCH_COMMANDS_FOR_GDBINIT)
+
+		file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/mbed-cmake.gdbinit CONTENT
+"# connect to GDB server
+target ${UPLOAD_GDB_REMOTE_KEYWORD} localhost:${GDB_PORT}
+${MBED_UPLOAD_LAUNCH_COMMANDS_FOR_GDBINIT}
+c"
+)
+
+		# Create target to start the GDB server
+		add_custom_target(gdbserver
+			COMMENT "Starting ${UPLOAD_METHOD} GDB server"
+			COMMAND ${MBED_UPLOAD_GDBSERVER_DEBUG_COMMAND}
+			USES_TERMINAL
+			VERBATIM)
 	endfunction(mbed_finalize_ide_debug_configurations)
 
+else()
+
+	# No-ops
+	function(mbed_generate_ide_debug_configuration CMAKE_TARGET)
+	endfunction()
+
+	function(mbed_finalize_ide_debug_configurations)
+	endfunction()
+
 endif()

tools/cmake/mbed_target_functions.cmake

@@ -181,7 +181,7 @@ function(mbed_set_post_build target)
         mbed_generate_map_file(${target})
     endif()
 
-    mbed_generate_upload_debug_targets(${target})
+    mbed_generate_upload_target(${target})
     mbed_generate_ide_debug_configuration(${target})
 endfunction()