Support PKCS#11 for mutual TLS on Unix platforms (#356)

- Update to latest `aws-crt-cpp`, which exposes PKCS#11 functionality (see awslabs/aws-crt-cpp#315)
- Add `pkcs11-pub-sub` sample, demonstrating an MQTT connection where the private key is stored in PKCS#11 token.
  - Add docs for sample
  - Sample runs in CI

Author: graebm

.builder/actions/build_samples.py

@@ -12,6 +12,7 @@ def run(self, env):
         steps = []
         samples = [
             'samples/mqtt/basic_pub_sub',
+            'samples/mqtt/pkcs11_pub_sub',
             'samples/mqtt/raw_pub_sub',
             'samples/shadow/shadow_sync',
             'samples/greengrass/basic_discovery',

codebuild/samples/linux-smoke-tests.yml

@@ -6,7 +6,7 @@ phases:
       - add-apt-repository ppa:ubuntu-toolchain-r/test
       - apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main"
       - apt-get update -y
-      - apt-get install clang-8 cmake -y -f
+      - apt-get install clang-8 cmake softhsm -y -f
   pre_build:
     commands:
       - export CC=clang-8
@@ -16,6 +16,7 @@ phases:
       - echo Build started on `date`
       - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
+      - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-pubsub-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/securetunnel-linux.sh
   post_build:
     commands:

codebuild/samples/pkcs11-pubsub-linux.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+# from hereon commands are echoed. don't leak secrets
+set -x
+
+softhsm2-util --version
+
+# SoftHSM2's default tokendir path might be invalid on this machine
+# so set up a conf file that specifies a known good tokendir path
+mkdir -p /tmp/tokens
+export SOFTHSM2_CONF=/tmp/softhsm2.conf
+echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
+
+# create token
+softhsm2-util --init-token --free --label my-token --pin 0000 --so-pin 0000
+
+# add private key to token (must be in PKCS#8 format)
+openssl pkcs8 -topk8 -in /tmp/privatekey.pem -out /tmp/privatekey.p8.pem -nocrypt
+softhsm2-util --import /tmp/privatekey.p8.pem --token my-token --label my-key --id BEEFCAFE --pin 0000
+
+# build and run sample
+pushd $CODEBUILD_SRC_DIR/samples/mqtt/pkcs11_pub_sub
+
+mkdir _build
+cd _build
+cmake -DCMAKE_PREFIX_PATH=/tmp/install ..
+make -j
+
+./pkcs11-pub-sub \
+    --endpoint $ENDPOINT \
+    --cert /tmp/certificate.pem \
+    --pkcs11_lib /usr/lib/softhsm/libsofthsm2.so \
+    --pin 0000 \
+    --token_label my-token \
+    --key_label my-key
+
+popd

crt/aws-crt-cpp

@@ -1302,7 +1302,7 @@ namespace Aws
             const Crt::Optional<Crt::ByteBuf> &payload,
             uint32_t messageFlags)
         {
-            bool streamAlreadyTerminated = messageFlags & AWS_EVENT_STREAM_RPC_MESSAGE_FLAG_TERMINATE_STREAM;
+            bool streamAlreadyTerminated = (messageFlags & AWS_EVENT_STREAM_RPC_MESSAGE_FLAG_TERMINATE_STREAM) != 0;
 
             Crt::StringView payloadStringView;
             if (payload.has_value())

eventstream_rpc/source/EventStreamClient.cpp

@@ -7,6 +7,12 @@
 #include <awstest/EchoTestRpcClient.h>
 
 #include <aws/testing/aws_test_harness.h>
+#if defined(_WIN32)
+// aws_test_harness.h includes Windows.h, which is an abomination.
+// undef macros with clashing names...
+#    undef SetPort
+#    undef GetMessage
+#endif
 
 #include <iostream>
 #include <queue>

eventstream_rpc/tests/EventStreamClientTest.cpp

@@ -53,6 +53,13 @@ else ()
     target_compile_options(GreengrassIpc-cpp PRIVATE -Wall -Wno-long-long -pedantic -Werror)
 endif ()
 
+if (MSVC)
+    # Generated code files can be very big. Set /bigobj to avoid the following error:
+    # > fatal error C1128: number of sections exceeded object file format limit: compile with /bigobj
+    target_compile_options(GreengrassIpc-cpp PRIVATE /bigobj)
+endif ()
+
+
 if (CMAKE_BUILD_TYPE STREQUAL "" OR CMAKE_BUILD_TYPE MATCHES Debug)
     target_compile_definitions(GreengrassIpc-cpp PRIVATE "-DDEBUG_BUILD")
 endif ()

greengrass_ipc/CMakeLists.txt

@@ -1,6 +1,7 @@
 # Sample apps for the AWS IoT Device SDK for C++ v2
 
 * [Basic MQTT Pub-Sub](#basic-mqtt-pub-sub)
+* [PKCS#11 MQTT Pub-Sub](#pkcs11-mqtt-pub-sub)
 * [Raw MQTT Pub-Sub](#raw-mqtt-pub-sub)
 * [Fleet provisioning](#fleet-provisioning)
 * [Shadow](#shadow)
@@ -96,9 +97,63 @@ To run the basic MQTT Pub-Sub use the following command:
 --topic <topic name>
 ```
 
+## PKCS#11 MQTT Pub-Sub
+
+This sample is similar to the [Basic Pub-Sub](#basic-mqtt-pub-sub),
+but the private key for mutual TLS is stored on a PKCS#11 compatible smart card or Hardware Security Module (HSM)
+
+WARNING: Unix only. Currently, TLS integration with PKCS#11 is only available on Unix devices.
+
+source: `samples/mqtt/pkcs11_pub_sub/main/cpp`
+
+To run this sample using [SoftHSM2](https://www.opendnssec.org/softhsm/) as the PKCS#11 device:
+
+1)  Create an IoT Thing with a certificate and key if you haven't already.
+
+2)  Convert the private key into PKCS#8 format
+    ```sh
+    openssl pkcs8 -topk8 -in <private.pem.key> -out <private.p8.key> -nocrypt
+    ```
+
+3)  Install [SoftHSM2](https://www.opendnssec.org/softhsm/):
+    ```sh
+    sudo apt install softhsm
+    ```
+
+    Check that it's working:
+    ```sh
+    softhsm2-util --show-slots
+    ```
+
+    If this spits out an error message, create a config file:
+    *   Default location: `~/.config/softhsm2/softhsm2.conf`
+    *   This file must specify token dir, default value is:
+        ```
+        directories.tokendir = /usr/local/var/lib/softhsm/tokens/
+        ```
+
+4)  Create token and import private key.
+
+    You can use any values for the labels, PINs, etc
+    ```sh
+    softhsm2-util --init-token --free --label <token-label> --pin <user-pin> --so-pin <so-pin>
+    ```
+
+    Note which slot the token ended up in
+
+    ```sh
+    softhsm2-util --import <private.p8.key> --slot <slot-with-token> --label <key-label> --id <hex-chars> --pin <user-pin>
+    ```
+
+5)  Now you can run the sample:
+    ```sh
+    ./pkcs11-pub-sub --endpoint <xxxx-ats.iot.xxxx.amazonaws.com> --ca_file <AmazonRootCA1.pem> --cert <certificate.pem.crt> --pkcs11_lib <libsofthsm2.so> --pin <user-pin> --token_label <token-label> --key_label <key-label>
+    ```
+
+
 ## Raw MQTT Pub-Sub
 
-This sample is similar to the Basic Pub-Sub, but the connection setup is more manual.
+This sample is similar to the [Basic Pub-Sub](#basic-mqtt-pub-sub), but the connection setup is more manual.
 This is a starting point for using custom
 [Configurable Endpoints](https://docs.aws.amazon.com/iot/latest/developerguide/iot-custom-endpoints-configurable.html).
 
@@ -339,7 +394,7 @@ get the sample up and running. These steps assume you have the AWS CLI installed
 sufficient permission to perform all of the listed operations. You will also need python3 to be able to run parse_cert_set_result.py. These steps are based on provisioning setup steps
 that can be found at [Embedded C SDK Setup](https://docs.aws.amazon.com/freertos/latest/lib-ref/c-sdk/provisioning/provisioning_tests.html#provisioning_system_tests_setup).
 
-First, create the IAM role that will be needed by the fleet provisioning template. Replace `RoleName` with a name of the role you want to create. 
+First, create the IAM role that will be needed by the fleet provisioning template. Replace `RoleName` with a name of the role you want to create.
 ``` sh
 aws iam create-role \
     --role-name [RoleName] \
@@ -351,17 +406,17 @@ aws iam attach-role-policy \
         --role-name [RoleName] \
         --policy-arn arn:aws:iam::aws:policy/service-role/AWSIoTThingsRegistration
 ```
-Finally, create the template resource which will be used for provisioning by the demo application. This needs to be done only 
-once.  To create a template, the following AWS CLI command may be used. Replace `TemplateName` with the name of the fleet 
-provisioning template you want to create. Replace `RoleName` with the name of the role you created previously. Replace 
-`TemplateJSON` with the template body as a JSON string (containing escape characters). Replace `account` with your AWS 
-account number. 
+Finally, create the template resource which will be used for provisioning by the demo application. This needs to be done only
+once.  To create a template, the following AWS CLI command may be used. Replace `TemplateName` with the name of the fleet
+provisioning template you want to create. Replace `RoleName` with the name of the role you created previously. Replace
+`TemplateJSON` with the template body as a JSON string (containing escape characters). Replace `account` with your AWS
+account number.
 ``` sh
 aws iot create-provisioning-template \
         --template-name [TemplateName] \
         --provisioning-role-arn arn:aws:iam::[account]:role/[RoleName] \
         --template-body "[TemplateJSON]" \
-        --enabled 
+        --enabled
 ```
 The rest of the instructions assume you have used the following for the template body:
 ``` sh
@@ -371,13 +426,13 @@ If you use a different body, you may need to pass in different template paramete
 
 #### Running the sample and provisioning using a certificate-key set from a provisioning claim
 
-To run the provisioning sample, you'll need a certificate and key set with sufficient permissions. Provisioning certificates are normally 
+To run the provisioning sample, you'll need a certificate and key set with sufficient permissions. Provisioning certificates are normally
 created ahead of time and placed on your device, but for this sample, we will just create them on the fly. You can also
 use any certificate set you've already created if it has sufficient IoT permissions and in doing so, you can skip the step
 that calls `create-provisioning-claim`.
- 
+
 We've included a script in the utils folder that creates certificate and key files from the response of calling
-`create-provisioning-claim`. These dynamically sourced certificates are only valid for five minutes. When running the command, 
+`create-provisioning-claim`. These dynamically sourced certificates are only valid for five minutes. When running the command,
 you'll need to substitute the name of the template you previously created, and on Windows, replace the paths with something appropriate.
 
 (Optional) Create a temporary provisioning claim certificate set. This command is executed in the debug folder(`aws-iot-device-sdk-cpp-v2-build\samples\identity\fleet_provisioning\Debug`):
@@ -390,7 +445,7 @@ aws iot create-provisioning-claim \
 ```
 
 The provisioning claim's cert and key set have been written to `/tmp/provision*`. Now you can use these temporary keys
-to perform the actual provisioning. If you are not using the temporary provisioning certificate, replace the paths for `--cert` 
+to perform the actual provisioning. If you are not using the temporary provisioning certificate, replace the paths for `--cert`
 and `--key` appropriately:
 
 ``` sh

samples/README.md

@@ -481,11 +481,9 @@ int main(int argc, char *argv[])
 
     /*
      * Actually perform the connect dance.
-     * This will use default ping behavior of 1 hour and 3 second timeouts.
-     * If you want different behavior, those arguments go into slots 3 & 4.
      */
     fprintf(stdout, "Connecting...\n");
-    if (!connection->Connect(clientId.c_str(), false, 1000))
+    if (!connection->Connect(clientId.c_str(), false /*cleanSession*/, 1000 /*keepAliveTimeSecs*/))
     {
         fprintf(stderr, "MQTT Connection failed with error %s\n", ErrorDebugString(connection->LastError()));
         exit(-1);

samples/mqtt/basic_pub_sub/main.cpp

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.1)
+# note: cxx-17 requires cmake 3.8, cxx-20 requires cmake 3.12
+project(pkcs11-pub-sub CXX)
+
+file(GLOB SRC_FILES
+       "*.cpp"
+)
+
+add_executable(${PROJECT_NAME} ${SRC_FILES})
+
+set_target_properties(${PROJECT_NAME} PROPERTIES
+    CXX_STANDARD 14)
+
+# set warnings
+if (MSVC)
+    target_compile_options(${PROJECT_NAME} PRIVATE /W4 /WX /wd4068)
+else ()
+    target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wno-long-long -pedantic -Werror)
+endif ()
+
+find_package(aws-crt-cpp REQUIRED)
+
+target_link_libraries(${PROJECT_NAME} AWS::aws-crt-cpp)