Try to work around Ansible rc-13 race condition

There is a race condition in Ansible that can result in this failure:
  msg: |-
  MODULE FAILURE
  See stdout/stderr for the exact error
rc: -13
See ansible/ansible#78344 and
ansible/ansible#81777.

In stackhpc/stackhpc-kayobe-config#1108 we applied
a workaround to increase the ControlPersist timeout to 1 hour, but this
does not always work.

Try another workaround of removing the ControlPersist sockets in between
Kayobe runs.

Author: markgoddard

ansible/files/multinode.sh

@@ -82,13 +82,36 @@ function decrypt_file() {
   ansible-vault decrypt --vault-password-file ~/vault.password $file
 }
 
+function workaround_ansible_rc13_bug() {
+  # Call this function in between long-running Ansible executions to attempt to
+  # work around an Ansible race condition.
+
+  # There is a race condition in Ansible that can result in this failure:
+  #   msg: |-
+  #   MODULE FAILURE
+  #   See stdout/stderr for the exact error
+  # rc: -13
+  # See https://github.com/ansible/ansible/issues/78344 and
+  # https://github.com/ansible/ansible/issues/81777.
+  # In https://github.com/stackhpc/stackhpc-kayobe-config/pull/1108 we applied
+  # a workaround to increase the ControlPersist timeout to 1 hour, but this
+  # does not always work.
+  # Try another workaround of removing the ControlPersist sockets.
+  rm -f ~/.ansible/cp/*
+}
+
+function run_kayobe() {
+  workaround_ansible_rc13_bug
+  kayobe $*
+}
+
 function deploy_seed() {
-  kayobe seed host configure
+  run_kayobe seed host configure
 }
 
 function deploy_seed_vault() {
   # Deploy hashicorp vault to the seed
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/OS-TLS-INT.pem
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/*.key
@@ -97,8 +120,8 @@ function deploy_seed_vault() {
 function get_seed_ssh() {
   # NOTE: Bash clears the -e option in subshells when not in Posix mode.
   set -e
-  ssh_user=$(kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"')
-  seed_addr=$(kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"')
+  ssh_user=$(run_kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"')
+  seed_addr=$(run_kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"')
   echo "${ssh_user}@${seed_addr}"
 }
 
@@ -117,37 +140,37 @@ function copy_ca_to_seed() {
 }
 
 function deploy_ceph() {
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml
   sleep 30
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml
 }
 
 function deploy_overcloud_vault() {
   # NOTE: Previously it was necessary to first deploy HAProxy with TLS disabled.
   if [[ -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals-tls-config.yml ]]; then
     # Skip os_capacity deployment since it requires admin-openrc.sh which doesn't exist yet.
-    kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
+    run_kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
   fi
 
   # Deploy hashicorp vault to the controllers
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud-vault-keys.json
 }
 
 function generate_overcloud_certs() {
   # Generate external tls certificates
   if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]]; then
-    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+    run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
     encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
   fi
 
   # Generate internal tls certificates
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy-internal.pem
 
   # Generate backend tls certificates
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml
   for cert in $(ls -1 $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/*-key.pem); do
     encrypt_file $cert
   done
@@ -164,15 +187,15 @@ function generate_barbican_secrets() {
   decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   sed -i "s/secret_id:.*/secret_id: $(uuidgen)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml
   decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   sed -i "s/role_id:.*/role_id: $(cat /tmp/barbican-role-id)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   rm /tmp/barbican-role-id
 }
 
 function deploy_overcloud() {
-  kayobe overcloud host configure
+  run_kayobe overcloud host configure
 
   deploy_ceph
 
@@ -185,19 +208,19 @@ function deploy_overcloud() {
   generate_barbican_secrets
 
   # Deploy all services
-  kayobe overcloud service deploy
+  run_kayobe overcloud service deploy
 
   copy_ca_to_seed
 }
 
 function deploy_wazuh() {
-  kayobe infra vm host configure
+  run_kayobe infra vm host configure
 
   # Deploy Wazuh
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/wazuh-secrets.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml
 }
 
 function create_resources() {
@@ -317,7 +340,7 @@ function deploy_full() {
 
   deploy_seed
   deploy_overcloud
-  if kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then
+  if run_kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then
     deploy_wazuh
   fi
   create_resources
@@ -327,13 +350,13 @@ function deploy_full() {
 function upgrade_overcloud() {
   # Generate external tls certificates if it was previously disabled.
   if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]] && [[ ! -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem ]]; then
-    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+    run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
     encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
   fi
 
-  kayobe overcloud host upgrade
-  kayobe overcloud host configure
-  kayobe overcloud service upgrade
+  run_kayobe overcloud host upgrade
+  run_kayobe overcloud host configure
+  run_kayobe overcloud service upgrade
 }
 
 function usage() {