Modify rc-13 workaround to use an environment variable

ansible/deploy-openstack-config.yml

@@ -118,25 +118,6 @@
       vars:
         kayobe_environment_path: "{{ src_directory }}/kayobe-config/etc/kayobe/environments/{{ kayobe_config_environment }}"
 
-    # WORKAROUND:
-    # There is a race condition in Ansible that can result in this failure:
-    #   msg: |-
-    #   MODULE FAILURE
-    #   See stdout/stderr for the exact error
-    # rc: -13
-    # See https://github.com/ansible/ansible/issues/78344 and
-    # https://github.com/ansible/ansible/issues/81777.
-    # In https://github.com/stackhpc/stackhpc-kayobe-config/pull/1108 we applied
-    # a workaround to increase the ControlPersist timeout to 1 hour, but this
-    # does not always work.
-    # Disabling SSH pipelining prevents the issue at the cost of Ansible
-    # execution duration.
-    - name: Disable SSH pipelining
-      ansible.builtin.replace:
-        path: "{{ src_directory }}/{{ kayobe_config_name }}/etc/kayobe/ansible.cfg"
-        regexp: "^pipelining.*=.*"
-        replace: "pipelining = False"
-
     - name: Ensure hooks directory are present
       ansible.builtin.file:
         dest: "{{ src_directory }}/{{ kayobe_config_name }}/etc/kayobe/hooks/{{ item }}"

ansible/files/multinode.sh

@@ -82,13 +82,40 @@ function decrypt_file() {
   ansible-vault decrypt --vault-password-file ~/vault.password $file
 }
 
+function workaround_ansible_rc13_bug() {
+  # Call this function in between long-running Ansible executions to attempt to
+  # work around an Ansible race condition.
+
+  # There is a race condition in Ansible that can result in this failure:
+  #   msg: |-
+  #   MODULE FAILURE
+  #   See stdout/stderr for the exact error
+  # rc: -13
+  # See https://github.com/ansible/ansible/issues/78344 and
+  # https://github.com/ansible/ansible/issues/81777.
+  # In https://github.com/stackhpc/stackhpc-kayobe-config/pull/1108 we applied
+  # a workaround to increase the ControlPersist timeout to 1 hour, but this
+  # does not always work.
+  # Here we use a different workaround of disabling SSH pipelining. This has
+  # performance implications for Ansible, but is a reasonable trade-off for
+  # reliability.
+  # We set the config option as an environment variable rather than in
+  # ansible.cfg in Kayobe configuration, to avoid a merge conflict on upgrade.
+  export ANSIBLE_PIPELINING=False
+}
+
+function run_kayobe() {
+  workaround_ansible_rc13_bug
+  kayobe $*
+}
+
 function deploy_seed() {
-  kayobe seed host configure
+  run_kayobe seed host configure
 }
 
 function deploy_seed_vault() {
   # Deploy hashicorp vault to the seed
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/OS-TLS-INT.pem
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/*.key
@@ -97,8 +124,8 @@ function deploy_seed_vault() {
 function get_seed_ssh() {
   # NOTE: Bash clears the -e option in subshells when not in Posix mode.
   set -e
-  ssh_user=$(kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"')
-  seed_addr=$(kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"')
+  ssh_user=$(run_kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"')
+  seed_addr=$(run_kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"')
   echo "${ssh_user}@${seed_addr}"
 }
 
@@ -117,37 +144,37 @@ function copy_ca_to_seed() {
 }
 
 function deploy_ceph() {
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml
   sleep 30
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml
 }
 
 function deploy_overcloud_vault() {
   # NOTE: Previously it was necessary to first deploy HAProxy with TLS disabled.
   if [[ -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals-tls-config.yml ]]; then
     # Skip os_capacity deployment since it requires admin-openrc.sh which doesn't exist yet.
-    kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
+    run_kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
   fi
 
   # Deploy hashicorp vault to the controllers
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud-vault-keys.json
 }
 
 function generate_overcloud_certs() {
   # Generate external tls certificates
   if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]]; then
-    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+    run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
     encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
   fi
 
   # Generate internal tls certificates
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy-internal.pem
 
   # Generate backend tls certificates
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml
   for cert in $(ls -1 $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/*-key.pem); do
     encrypt_file $cert
   done
@@ -164,15 +191,15 @@ function generate_barbican_secrets() {
   decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   sed -i "s/secret_id:.*/secret_id: $(uuidgen)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml
   decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   sed -i "s/role_id:.*/role_id: $(cat /tmp/barbican-role-id)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml
   rm /tmp/barbican-role-id
 }
 
 function deploy_overcloud() {
-  kayobe overcloud host configure
+  run_kayobe overcloud host configure
 
   deploy_ceph
 
@@ -185,19 +212,19 @@ function deploy_overcloud() {
   generate_barbican_secrets
 
   # Deploy all services
-  kayobe overcloud service deploy
+  run_kayobe overcloud service deploy
 
   copy_ca_to_seed
 }
 
 function deploy_wazuh() {
-  kayobe infra vm host configure
+  run_kayobe infra vm host configure
 
   # Deploy Wazuh
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml
   encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/wazuh-secrets.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml
-  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml
+  run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml
 }
 
 function create_resources() {
@@ -317,7 +344,7 @@ function deploy_full() {
 
   deploy_seed
   deploy_overcloud
-  if kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then
+  if run_kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then
     deploy_wazuh
   fi
   create_resources
@@ -327,13 +354,13 @@ function deploy_full() {
 function upgrade_overcloud() {
   # Generate external tls certificates if it was previously disabled.
   if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]] && [[ ! -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem ]]; then
-    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+    run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
     encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
   fi
 
-  kayobe overcloud host upgrade
-  kayobe overcloud host configure
-  kayobe overcloud service upgrade
+  run_kayobe overcloud host upgrade
+  run_kayobe overcloud host configure
+  run_kayobe overcloud service upgrade
 }
 
 function usage() {