Skip to content

Commit 03bf7f0

Browse files
markgoddardmarkgoddard
committed
Add diagnostics.yml playbook
This playbook runs a script that collects diagnostic information from hosts. The diagnostics are aggregated to a directory (diagnostics_path_local/inventory_hostname) on localhost. NOTE: The diagnostic information contains sensitive information such as passwords in configuration files.
1 parent 36dc6d1 commit 03bf7f0

File tree

3 files changed

+113
-52
lines changed

3 files changed

+113
-52
lines changed

‎etc/kayobe/ansible/diagnostics.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
---
2+
# This playbook runs a script that collects diagnostic information from hosts.
3+
# The diagnostics are aggregated to a directory
4+
# (diagnostics_path_local/inventory_hostname) on localhost.
5+
#
6+
# NOTE: The diagnostic information contains sensitive information such as
7+
# passwords in configuration files.
8+
9+
- name: Collect diagnostic information
10+
hosts: seed-hypervisor:seed:overcloud:infra-vms
11+
vars:
12+
diagnostics_path_local: "{{ lookup('env', 'PWD') }}/diagnostics"
13+
tasks:
14+
- block:
15+
- name: Create a temporary directory for diagnostics
16+
ansible.builtin.tempfile:
17+
state: directory
18+
suffix: diagnostics
19+
register: diagnostics_tmpdir
20+
21+
- name: Write host variables to a file
22+
ansible.builtin.copy:
23+
content: "{{ hostvars[inventory_hostname].ansible_facts | to_nice_json }}"
24+
dest: "{{ diagnostics_tmpdir.path }}/facts.json"
25+
26+
- name: Run diagnostics script
27+
ansible.builtin.script: "{{ kayobe_config_path }}/../../tools/diagnostics.sh"
28+
become: true
29+
failed_when: diagnostics_result.rc is not defined
30+
register: diagnostics_result
31+
environment:
32+
LOG_DIR: "{{ diagnostics_tmpdir.path }}"
33+
CONFIG_DIR: "{{ kayobe_config_path }}/../.."
34+
35+
- name: Download diagnostic logs to localhost
36+
ansible.posix.synchronize:
37+
src: "{{ diagnostics_tmpdir.path }}/"
38+
dest: "{{ diagnostics_path_local }}/{{ inventory_hostname }}"
39+
mode: pull
40+
archive: no
41+
recursive: true
42+
copy_links: true
43+
verify_host: true
44+
# For jump host
45+
use_ssh_args: true
46+
always:
47+
- name: Clean up temporary directory
48+
ansible.builtin.file:
49+
path: "{{ diagnostics_tmpdir.path }}"
50+
state: absent
51+
52+
- name: Display diagnostics collection stdout
53+
ansible.builtin.debug:
54+
msg: "{{ diagnostics_result.stdout }}"
55+
when: diagnostics_result.stdout is defined
56+
57+
- name: Display diagnostics collection stderr
58+
ansible.builtin.debug:
59+
msg: "{{ diagnostics_result.stderr }}"
60+
when: diagnostics_result.stderr is defined
61+
62+
- name: Fail if diagnostics collection failed
63+
ansible.builtin.fail:
64+
msg: Diagnostics collection failed
65+
when: diagnostics_result.rc != 0
66+
67+
- name: Display location of diagnostics archive
68+
ansible.builtin.debug:
69+
msg: >-
70+
Wrote diagnostics to {{ diagnostics_path_local }} on localhost

‎releasenotes/notes/diagnostics-378a6693a64d0b3c.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
features:
3+
- |
4+
Adds a new ``diagnostics.yml`` playbook that collects diagnostic
5+
information from hosts. The diagnostics are aggregated to a directory
6+
(``$PWD/diagnostics/`` by default) on localhost. The diagnostics include:
7+
8+
* Docker container logs
9+
* Kolla configuration files
10+
* Log files
11+
12+
*The collected diagnostic information contains sensitive information such
13+
as passwords in configuration files.*
14+

‎tools/diagnostics.sh

Lines changed: 29 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,34 @@
11
#!/bin/bash
22

3-
# NOTE(mgoddard): This has been adapted from tests/get_logs.sh in Kolla
4-
# Ansible.
3+
# NOTE(mgoddard): This has been adapted from
4+
# roles/kayobe-diagnostics/files/get_logs.sh in Kayobe.
55

66
# Environment variables:
77
# $LOG_DIR is the directory to copy logs to.
8-
# $CONFIG_DIR is the directory to copy configuration from.
9-
# $PREVIOUS_CONFIG_DIR is the directory to copy previous configuration, prior
10-
# to an upgrade, from.
118

9+
# TODO: Make this script more robust and use set -e.
1210
set +o errexit
11+
set -u
1312

1413
copy_logs() {
15-
cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/
16-
if [[ -d ${CONFIG_DIR} ]]; then
17-
cp -rnL ${CONFIG_DIR}/etc/kayobe/* ${LOG_DIR}/kayobe_configs
18-
cp -rnL ${CONFIG_DIR}/etc/kolla/* ${LOG_DIR}/kolla_configs
19-
cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_node_configs
20-
# Don't save the IPA images.
21-
rm ${LOG_DIR}/kayobe_configs/kolla/config/ironic/ironic-agent.{kernel,initramfs}
22-
rm ${LOG_DIR}/kolla_configs/config/ironic/ironic-agent.{kernel,initramfs}
23-
rm ${LOG_DIR}/kolla_node_configs/ironic-http/ironic-agent.{kernel,initramfs}
24-
rm ${LOG_DIR}/kolla_node_configs/ironic-tftp/ironic-agent.{kernel,initramfs}
25-
fi
26-
if [[ -n ${PREVIOUS_CONFIG_DIR} ]] && [[ -d ${PREVIOUS_CONFIG_DIR} ]]; then
27-
mkdir -p ${LOG_DIR}/previous_{kayobe,kolla}_configs
28-
cp -rnL ${PREVIOUS_CONFIG_DIR}/etc/kayobe/* ${LOG_DIR}/previous_kayobe_configs
29-
cp -rnL ${PREVIOUS_CONFIG_DIR}/etc/kolla/* ${LOG_DIR}/previous_kolla_configs
30-
# NOTE: we can't save node configs in /etc/kolla for the pervious
31-
# release since they'll have been overwritten at this point.
32-
# Don't save the IPA images.
33-
rm ${LOG_DIR}/previous_kayobe_configs/kolla/config/ironic/ironic-agent.{kernel,initramfs}
34-
rm ${LOG_DIR}/previous_kolla_configs/config/ironic/ironic-agent.{kernel,initramfs}
35-
fi
14+
mkdir -p ${LOG_DIR}/{docker_logs,kolla_node_configs,system_logs}
15+
16+
cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_node_configs
17+
# Don't save the IPA images.
18+
rm ${LOG_DIR}/kolla_node_configs/ironic-http/ironic-agent.{kernel,initramfs}
19+
rm ${LOG_DIR}/kolla_node_configs/ironic-tftp/ironic-agent.{kernel,initramfs}
3620

3721
if [[ -d /opt/kayobe/etc/kolla ]]; then
22+
mkdir -p ${LOG_DIR}/kolla_build_configs
3823
cp -rnL /opt/kayobe/etc/kolla/* ${LOG_DIR}/kolla_build_configs/
3924
fi
4025

4126
cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
4227

43-
if [[ -x "$(command -v journalctl)" ]]; then
44-
journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
45-
journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log
46-
journalctl --no-pager -u vbmcd.service > ${LOG_DIR}/system_logs/vbmcd.log
47-
journalctl --no-pager -u NetworkManager.service > ${LOG_DIR}/system_logs/NetworkManager.log
48-
else
49-
cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log
50-
fi
28+
journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.log
29+
journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log
30+
journalctl --no-pager -u vbmcd.service > ${LOG_DIR}/system_logs/vbmcd.log
31+
journalctl --no-pager -u NetworkManager.service > ${LOG_DIR}/system_logs/NetworkManager.log
5132

5233
if [[ -d /etc/sysconfig/network-scripts/ ]]; then
5334
cp -r /etc/sysconfig/network-scripts/ ${LOG_DIR}/system_logs/
@@ -81,6 +62,9 @@ copy_logs() {
8162
ip route > ${LOG_DIR}/system_logs/ip-route.txt
8263
ip route show table all > ${LOG_DIR}/system_logs/ip-route-all-tables.txt
8364
ip rule list > ${LOG_DIR}/system_logs/ip-rule-list.txt
65+
pvs > ${LOG_DIR}/system_logs/pvs.txt
66+
vgs > ${LOG_DIR}/system_logs/vgs.txt
67+
lvs > ${LOG_DIR}/system_logs/lvs.txt
8468

8569
iptables-save > ${LOG_DIR}/system_logs/iptables.txt
8670

@@ -106,42 +90,35 @@ copy_logs() {
10690

10791
# Bifrost: grab config files and logs from the container.
10892
if [[ $(docker ps -q -f name=bifrost_deploy) ]]; then
93+
mkdir -p ${LOG_DIR}/bifrost
10994
for service in dnsmasq ironic-api ironic-conductor ironic-inspector mariadb nginx rabbitmq-server; do
110-
mkdir -p ${LOG_DIR}/kolla/$service
95+
mkdir -p ${LOG_DIR}/bifrost/$service
11196
docker exec bifrost_deploy \
112-
systemctl status $service -l -n 10000 > ${LOG_DIR}/kolla/$service/${service}-systemd-status.txt
97+
systemctl status $service -l -n 10000 > ${LOG_DIR}/bifrost/$service/${service}-systemd-status.txt
11398
docker exec bifrost_deploy \
114-
journalctl -u $service --no-pager > ${LOG_DIR}/kolla/$service/${service}-journal.txt
99+
journalctl -u $service --no-pager > ${LOG_DIR}/bifrost/$service/${service}-journal.txt
115100
done
116101
docker exec -it bifrost_deploy \
117-
journalctl --no-pager > ${LOG_DIR}/kolla/bifrost-journal.log
102+
journalctl --no-pager > ${LOG_DIR}/bifrost/bifrost-journal.log
118103
for d in dnsmasq.conf ironic ironic-inspector nginx/nginx.conf; do
119104
docker cp bifrost_deploy:/etc/$d ${LOG_DIR}/kolla_node_configs/bifrost/
120105
done
121-
docker cp bifrost_deploy:/var/log/mariadb/mariadb.log ${LOG_DIR}/kolla/mariadb/
106+
docker cp bifrost_deploy:/var/log/mariadb/mariadb.log ${LOG_DIR}/bifrost/mariadb/
122107
fi
123108

124109
# IPA build logs
125110
if [[ -f /opt/kayobe/images/ipa/ipa.stderr ]] || [[ -f /opt/kayobe/images/ipa/ipa.stdout ]]; then
126-
mkdir -p ${LOG_DIR}/kayobe
127-
cp /opt/kayobe/images/ipa/ipa.stderr /opt/kayobe/images/ipa/ipa.stdout ${LOG_DIR}/kayobe/
111+
mkdir -p ${LOG_DIR}/ipa
112+
cp /opt/kayobe/images/ipa/ipa.stderr /opt/kayobe/images/ipa/ipa.stdout ${LOG_DIR}/ipa/
128113
fi
129114

130115
# Overcloud host image build logs
131116
if [[ -f /opt/kayobe/images/deployment_image/deployment_image.stderr ]] || [[ -f /opt/kayobe/images/deployment_image/deployment_image.stdout ]]; then
132-
mkdir -p ${LOG_DIR}/kayobe
133-
cp /opt/kayobe/images/deployment_image/deployment_image.stderr /opt/kayobe/images/deployment_image/deployment_image.stdout ${LOG_DIR}/kayobe/
117+
mkdir -p ${LOG_DIR}/deployment_image
118+
cp /opt/kayobe/images/deployment_image/deployment_image.stderr /opt/kayobe/images/deployment_image/deployment_image.stdout ${LOG_DIR}/deployment_image/
134119
fi
135120

136-
# Rename files to .txt; this is so that when displayed via
137-
# logs.openstack.org clicking results in the browser shows the
138-
# files, rather than trying to send it to another app or make you
139-
# download it, etc.
140-
for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do
141-
mv $f ${f/.log/.txt}
142-
done
143-
144-
chmod -R 777 ${LOG_DIR}
121+
chown -R stack: ${LOG_DIR}
145122
}
146123

147124
copy_logs

0 commit comments

Comments
 (0)