test/ansible, pkg/scaffold; Update Ansible Operator CI to prevent failures (#985)

<!--

Before making a PR, please read our contributing guidelines https://github.com/operator-framework/operator-sdk/blob/master/CONTRIBUTING.MD

Note: Make sure your branch is rebased to the latest upstream master.

-->

**Description of the change:**
Updates Travis CI Molecule tasks to prevent transient errors. Also updates CI to dump operator logs if reconciliation fails.

**Motivation for the change:**
Fail CI less often for transient errors and make CI more robust.
<!--

Note: If this PR is fixing an issue make sure to add a note saying:
Closes #<ISSUE_NUMBER>

-->

Author: dymurray

pkg/scaffold/ansible/molecule_test_cluster_playbook.go

@@ -47,6 +47,7 @@ const moleculeTestClusterPlaybookAnsibleTmpl = `---
     ansible_python_interpreter: '{{ "{{ ansible_playbook_python }}" }}'
     deploy_dir: "{{"{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy"}}"
     image_name: {{.Resource.FullGroup}}/{{.ProjectName}}:testing
+    custom_resource: "{{"{{"}} lookup('file', '/'.join([deploy_dir, 'crds/{{.Resource.Group}}_{{.Resource.Version}}_{{.Resource.LowerKind}}_cr.yaml'])) | from_yaml {{"}}"}}"
   tasks:
   - name: Create the {{.Resource.FullGroup}}/{{.Resource.Version}}.{{.Resource.Kind}}
     k8s:
@@ -55,9 +56,19 @@ const moleculeTestClusterPlaybookAnsibleTmpl = `---
 
   - name: Get the newly created Custom Resource
     debug:
-      msg: "{{"{{"}} lookup('k8s', group='{{.Resource.FullGroup}}', api_version='{{.Resource.Version}}', kind='{{.Resource.Kind}}', namespace=namespace, resource_name=cr.metadata.name) {{"}}"}}"
-    vars:
-      cr: "{{"{{"}} lookup('file', '/'.join([deploy_dir, 'crds/{{.Resource.Group}}_{{.Resource.Version}}_{{.Resource.LowerKind}}_cr.yaml'])) | from_yaml {{"}}"}}"
+      msg: "{{"{{"}} lookup('k8s', group='{{.Resource.FullGroup}}', api_version='{{.Resource.Version}}', kind='{{.Resource.Kind}}', namespace=namespace, resource_name=custom_resource.metadata.name) {{"}}"}}"
+
+  - name: Wait 40s for reconcilation to run
+    k8s_facts:
+      api_version: '{{.Resource.Version}}'
+      kind: '{{.Resource.Kind }}'
+      namespace: '{{"{{"}} namespace {{"}}"}}'
+      name: '{{"{{"}} custom_resource.metadata.name {{"}}"}}'
+    register: reconcile_cr
+    until:
+    - "'Successful' in (reconcile_cr | json_query('resources[].status.conditions[].reason'))"
+    delay: 4
+    retries: 10
 
 - import_playbook: "{{"{{ playbook_dir }}/../default/asserts.yml"}}"
 `

pkg/scaffold/ansible/molecule_test_local_playbook.go

@@ -66,50 +66,92 @@ const moleculeTestLocalPlaybookAnsibleTmpl = `---
     REPLACE_IMAGE: {{.Resource.FullGroup}}/{{.ProjectName}}:testing
     custom_resource: "{{"{{"}} lookup('file', '/'.join([deploy_dir, 'crds/{{.Resource.Group}}_{{.Resource.Version}}_{{.Resource.LowerKind}}_cr.yaml'])) | from_yaml {{"}}"}}"
   tasks:
-  - name: Delete the Operator Deployment
-    k8s:
-      state: absent
-      namespace: '{{ "{{ namespace }}" }}'
-      definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) {{"}}"}}"
-    register: delete_deployment
-    when: hostvars[groups.k8s.0].build_cmd.changed
-
-  - name: Wait 30s for Operator Deployment to terminate
-    k8s_facts:
-      api_version: '{{"{{"}} definition.apiVersion {{"}}"}}'
-      kind: '{{"{{"}} definition.kind {{"}}"}}'
-      namespace: '{{"{{"}} namespace {{"}}"}}'
-      name: '{{"{{"}} definition.metadata.name {{"}}"}}'
-    vars:
-      definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml {{"}}"}}"
-    register: deployment
-    until: not deployment.resources
-    delay: 3
-    retries: 10
-    when: delete_deployment.changed
-
-  - name: Create the Operator Deployment
-    k8s:
-      namespace: '{{ "{{ namespace }}" }}'
-      definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) {{"}}"}}"
-
-  - name: Create the {{.Resource.FullGroup}}/{{.Resource.Version}}.{{.Resource.Kind}}
-    k8s:
-      state: present
-      namespace: '{{ "{{ namespace }}" }}'
-      definition: "{{ "{{ custom_resource }}" }}"
-
-  - name: Wait 40s for reconcilation to run
-    k8s_facts:
-      api_version: '{{"{{"}} custom_resource.apiVersion {{"}}"}}'
-      kind: '{{"{{"}} custom_resource.kind {{"}}"}}'
-      namespace: '{{"{{"}} namespace {{"}}"}}'
-      name: '{{"{{"}} custom_resource.metadata.name {{"}}"}}'
-    register: cr
-    until:
-    - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))"
-    delay: 4
-    retries: 10
+  - block:
+    - name: Delete the Operator Deployment
+      k8s:
+        state: absent
+        namespace: '{{ "{{ namespace }}" }}'
+        definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) {{"}}"}}"
+      register: delete_deployment
+      when: hostvars[groups.k8s.0].build_cmd.changed
+
+    - name: Wait 30s for Operator Deployment to terminate
+      k8s_facts:
+        api_version: '{{"{{"}} definition.apiVersion {{"}}"}}'
+        kind: '{{"{{"}} definition.kind {{"}}"}}'
+        namespace: '{{"{{"}} namespace {{"}}"}}'
+        name: '{{"{{"}} definition.metadata.name {{"}}"}}'
+      vars:
+        definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml {{"}}"}}"
+      register: deployment
+      until: not deployment.resources
+      delay: 3
+      retries: 10
+      when: delete_deployment.changed
+
+    - name: Create the Operator Deployment
+      k8s:
+        namespace: '{{ "{{ namespace }}" }}'
+        definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) {{"}}"}}"
+
+    - name: Create the {{.Resource.FullGroup}}/{{.Resource.Version}}.{{.Resource.Kind}}
+      k8s:
+        state: present
+        namespace: '{{ "{{ namespace }}" }}'
+        definition: "{{ "{{ custom_resource }}" }}"
+
+    - name: Wait 40s for reconcilation to run
+      k8s_facts:
+        api_version: '{{"{{"}} custom_resource.apiVersion {{"}}"}}'
+        kind: '{{"{{"}} custom_resource.kind {{"}}"}}'
+        namespace: '{{"{{"}} namespace {{"}}"}}'
+        name: '{{"{{"}} custom_resource.metadata.name {{"}}"}}'
+      register: cr
+      until:
+      - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))"
+      delay: 4
+      retries: 10
+    rescue:
+    - name: debug cr
+      ignore_errors: yes
+      failed_when: false
+      debug:
+        var: debug_cr
+      vars:
+        debug_cr: '{{"{{"}} lookup("k8s",
+          kind=custom_resource.kind,
+          api_version=custom_resource.apiVersion,
+          namespace=namespace,
+          resource_name=custom_resource.metadata.name
+        ){{"}}"}}'
+
+    - name: debug memcached lookup
+      ignore_errors: yes
+      failed_when: false
+      debug:
+        var: deploy
+      vars:
+        deploy: '{{"{{"}} lookup("k8s",
+          kind="Deployment",
+          api_version="apps/v1",
+          namespace=namespace,
+          label_selector="app=memcached"
+        ){{"}}"}}'
+
+    - name: get operator logs
+      ignore_errors: yes
+      failed_when: false
+      command: kubectl logs deployment/{{"{{"}} definition.metadata.name {{"}}"}} -n {{"{{"}} namespace {{"}}"}}
+      environment:
+        KUBECONFIG: '{{"{{"}} lookup("env", "KUBECONFIG") {{"}}"}}'
+      vars:
+        definition: "{{"{{"}} lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml {{"}}"}}"
+      register: log
+
+    - debug: var=log.stdout_lines
+
+    - fail:
+        msg: "Failed on action: converge"
 
 - import_playbook: '{{"{{ playbook_dir }}/../default/asserts.yml"}}'
 `

test/ansible-memcached/asserts.yml

@@ -8,81 +8,118 @@
     deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy"
     custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/ansible_v1alpha1_memcached_cr.yaml'])) | from_yaml }}"
   tasks:
+    - block:
+      - name: debug memcached lookup
+        debug:
+          var: deploy
+        vars:
+          deploy: '{{ lookup("k8s",
+            kind="Deployment",
+            api_version="apps/v1",
+            namespace=namespace,
+            label_selector="app=memcached"
+          )}}'
 
-    - name: debug memcached lookup
-      debug:
-        var: deploy
-      vars:
-        deploy: '{{ lookup("k8s",
-          kind="Deployment",
-          api_version="apps/v1",
-          namespace=namespace,
-          label_selector="app=memcached"
-        )}}'
+      - name: Wait 2 minutes for memcached deployment
+        debug:
+          var: deploy
+        until: deploy and deploy.status and deploy.status.replicas == deploy.status.get("availableReplicas", 0)
+        retries: 12
+        delay: 10
+        vars:
+          deploy: '{{ lookup("k8s",
+            kind="Deployment",
+            api_version="apps/v1",
+            namespace=namespace,
+            label_selector="app=memcached"
+          )}}'
 
-    - name: Wait 2 minutes for memcached deployment
-      debug:
-        var: deploy
-      until: deploy and deploy.status and deploy.status.replicas == deploy.status.get("availableReplicas", 0)
-      retries: 12
-      delay: 10
-      vars:
-        deploy: '{{ lookup("k8s",
-          kind="Deployment",
-          api_version="apps/v1",
-          namespace=namespace,
-          label_selector="app=memcached"
-        )}}'
+      - name: Create ConfigMap that the Operator should delete
+        k8s:
+          definition:
+            apiVersion: v1
+            kind: ConfigMap
+            metadata:
+              name: deleteme
+              namespace: '{{ namespace }}'
+            data:
+              delete: me
 
-    - name: Create ConfigMap that the Operator should delete
-      k8s:
-        definition:
-          apiVersion: v1
-          kind: ConfigMap
-          metadata:
-            name: deleteme
-            namespace: '{{ namespace }}'
-          data:
-            delete: me
+      - name: Verify custom status exists
+        assert:
+          that: debug_cr.status.get("test") == "hello world"
+        vars:
+          debug_cr: '{{ lookup("k8s",
+            kind=custom_resource.kind,
+            api_version=custom_resource.apiVersion,
+            namespace=namespace,
+            resource_name=custom_resource.metadata.name
+          )}}'
 
-    - name: debug CR lookup
-      debug:
-        var: cr
-      vars:
-        deploy: '{{ lookup("k8s",
-          kind=custom_resource.kind,
-          api_version=custom_resource.apiVersion,
-          namespace=namespace,
-          resource_name=custom_resource.metadata.name
-        )}}'
+      - name: Delete the custom resource
+        k8s:
+          state: absent
+          namespace: '{{ namespace }}'
+          definition: '{{ custom_resource }}'
 
-    - name: Verify custom status exists
-      assert:
-        that: cr.resources[0].status.get("test") == "hello world"
-      when: cr is defined
+      - name: Wait for the custom resource to be deleted
+        k8s_facts:
+          api_version: '{{ custom_resource.apiVersion }}'
+          kind: '{{ custom_resource.kind }}'
+          namespace: '{{ namespace }}'
+          name: '{{ custom_resource.metadata.name }}'
+        register: cr
+        retries: 10
+        delay: 2
+        until: not cr.resources
+        failed_when: cr.resources
 
-    - name: Delete the custom resource
-      k8s:
-        state: absent
-        namespace: '{{ namespace }}'
-        definition: '{{ custom_resource }}'
+      - name: Verify the ConfigMap was deleted
+        assert:
+          that: not lookup('k8s', kind='ConfigMap', api_version='v1', namespace=namespace, resource_name='deleteme')
 
-    - name: Wait for the custom resource to be deleted
-      k8s_facts:
-        api_version: '{{ custom_resource.apiVersion }}'
-        kind: '{{ custom_resource.kind }}'
-        namespace: '{{ namespace }}'
-        name: '{{ custom_resource.metadata.name }}'
-      register: cr
-      retries: 10
-      delay: 2
-      until: not cr.resources
-      failed_when: cr.resources
+      - name: Verify the Deployment was deleted (wait 30s)
+        assert:
+          that: not lookup('k8s', kind='Deployment', api_version='apps/v1', namespace=namespace, label_selector='app=memcached')
+        retries: 10
+        delay: 3
 
-    - name: Verify the ConfigMap was deleted
-      assert:
-        that: not lookup('k8s', kind='ConfigMap', api_version='v1', namespace=namespace, resource_name='deleteme')
+      rescue:
+      - name: debug cr
+        ignore_errors: yes
+        failed_when: false
+        debug:
+          var: debug_cr
+        vars:
+          debug_cr: '{{ lookup("k8s",
+            kind=custom_resource.kind,
+            api_version=custom_resource.apiVersion,
+            namespace=namespace,
+            resource_name=custom_resource.metadata.name
+          )}}'
 
-    - name: Verify the Deployment was deleted
-      assert:  
-        that: not lookup('k8s', kind='Deployment', api_version='apps/v1', namespace=namespace, label_selector='app=memcached')
+      - name: debug memcached lookup
+        ignore_errors: yes
+        failed_when: false
+        debug:
+          var: deploy
+        vars:
+          deploy: '{{ lookup("k8s",
+            kind="Deployment",
+            api_version="apps/v1",
+            namespace=namespace,
+            label_selector="app=memcached"
+          )}}'
+
+      - name: get operator logs
+        ignore_errors: yes
+        failed_when: false
+        command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ namespace }}
+        vars:
+          definition: "{{ lookup('file', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
+        register: log
+
+      - debug: var=log.stdout_lines
+
+      - fail:
+          msg: "Failed in asserts.yml"