branch-validation.yml

# playbook for validating a branch for release tagging
# interactive, may prompt runner to affirm things have completed
---
- name: Run rally and validate
  hosts: localhost
  gather_facts: false

  tasks:
    - name: run rally
      pause: prompt="Was rally validation run successful? (yes/no)"
      register: rally

    - name: assert rally successful
      assert:
        that: rally.user_input|bool

- name: test instance connectivity
  hosts: controller[0]

  tasks:
### Build up
    - name: create keypair
      os_keypair:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        name: validate
        state: present
      register: keypair

    - name: persist private key
      copy:
        content: "{{ keypair.key.private_key }}"
        dest: /root/validate-key
        mode: 0600
      when: keypair.changed

    - name: create security group
      os_security_group:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        name: validate_ssh
        description: "validate group allowing ssh, icmp ipv4 ingress"
        state: present

    - name: create security group rules
      os_security_group_rule:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        security_group: validate_ssh
        protocol: "{{ item.ip_protocol }}"
        port_range_min: "{{ item.from_port }}"
        port_range_max: "{{ item.to_port }}"
        remote_ip_prefix: "{{ item.cidr }}"
        state: present
        with_items:
          - ip_protocol: tcp
            from_port: 22
            to_port: 22
            cidr: 0.0.0.0/0
          - ip_protocol: icmp
            from_port: -1
            to_port: -1
            cidr: 0.0.0.0/0
            state: present

    - name: launch instances in each host az
      os_server:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        flavor: 1
        image: cirros
        key_name: validate
        auto_floating_ip: no
        security_groups: default,validate_ssh
        #floating_ip_pools:
        #  - external
        nics:
          - net-name: internal
        availability_zone: "nova:{{ item }}"
        name: validate-{{ item }}
        wait: "yes"
      with_items: groups['compute']
      register: instances

    - name: associate a floating ip
      os_floating_ip:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        network: external
        server: validate-{{ groups['compute'][0] }}
        reuse: true
      register: fip

    - name: wait for instances to be reachable
      wait_for:
        port: 22
        delay: 5
        timeout: 300
        host: "{{ fip.floating_ip.floating_ip_address }}"

### Node connectivity tests

    - name: cross-node inter-instance connectivity test
      command: ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
               -o IdentityFile=/root/validate-key
               cirros@{{ fip.floating_ip.floating_ip_address }}
               ping -c 5 {{ item.private_ip }}
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      with_items: instances.results

    - name: instance internet connectivity test w/ floating-ip
      command: ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
               -o IdentityFile=/root/validate-key
               cirros@{{ fip.floating_ip.floating_ip_address }}
               ping -c 5 google.com
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5

    - name: wait for user to login
      pause: prompt="From {{ groups['controller'][0] }} ssh -o
                     UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
                     -o IdentityFile=/root/validate-key
                     cirros@{{ fip.floating_ip.floating_ip_address }}
                     (press enter when done)"

    - name: disassociate the floating ip
      os_floating_ip:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        network: external
        server: validate-{{ groups['compute'][0] }}
        floating_ip_address: "{{ fip.floating_ip.floating_ip_address }}"
        state: absent

    - name: instance internet connectivity test w/out floating-ip
      pause: prompt="Can you ping -c 5 google.com ? (yes/no)"
      register: ping

    - name: assert ping
      assert:
        that: ping.user_input|bool

### HA testing

    - name: add floating IPs to instances on controllers
      os_floating_ip:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        network: external
        server: validate-{{ item }}
        reuse: true
      register: fip_controllers
      with_items: groups['controller']

    - name: add floating ip to a non-controller instance
      os_floating_ip:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        network: external
        server: validate-{{ item }}
        reuse: true
      register: fip_compute
      with_items:
        - "{{ (groups['compute']|difference(groups['controller']))[0] }}"

    - name: floating ips are reachable
      command: ping -c 5 {{ item.floating_ip.floating_ip_address }}
      with_items: fip_controllers.results + fip_compute.results
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      delegate_to: localhost

    - name: define ha master
      set_fact: master=true
      when: floating_ip in ansible_all_ipv4_addresses

    # initiate hard reboot on the current master controller
    - name: initiate a failover on controller 0
      shell: echo 'b' | tee /proc/sysrq-trigger
      when: master is defined

    - name: initiate a failover on controller 1
      delegate_to: "{{ groups['controller'][1] }}"
      shell: echo 'b' | tee /proc/sysrq-trigger
      when: master is not defined

    - name: wait for the panic
      wait_for: host={{ inventory_hostname }} port=22 state=stopped delay=10
                timeout=30
      when: master is defined
      delegate_to: localhost

    - name: wait for the panic
      wait_for: host={{ groups['controller'][1] }} port=22 state=stopped
                delay=10 timeout=30
      when: master is not defined
      delegate_to: localhost

    - name: wait for the node to boot
      pause: seconds=30

    - name: check non-controller instance connectivity by way of floating IPs
      command: ping -c 5
               {{ fip_compute.results[0].floating_ip.floating_ip_address }}
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      delegate_to: localhost

    - name: check remaining controller instance connectivity
      command: ping -c 5
               {{ fip_controllers.results[0].floating_ip.floating_ip_address }}
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      when: master is not defined
      delegate_to: localhost

    - name: check remaining controller instance connectivity
      command: ping -c 5
               {{ fip_controllers.results[1].floating_ip.floating_ip_address }}
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      delegate_to: localhost
      when: master is defined

    - name: wait for the reboot
      wait_for: host={{ inventory_hostname }} port=22 delay=10
                timeout=300
      delegate_to: localhost
      when: master is defined

    - name: wait for the reboot
      wait_for: host={{ groups['controller'][1] }} port=22 delay=10
                timeout=300
      delegate_to: localhost
      when: master is not defined

    - name: floating ips are reachable
      command: ping -c 5 {{ item.floating_ip.floating_ip_address }}
      with_items: fip_controllers.results + fip_compute.results
      changed_when: false
      register: pings
      until: pings|success
      delay: 1
      retries: 5
      delegate_to: localhost

### Tear down

    - name: delete instances
      os_server:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        name: validate-{{ item }}
        state: absent
        wait: "yes"
      with_items: groups['compute']
      tags: delete

    - name: delete security group
      os_security_group:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        name: validate_ssh
        state: absent
      tags: delete

    - name: delete private key file
      file:
        dest: /root/validate-key
        state: absent
      tags: delete

    - name: delete keypair
      os_keypair:
        auth:
          username: admin
          project_name: admin
          password: "{{ secrets.admin_password }}"
          auth_url: "http://127.0.0.1:{{ endpoints.keystone_admin.port.backend_api }}/{{ endpoints.keystone.version }}"
        name: validate
        state: absent
      tags: delete