diff --git a/collection/stages/roles/cpms_test/defaults/main.yml b/collection/stages/roles/cpms_test/defaults/main.yml new file mode 100644 index 00000000..b9bc650e --- /dev/null +++ b/collection/stages/roles/cpms_test/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# defaults file for cpms_test +cpms_test_name: cluster-control-plane-machine-set-operator +cpms_test_dir: "{{ artifacts_dir }}/{{ cpms_test_name }}" +cpms_test_results_dir: "{{ artifacts_dir }}/cpms_test-results" +cpms_test_testsuite_name: cpms +cpms_tests_go_version: "{{ tests.default_go_version_target }}" +cpms_test_junit_filename: junit_control_plane_machine_set_operator.xml diff --git a/collection/stages/roles/cpms_test/meta/main.yml b/collection/stages/roles/cpms_test/meta/main.yml new file mode 100644 index 00000000..ce0638dc --- /dev/null +++ b/collection/stages/roles/cpms_test/meta/main.yml @@ -0,0 +1,3 @@ +--- +collections: + - shiftstack.tools diff --git a/collection/stages/roles/cpms_test/tasks/main.yml b/collection/stages/roles/cpms_test/tasks/main.yml new file mode 100644 index 00000000..0fedc0ba --- /dev/null +++ b/collection/stages/roles/cpms_test/tasks/main.yml @@ -0,0 +1,25 @@ +--- +# tasks file for cpms_test +- name: Prepare CPMS test + ansible.builtin.include_role: + name: tools_openshift_tests + tasks_from: prepare_openshift_tests.yml + vars: + repo_name: "{{ cpms_test_name }}" + results_dir: "{{ cpms_test_results_dir }}" + go_version_target: "{{ cpms_tests_go_version }}" + +- name: Run CPMS e2e-presubmit tests + ansible.builtin.include_tasks: run_cpms_test.yml + vars: + cpms_tests_type: e2e-presubmit + +- name: Run CPMS e2e-periodic tests + ansible.builtin.include_tasks: run_cpms_test.yml + vars: + cpms_tests_type: e2e-periodic + +- name: Remove the source directory after tests complete + ansible.builtin.file: + path: "{{ cpms_test_dir }}" + state: absent diff --git a/collection/stages/roles/cpms_test/tasks/run_cpms_test.yml b/collection/stages/roles/cpms_test/tasks/run_cpms_test.yml new file mode 100644 index 00000000..062736b1 --- /dev/null +++ b/collection/stages/roles/cpms_test/tasks/run_cpms_test.yml @@ -0,0 +1,57 @@ +--- +- name: Set result paths for {{ cpms_tests_type }} + ansible.builtin.set_fact: + cpms_results_subdir: "{{ cpms_test_results_dir }}/{{ cpms_tests_type }}" + +- name: Create results directory for {{ cpms_tests_type }} + ansible.builtin.file: + path: "{{ cpms_results_subdir }}" + state: directory + mode: u=rwx,g=rw,o=r + +- name: Run {{ cpms_test_name }} {{ cpms_tests_type }} tests + block: + - name: Run make {{ cpms_tests_type }} + ansible.builtin.shell: | + source {{ home_dir }}/.bashrc + make {{ cpms_tests_type }} + args: + chdir: "{{ cpms_test_dir }}" + environment: + KUBECONFIG: "{{ kubeconfig }}" + OS_CLOUD: "{{ user_cloud }}" + OPENSHIFT_CI: "true" + ARTIFACT_DIR: "{{ cpms_results_subdir }}" + changed_when: true + + - name: Rename JUnit XML to match expected prefix + ansible.builtin.copy: + src: "{{ cpms_results_subdir }}/{{ cpms_test_junit_filename }}" + dest: "{{ cpms_results_subdir }}/junit_e2e_{{ cpms_tests_type }}.xml" + remote_src: yes + mode: u=rw,g=rw,o=r + + - name: Post openshift-test + ansible.builtin.include_role: + name: tools_openshift_tests + tasks_from: post_openshift_tests.yml + vars: + testsuite_name: "{{ cpms_test_testsuite_name }}_{{ cpms_tests_type }}" + key_for_filtering_results: "cpms" + test_name: "{{ cpms_test_name }}-{{ cpms_tests_type }}" + results_dir: "{{ cpms_results_subdir }}" + + rescue: + - name: Mark the CPMS {{ cpms_tests_type }} tests as UNSTABLE + ansible.builtin.include_role: + name: tools_stage_results + tasks_from: mark_stage_unstable.yml + vars: + unstable_msg: >- + The {{ cpms_test_name }} {{ cpms_tests_type }} test suite failed. + + - name: Run must-gather + ansible.builtin.include_role: + name: tools_must-gather + vars: + must_gather_suffix: "cpms-{{ cpms_tests_type }}" diff --git a/collection/stages/roles/day2ops/tasks/procedures/cpms_replace_attrs.yml b/collection/stages/roles/day2ops/tasks/procedures/cpms_replace_attrs.yml new file mode 100644 index 00000000..95d0812c --- /dev/null +++ b/collection/stages/roles/day2ops/tasks/procedures/cpms_replace_attrs.yml @@ -0,0 +1,228 @@ +--- +# Procedure that updates the CPMS object by adding a fake extra network/subnet and SG to the +# masters and replacing failureDomain params on master-0 to have the same attributes as master-1. +# After reconciliation, master-0 and master-1 will have same nova AZ, cinderAZ, and volumeType. +- name: Set log directory for cpms_replace_attrs procedure + ansible.builtin.set_fact: + cpms_log_directory: "{{ artifacts_dir }}/cpms_replace_attrs" + +- name: Run cpms_replace_attrs procedure + block: + - name: Create {{ cpms_log_directory }} directory + ansible.builtin.file: + path: "{{ cpms_log_directory }}" + state: directory + mode: u=rwx,g=rw,o=r + + - name: Set artifact paths + ansible.builtin.set_fact: + cpms_orig_path: "{{ cpms_log_directory }}/orig_cpms.yaml" + cpms_patch_to_apply_path: "{{ cpms_log_directory }}/cpms_patch_to_apply.yaml" + cpms_applied_path: "{{ cpms_log_directory }}/applied_cpms.yaml" + cpms_patch_to_restore_path: "{{ cpms_log_directory }}/cpms_patch_to_restore_NOTAPPLIED.yaml" + + - name: Get original CPMS definition + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: machine.openshift.io/v1 + kind: ControlPlaneMachineSet + namespace: openshift-machine-api + register: original_cpms + + - name: Save the original CPMS definition + ansible.builtin.copy: + content: "{{ original_cpms.resources[0] | to_nice_yaml(indent=2) }}" + dest: "{{ cpms_orig_path }}" + mode: u=rw,g=rw,o=r + + - name: Create network {{ cpms_replacements.network_name }} + openstack.cloud.network: + cloud: "{{ user_cloud }}" + name: "{{ cpms_replacements.network_name }}" + state: present + register: network_to_add + + - name: Create subnet {{ cpms_replacements.subnet_name }} + openstack.cloud.subnet: + cloud: "{{ user_cloud }}" + name: "{{ cpms_replacements.subnet_name }}" + cidr: "{{ cpms_replacements.cidr }}" + network_name: "{{ cpms_replacements.network_name }}" + enable_dhcp: false + state: present + + - name: Create security group {{ cpms_replacements.sg_name }} + openstack.cloud.security_group: + cloud: "{{ user_cloud }}" + name: "{{ cpms_replacements.sg_name }}" + state: present + register: sg_to_add + + - name: Store original CPMS attributes that will be updated + ansible.builtin.set_fact: + orig_failuredomains: >- + {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.failureDomains.openstack | list }} + orig_networks: >- + {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.networks | list }} + orig_sgs: >- + {{ original_cpms.resources[0].spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value.securityGroups | list }} + + - name: Create patch for CPMS + ansible.builtin.template: + src: control-plane-machine-set-patch.yaml.j2 + dest: "{{ cpms_patch_to_apply_path }}" + mode: u=rw,g=rw,o=r + vars: + cpms_failuredomains: >- + {{ orig_failuredomains | + replace(orig_failuredomains[0].availabilityZone, orig_failuredomains[1].availabilityZone) | + replace(orig_failuredomains[0].rootVolume.availabilityZone, orig_failuredomains[1].rootVolume.availabilityZone) | + replace(orig_failuredomains[0].rootVolume.volumeType, orig_failuredomains[1].rootVolume.volumeType) }} + cpms_networks: "{{ orig_networks + [{'filter': {}, 'uuid': network_to_add.id}] }}" + cpms_sgs: "{{ orig_sgs + [{'filter': {}, 'name': cpms_replacements.sg_name}] }}" + + - name: Apply patch for CPMS + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + apply: true + src: "{{ cpms_patch_to_apply_path }}" + + - name: Wait for CPMS reconciliation to complete + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: wait_cpms_updated.yml + vars: + wait_retries: 180 + wait_delay: 60 + + - name: Check cluster health after CPMS patch application + block: + - name: Check the Control Plane MachineSet is healthy + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: check_controlplane_machinesets.yml + + - name: Wait until OCP cluster is healthy + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: wait_until_cluster_is_healthy.yml + + - name: Get resulting CPMS definition after application + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: machine.openshift.io/v1 + kind: ControlPlaneMachineSet + namespace: openshift-machine-api + register: applied_cpms + + - name: Save the resulting CPMS after application + ansible.builtin.copy: + content: "{{ applied_cpms.resources[0] | to_nice_yaml(indent=2) }}" + dest: "{{ cpms_applied_path }}" + mode: u=rw,g=rw,o=r + + - name: Get all the master VMs after procedure + openstack.cloud.server_info: + cloud: "{{ user_cloud }}" + server: "*master*" + register: result + + - name: Store the master VMs after procedure + ansible.builtin.set_fact: + master_after: "{{ result.servers | sort(attribute='name') | list }}" + + - name: Get root volumes info + ansible.builtin.shell: > + openstack volume show {{ item }} -c type -c availability_zone -f shell + environment: + OS_CLOUD: "{{ user_cloud }}" + register: root_volumes_after + changed_when: false + with_items: + - "{{ master_after[0].volumes[0].id }}" + - "{{ master_after[1].volumes[0].id }}" + - "{{ master_after[2].volumes[0].id }}" + + - name: Confirm that master VMs reflect FailureDomain changes after procedure + ansible.builtin.assert: + that: + - master_after | length == 3 + - master_after | selectattr('location.zone', 'eq', expected_fd_az) | list | length == 2 + - root_volumes_after.results | selectattr('stdout', 'eq', + 'availability_zone="' + expected_fd_cinderaz + '"\ntype="' + expected_fd_volumeType + '"') | list | length == 2 + vars: + expected_fd_az: "{{ orig_failuredomains[1].availabilityZone }}" + expected_fd_cinderaz: "{{ orig_failuredomains[1].rootVolume.availabilityZone }}" + expected_fd_volumeType: "{{ orig_failuredomains[1].rootVolume.volumeType }}" + + - name: Confirm that the VMs include the added network and SG + ansible.builtin.assert: + that: + - cpms_replacements.network_name in item.addresses.keys() + - cpms_replacements.sg_name in item.security_groups | json_query('[*].name') + with_items: "{{ master_after }}" + + rescue: + - name: Run must-gather after cpms_replace_attrs failure + ansible.builtin.include_role: + name: tools_must-gather + vars: + must_gather_suffix: "cpms-replace-attrs" + + - name: Fail inside rescue block + ansible.builtin.fail: + msg: "The cpms_replace_attrs procedure failed" + + always: + - name: Create restore patch for CPMS + ansible.builtin.template: + src: control-plane-machine-set-patch.yaml.j2 + dest: "{{ cpms_patch_to_restore_path }}" + mode: u=rw,g=rw,o=r + vars: + cpms_failuredomains: "{{ orig_failuredomains }}" + cpms_networks: "{{ orig_networks }}" + cpms_sgs: "{{ orig_sgs }}" + + - name: Apply restore patch for CPMS + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + apply: true + src: "{{ cpms_patch_to_restore_path }}" + + - name: Wait for CPMS restore reconciliation to complete + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: wait_cpms_updated.yml + vars: + wait_retries: 180 + wait_delay: 60 + + - name: Check cluster health after restore + block: + - name: Check the Control Plane MachineSet is healthy + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: check_controlplane_machinesets.yml + + - name: Wait until OCP cluster is healthy + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: wait_until_cluster_is_healthy.yml + + - name: Clean up test OpenStack resources + block: + - name: Delete test network + openstack.cloud.network: + cloud: "{{ user_cloud }}" + name: "{{ cpms_replacements.network_name }}" + state: absent + + - name: Delete test security group + openstack.cloud.security_group: + cloud: "{{ user_cloud }}" + name: "{{ cpms_replacements.sg_name }}" + state: absent + ignore_errors: true diff --git a/collection/stages/roles/day2ops/templates/control-plane-machine-set-patch.yaml.j2 b/collection/stages/roles/day2ops/templates/control-plane-machine-set-patch.yaml.j2 new file mode 100644 index 00000000..0082d7da --- /dev/null +++ b/collection/stages/roles/day2ops/templates/control-plane-machine-set-patch.yaml.j2 @@ -0,0 +1,19 @@ +apiVersion: machine.openshift.io/v1 +metadata: + name: cluster + namespace: openshift-machine-api +kind: ControlPlaneMachineSet +spec: + template: + machines_v1beta1_machine_openshift_io: + failureDomains: + openstack: + {{cpms_failuredomains| to_nice_yaml| indent(8, false)}} + spec: + providerSpec: + value: + networks: + {{cpms_networks| to_nice_yaml| indent(12, false)}} + securityGroups: + {{cpms_sgs| to_nice_yaml| indent(12, false)}} +status: {} diff --git a/jobs_definitions/4.17_ovnkubernetes_ipi.yaml b/jobs_definitions/4.17_ovnkubernetes_ipi.yaml index c722992f..d948af1a 100644 --- a/jobs_definitions/4.17_ovnkubernetes_ipi.yaml +++ b/jobs_definitions/4.17_ovnkubernetes_ipi.yaml @@ -10,6 +10,7 @@ stages: - post - verification - day2ops + - cpms_test - openstack_test - conformance_test - cinder_csi_tests @@ -17,6 +18,13 @@ stages: day2ops_procedures: - moving-etcd-to-ephemeral + - cpms_replace_attrs + +cpms_replacements: + network_name: cpms-test-network + subnet_name: cpms-test-subnet + cidr: 192.168.240.0/24 + sg_name: cpms-test-sg ocp_deployment_topology: network_type: OVNKubernetes diff --git a/jobs_definitions/osp_verification.yaml b/jobs_definitions/osp_verification.yaml index 98d341b6..a4912650 100644 --- a/jobs_definitions/osp_verification.yaml +++ b/jobs_definitions/osp_verification.yaml @@ -19,6 +19,7 @@ stages: - install - post - verification + - cpms_test - openstack_test - lb_tests diff --git a/playbooks/ocp_testing.yaml b/playbooks/ocp_testing.yaml index ac386333..71715bbc 100644 --- a/playbooks/ocp_testing.yaml +++ b/playbooks/ocp_testing.yaml @@ -96,6 +96,10 @@ ansible.builtin.import_playbook: plays/conformance_test.yaml when: "'conformance_test' in stages" +- name: Run CPMS e2e tests on OpenShift + ansible.builtin.import_playbook: plays/cpms_test.yaml + when: "'cpms_test' in stages" + - name: Run Openshift Cinder CSI Tests on OpenShift ansible.builtin.import_playbook: plays/cinder_csi.yaml when: "'cinder_csi_tests' in stages" diff --git a/playbooks/plays/cpms_test.yaml b/playbooks/plays/cpms_test.yaml new file mode 100644 index 00000000..1c613a3f --- /dev/null +++ b/playbooks/plays/cpms_test.yaml @@ -0,0 +1,17 @@ +--- +- name: Run CPMS e2e tests on OpenShift + hosts: installer + gather_facts: no + vars_files: + - "../../configs/global.yml" + tasks: + - name: Main block + block: + - name: Run CPMS e2e tests on OCP + ansible.builtin.include_role: + name: shiftstack.stages.cpms_test + always: + - name: Synchronize artifacts from the Ansible Managed Node to Ansible Controller + ansible.builtin.include_role: + name: shiftstack.tools.tools_ansible_inventory + tasks_from: sync_artifacts.yml