refactor(pgsql): patroni restart node

tested scenarios:
- `./pgsql.yml -l <pg_cls> -t pt_restart -e "pt_restart=true"`
   - all nodes have been restarted
- `./pgsql.yml -l '<ip>,&<pg_cls>' -t pt_restart -e "pt_restart=true"`
   - nodes in ansible_play_hosts_all have been restarted, other nodes in the same pg_cls ignored
- `./pgsql.yml -l '<replica>,<primary>,&<pg_cls>' -t pt_restart -e "pt_restart=true"`
   - nodes in ansible_play_hosts_all have been restarted, other nodes in the same pg_cls ignored
   - switchover triggered when patroni restart primary failed

Author: waitingsong

roles/pgsql/tasks/grace_patroni_restart.yml

@@ -14,10 +14,12 @@
 #--------------------------------------------------------------#
 # tested scenarios:
 #   - ./pgsql.yml -l <pg_cls> -t pt_restart -e "pt_restart=true"
-#     all nodes have been restarted
+#      - all nodes have been restarted
 #   - ./pgsql.yml -l '<ip>,&<pg_cls>' -t pt_restart -e "pt_restart=true"
-#     nodes in ansible_play_hosts_all have been restarted,
-#     other nodes in the same pg_cls ignored
+#      - nodes in ansible_play_hosts_all have been restarted, other nodes in the same pg_cls ignored
+#   - ./pgsql.yml -l '<replica>,<primary>,&<pg_cls>' -t pt_restart -e "pt_restart=true"
+#      - nodes in ansible_play_hosts_all have been restarted, other nodes in the same pg_cls ignored
+#      - switchover triggered when patroni restart primary failed
 #--------------------------------------------------------------#
 
 - name: set variable

roles/pgsql/tasks/util/patroni_current_member.yml

@@ -0,0 +1,20 @@
+#!/usr/bin/ansible-playbook
+---
+#--------------------------------------------------------------#
+# get current member of patroni cluster
+#--------------------------------------------------------------#
+
+- name: get current member of patroni cluster
+  tags: [ pg_hugepage, patroni, pg_launch, pt_restart ]
+  become_user: "{{ pg_dbsu }}"
+  block:
+    - name: retrieve current patroni member of {{ pg_cluster }}
+      command: /pg/bin/pg-member
+      register: pg_member_name_cmd
+
+    - name: set variable curr_pg_member_name
+      set_fact:
+        curr_pg_member_name: "{{ pg_member_name_cmd.stdout | default('') | trim }}"
+      changed_when: false
+
+...

roles/pgsql/tasks/util/patroni_restart.yml

@@ -0,0 +1,31 @@
+#!/usr/bin/ansible-playbook
+---
+#--------------------------------------------------------------#
+# patroni restart nodes individually
+#--------------------------------------------------------------#
+
+- name: patroni restart node
+  tags: [ pg_hugepage, patroni, pg_launch, pt_restart ]
+  become_user: "{{ pg_dbsu }}"
+  when: patroni_mode != 'remove'
+  block:
+    - name: get current patroni member of {{ pg_cluster }}
+      import_tasks: patroni_current_member.yml
+
+    # result may be success but contains 'Failed' message,
+    # due to node has multiple ip addresses and got "Failed: ... status code=403, (Access is denied)" for replica
+    - name: patroni restart members of {{ pg_cluster }}
+      when: curr_pg_member_name is defined and curr_pg_member_name|length>0
+      command: /usr/bin/patronictl -c /pg/bin/patroni.yml restart --force {{ pg_cluster }} {{ curr_pg_member_name }}
+      register: patroni_restart_cmd
+      failed_when: patroni_restart_cmd.rc != 0 or patroni_restart_cmd.stdout.find('Failed') != -1
+
+    - name: sleep after restart if succeed at primary
+      when: pg_role_runtime is defined and pg_role_runtime == 'primary'
+      args: { executable: /bin/bash }
+      shell: |
+        sync; sync;
+        sleep 3
+      changed_when: false
+
+...

roles/pgsql/tasks/util/patroni_restart_at_primary.yml

@@ -8,21 +8,12 @@
   vars:
     dbsu: "{{ pg_dbsu|default('postgres') }}"
   block:
-    # if ansible_play_hosts_all contains primary node, restart all nodes at primary
-    - name: restart all nodes of {{ pg_cluster }} at primary
+    - name: restart primary of {{ pg_cluster }}
       when: pg_role == 'primary'
-      include_tasks: patroni_restart_at_primary.yml
+      include_tasks: patroni_restart.yml
 
-    - name: set variable need_restart_at_replica at all nodes
-      set_fact:
-        need_restart_at_replica: "{{ not (hostvars[pg_primary_host_runtime].pt_restart_full_succeed | default(false) | string | trim | bool) }}"
-      changed_when: false
-
-    # if ansible_play_hosts_all not contains primary node,
-    # or primary node restart failed,
-    # restart affected replica nodes at replica
-    - name: patroni restart ansible_play_hosts_all at replica if needed
-      include_tasks: patroni_restart_at_replica.yml
-      when: pg_role != 'primary' and need_restart_at_replica == true
+    - name: patroni restart replica of {{ pg_cluster }}
+      include_tasks: patroni_restart.yml
+      when: pg_role != 'primary'
 
 ...