Merge pull request #2216 from broadinstitute/jb-staging-deploy-schedule

Adding scripts for dealing with scheduled instance restart (SCP-5949)

Author: bistline

bin/remount_portal_source.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+# script to add to the root crontab to remount attached disk in correct location after scheduled instance restart,
+# which is used for staging and development VMs, but not production
+
+# crontab should be entered as follows
+# @reboot /root/remout_portal_source.sh > /dev/null 2>&1
+
+# More context: https://github.com/broadinstitute/single_cell_portal_core/pull/2216
+MOUNT_DIR=$(ls -l /dev/disk/by-id/google-* | grep google-singlecell-data-disk | awk -F '/' '{ print $NF }')
+if [[ -n "$MOUNT_DIR" ]]; then
+  echo "$(date): remounting google-singlecell-data-disk from /dev/$MOUNT_DIR" >> /home/jenkins/remount_log.txt
+  mount -o discard,defaults /dev/$MOUNT_DIR /home/jenkins/deployments
+else
+  echo -e "$(date): cannot remount google-singlecell-data-disk, available disks:\n$(ls -l /dev/disk/by-id/google-*)" >> /home/jenkins/remount_log.txt
+fi

bin/restart_portal_container.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+# script to add to root crontab on a deployed host to check for crashed Docker containers and restart
+# crontab entry should be as follows:
+# */5 * * * * /root/restart_portal_container.sh > /dev/null 2>&1
+
+# More context: https://github.com/broadinstitute/single_cell_portal_core/pull/2216
+docker ps --filter "status=exited" | grep -e 'single_cell' | while read -r line ; do
+  container_id=`echo $line | awk '{print $1}'`
+  container_name=`echo $line | awk '{print $NF}'`
+  echo "Restarting $container_name ($container_id) on $(date)" >> /home/jenkins/deployments/single_cell_portal_core/log/cron_out.log
+  docker restart $container_id
+done