Update germinator deployment for modern Ubuntu

deploy/as-admin.sh

@@ -14,6 +14,14 @@ OPENTREE_USER=$2
 CERTIFICATE_FILE=$3
 CERTIFICATE_KEY_FILE=$4
 
+# copied from setup/functions.sh (not yet available), so we can log admin stuff
+ADMIN_HOME=$(bash <<< "echo ~")
+function log() {
+    mkdir -p "$ADMIN_HOME/log" || exit 1
+    (echo `date` as-admin.sh: " $*") >>"$ADMIN_HOME/log/messages"  || exit
+}
+log ">>> as-admin.sh starting"
+
 if [ x$OPENTREE_USER = x ]; then
     OPENTREE_USER=opentree
 fi
@@ -42,6 +50,16 @@ function apt_get_install_one
     fi
 }
 
+# Add Bionic repository for these python2.* packages (in newer Ubuntu)
+#  python-venv (vs python3-venv)
+#  python-pip (vs python3-pip)
+#  python-requests (vs python3-requests)
+# These are required by our current version of web2py
+apt_get_install_one software-properties-common  # needed for add-apt-repository
+sudo apt-add-repository "deb http://us.archive.ubuntu.com/ubuntu/ bionic main universe"
+$APTGET update
+
+
 # ---------- UPDATE ----------
 
 if [ `which dialog`x = x ]; then
@@ -90,7 +108,8 @@ apt_get_install_one libboost-system-dev
 apt_get_install_one libtool
 apt_get_install_one pkg-config
 apt_get_install_one psmisc # for killall
-apt_get_install_one python3-venv
+apt_get_install_one elpa-pyvenv # for web2py
+apt_get_install_one python3-venv # for everyone else
 #apt_get_install_one g++-8
 apt_get_install_one ninja-build
 
@@ -167,6 +186,8 @@ fi
 # ---------- PIP ----------
 # Get pip
 if [ `which pip`x = x ]; then
+    apt_get_install --allow-downgrades python-pip-whl=9.0.1-2
+    # this python-pip dependency is very fussy about version
     apt_get_install python-pip
 fi
 
@@ -199,10 +220,10 @@ fi
 # ---------- PYTHON VIRTUALENV ----------
 # Get virtualenv
 if [ `which virtualenv`x = x ]; then
-    apt_get_install python-virtualenv virtualenv
+    # install for both python2 and python3
+    apt_get_install python-virtualenv python3-virtualenv virtualenv
 fi
 
-
 # ---------- MAVEN 3 ----------
 if [ `which mvn`x = x ]; then
     apt_get_install maven
@@ -219,22 +240,23 @@ if [ ! -r /etc/ntp.conf ]; then
     apt_get_install ntp
 fi
 
-
 # ---------- BACKUP EXISTING CONFIG FILES ---------
 
 # Copy all possible Open Tree config files. A given target machine will have
 # some but not all of these, so suppress "file not found" messages.
 CONFIG_BACKUP_DIR="/home/admin/otol-backups.$(date +'%b-%d-%Y-%H%M%S')"
-mkdir -p $CONFIG_BACKUP_DIR
-cp --parents --preserve=all \
+sudo mkdir -p $CONFIG_BACKUP_DIR
+sudo cp --parents --preserve=all \
         /etc/apache2/sites-available/opentree.conf \
         /etc/apache2/sites-available/opentree-ssl.conf \
         /etc/apache2/opentree-shared.conf \
         /home/opentree/repo/opentree/curator/private/config \
         /home/opentree/repo/opentree/webapp/private/config \
         /home/phylesystemapi/repo/phylesystem-api/private/config \
-        $CONFIG_BACKUP_DIR 2>/dev/null
-
+        $CONFIG_BACKUP_DIR 2>/dev/null || true
+# NB the final '|| true' (alternately, '|| :') makes sure we keep on running even if
+# any of the above source files are missing. Else this script fails silently before
+# doing the steps below!
 
 # ---------- APACHE VHOST ----------
 
@@ -287,6 +309,18 @@ else
     sudo rm -f /etc/apache2/sites-enabled/001-opentree-ssl.conf
 fi
 
+# Add global ServerName to main apache config, to silence warnings on restart
+# NB we use 'localhost' here as a reminder that we might have multiple virtual
+# hosts defined for this server.
+APACHE_GLOBAL_CONFIG_FILE=/etc/apache2/apache2.conf
+if grep -q "^ServerName" $APACHE_GLOBAL_CONFIG_FILE; then
+    log "global ServerName found in main apache config"
+else
+    log "adding global ServerName to main apache config"
+    echo "ServerName localhost" | sudo tee -a $APACHE_GLOBAL_CONFIG_FILE > /dev/null
+    log "DONE!"
+fi
+
 # Apache 2.4 is finicky about protection of the key file
 
 
@@ -304,13 +338,20 @@ if [ ! -e $OTHOME ]; then
     sudo chsh -s /bin/bash $OPENTREE_USER
 fi
 
+log "OPENTREE_USER added"
+
 if [ ! -e $OTHOME/.ssh ]; then
     sudo mkdir $OTHOME/.ssh
     sudo cp -p .ssh/authorized_keys $OTHOME/.ssh/
     sudo chmod 700 $OTHOME/.ssh/
     sudo chown -R $OPENTREE_USER:$OPENTREE_USER $OTHOME
 fi
 
+log ".ssh/authorized_keys created"
+
+# Update the canonical hostname on this server
+sudo hostnamectl set-hostname "$OPENTREE_HOST"
+
 # Ideally stowing the hostname one would be done every time, but we
 # want to avoid unsatisfiable sudo prompt demands, so let's assume it
 # stays the same.
@@ -322,4 +363,8 @@ if [ x$OPENTREE_HOST != x -a ! -r $OTHOME/hostname ]; then
 	chmod go+r $HOSTFILE
 	chown $OPENTREE_USER $HOSTFILE
 EOF
+
+    log "hostname updated to $OPENTREE_HOST"
 fi
+
+log "<<< as-admin.sh finished"

deploy/push.sh

@@ -80,10 +80,10 @@ done
 [ "x$OPENTREE_HOST" != x ] || err "OPENTREE_HOST not specified"
 
 # On ubuntu, the admin user is called 'ubuntu'; on debian it's 'admin'
-[ "x$OPENTREE_ADMIN" != x ] || OPENTREE_ADMIN=admin
+[ "x$OPENTREE_ADMIN" != x ] || OPENTREE_ADMIN=ubuntu
 
 # Unprivileged user that runs all the services
-[ "x$OPENTREE_USER" != x ] || OPENTREE_USER=opentree
+[ "x$OPENTREE_USER" != x ] || OPENTREE_USER=deploy
 
 # OPENTREE_SECRETS is the *local* directory where .pem and other
 # private files are kept
@@ -280,9 +280,21 @@ function sync_system {
     if [ $DRYRUN = "yes" ]; then echo "[sync]"; return; fi
     # Do privileged stuff
     # Don't use rsync - might not be installed yet
+
+    ##echo ">>>>>>>>>>>>"
+    ##echo "BEFORE as-admin.sh..."
+    ##echo "  scp -p -i '${ADMIN_IDENTITY}' as-admin.sh '$OPENTREE_ADMIN@$OPENTREE_HOST':"
+    ##echo "  ${ASSH} '$ADMIN@$OPENTREE_HOST' ./as-admin.sh '$OPENTREE_HOST' '$OPENTREE_USER' '$CERTIFICATE_FILE' '$CERTIFICATE_KEY_FILE'"
+    ## echo ">>>>>>>>>>>>"
+
     scp -p -i "${ADMIN_IDENTITY}" as-admin.sh "$OPENTREE_ADMIN@$OPENTREE_HOST":
     ${ASSH} "$ADMIN@$OPENTREE_HOST" ./as-admin.sh "$OPENTREE_HOST" "$OPENTREE_USER" \
        "$CERTIFICATE_FILE" "$CERTIFICATE_KEY_FILE"
+
+    ##echo "<<<<<<<<<<<<"
+    ##echo "AFTER as-admin.sh!"
+    ##echo "<<<<<<<<<<<<"
+
     # Copy files over
     rsync -pr -e "${SSH}" "--exclude=*~" "--exclude=#*" setup "$OT_USER@$OPENTREE_HOST":
     # Bleh

deploy/setup/functions.sh

@@ -22,7 +22,9 @@ fi
 # ---------- Setup TWO VIRTUALENVs ----------
 # Set up python env
 if [ ! -d venv ]; then
-    virtualenv venv || exit 1
+    # specify python2.* or you'll now get python3
+    virtualenv --python=`which python2` venv || exit 1
+    # NB - later we'll probably apply our single-file patch to `activate_this.py`
 fi
 source venv/bin/activate || exit 1
 

deploy/setup/install-web2py.sh

@@ -38,6 +38,10 @@ cp -p setup/web2py-patches/oauth20_account.py web2py/gluon/contrib/login_methods
 
 # ---------- VIRTUALENV + WEB2PY + WSGI ----------
 
+# Patch broken virtualenv activation helper (mid-June 2022), used below
+echo "Applying patch (hot fix) to buggy virtualenv..."  || exit
+cp -p setup/virtualenv-patches/activate_this.py "$PWD/venv/bin/" || exit 1
+
 # Patch web2py's wsgihandler so that it does the equivalent of 'venv/activate'
 # when started by Apache.
 

deploy/setup/opentree-shared.conf

@@ -26,7 +26,7 @@
 
     Alias "/taxonomy/browse" "/home/opentree/repo/opentree/taxonomy/cgi-bin/browse.py"
 
-    <Directory "/home/opentree/repo/opentree/taxonomy/cgi-bin">
+    <Directory "/home/deploy/repo/opentree/taxonomy/cgi-bin">
       Require all granted
       Options +ExecCGI
       AddHandler cgi-script .py

deploy/setup/opentree-ssl.conf

@@ -43,23 +43,23 @@
     # Moving all WSGI configuration here, so we only run one instance of web2py on HTTPS.
     # This should prevent our broken session problem as documented here:
     #   https://github.com/OpenTreeOfLife/opentree/issues/804
-    WSGIDaemonProcess web2py user=opentree group=opentree display-name=%{GROUP}
+    WSGIDaemonProcess web2py user=deploy group=deploy display-name=%{GROUP}
     WSGIProcessGroup web2py
-    WSGIScriptAlias / /home/opentree/web2py/wsgihandler.py
+    WSGIScriptAlias / /home/deploy/web2py/wsgihandler.py
 
     # We don't want the scriptalias to override mappings for
     #  treemachine and taxomachine... seems not to.
 
-    <Directory /home/opentree/web2py>
+    <Directory /home/deploy/web2py>
       AllowOverride None
       Require all denied
       <Files wsgihandler.py>
         Require all granted
       </Files>
     </Directory>
 
-    AliasMatch ^/([^/]+)/static/(.*)            /home/opentree/web2py/applications/$1/static/$2
-    <Directory /home/opentree/web2py/applications/*/static/>
+    AliasMatch ^/([^/]+)/static/(.*)            /home/deploy/web2py/applications/$1/static/$2
+    <Directory /home/deploy/applications/*/static/>
       Require all granted
     </Directory>
 

deploy/setup/virtualenv-patches/README.md

@@ -0,0 +1,3 @@
+Once again, we want some fixes that aren't easily available in newer Ubuntu
+(mid-2022). In this case we want to fix bugs in `activate_this.py`, a helper
+that lets us use the python packages in a venv from Apache's mod_wsgi.

deploy/setup/virtualenv-patches/activate_this.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""Activate virtualenv for current interpreter:
+
+Use exec(open(this_file).read(), {'__file__': this_file}).
+
+This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
+"""
+import os
+import site
+import sys
+
+try:
+    abs_file = os.path.abspath(__file__)
+except NameError:
+    raise AssertionError("You must use exec(open(this_file).read(), {'__file__': this_file}))")
+
+bin_dir = os.path.dirname(abs_file)
+base = bin_dir[: -len("bin") - 1]  # strip away the bin part from the __file__, plus the path separator
+
+# prepend bin to PATH (this file is inside the bin directory)
+os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep))
+os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
+
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in "../lib/python2.7/site-packages".split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path.decode("utf-8") if "" else path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
+
+sys.real_prefix = sys.prefix
+sys.prefix = base

deploy/setup/web2py-patches/oauth20_account.py

@@ -83,7 +83,7 @@ def get_user(self):
                user = None
                try:
                    user = self.graph.get_object("me")
-               except GraphAPIError, e:
+               except GraphAPIError as e:
                    self.session.token = None
                    self.graph = None
 
@@ -226,7 +226,7 @@ def accessToken(self):
                     opener.close()
                 try:
                     return current.session.token['access_token']
-                except Exception, e:
+                except Exception as e:
                     raise Exception("No access_token found in data: %s %s" % (current.session.token, e))
                     return None
 

doc/ot-synthesis-v10.2.md

@@ -3,10 +3,10 @@
 Version 10.2 of the synthetic tree was generated on 9 January 2018 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity).
 
 ## Downloads
-There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html).
+There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html).
 
-* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
-* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
+* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
+* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
 
 ## Release notes
 

doc/ot-synthesis-v10.3.md

@@ -5,8 +5,8 @@ Version 10.3 of the synthetic tree was generated on 10 July 2018 using the [prop
 ## Downloads
 There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html).
 
-* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
-* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
+* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
+* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
 
 ## Release notes
 

doc/ot-synthesis-v10.4.md

@@ -3,10 +3,10 @@
 Version 10.4 of the synthetic tree was generated on 24 September 2018 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity).
 
 ## Downloads
-There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html).
+There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html).
 
-* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
-* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
+* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes)
+* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes)
 
 ## Release notes
 

doc/ot-synthesis-v11.4.md

@@ -3,10 +3,10 @@
 Version 11.4 of the synthetic tree was generated on 09 October 2019 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity).
 
 ## Downloads
-There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html).
+There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html).
 
-* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes)
-* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes)
+* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes)
+* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes)
 
 ## Release notes
 
@@ -60,4 +60,4 @@ There are two downloads. The first (smaller download) contain only tree and anno
    <td>9860</td>
    <td>2090</td>
 </tr>
-</table>
+</table>