# HG changeset patch
# User Chris Cannam
# Date 1502982276 -3600
# Node ID c18460da6620ad565f79b8868cd98fe36b865f4f
# Parent  94669513c53ce7e7ad4252790b691d2e84178e39
Numerous deployment updates

diff -r 94669513c53c -r c18460da6620 deploy/config/code.conf.in
--- a/deploy/config/code.conf.in	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/code.conf.in	Thu Aug 17 16:04:36 2017 +0100
@@ -18,6 +18,20 @@
         RailsSpawnMethod smart
         ExpiresDefault "access plus 1 minute"
 
+#        <Location /sys>
+#		AuthType Basic
+#		AuthUserFile "/etc/apache2/auth/user.htpasswd"
+#		AuthName "code.soundsoftware.ac.uk"
+#		Require user user
+#	</Location>
+
+#	<Location /admin>
+#		AuthType Digest
+#		AuthUserFile "/etc/apache2/auth/admin.htdigest"
+#		AuthName "code.soundsoftware.ac.uk admin interface"
+#		Require user admin
+#	</Location>
+
         <DirectoryMatch "^/.*/\.svn/">
                 Order allow,deny
                 Deny from all
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.daily/00-backup-db
--- a/deploy/config/cron.daily/00-backup-db	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.daily/00-backup-db	Thu Aug 17 16:04:36 2017 +0100
@@ -1,6 +1,10 @@
 #!/bin/sh
+
 outfile="/var/files/backups/postgres-dumpall-`date +%Y%m%d%H%M`"
+
 oldmask=`umask`
 umask 0277
+
 su postgres -c /usr/bin/pg_dumpall > "$outfile" && bzip2 "$outfile"
+
 umask "$oldmask"
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.daily/10-extract-docs
--- a/deploy/config/cron.daily/10-extract-docs	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.daily/10-extract-docs	Thu Aug 17 16:04:36 2017 +0100
@@ -1,3 +1,6 @@
 #!/bin/bash
+
 cd /tmp
+
 /var/www/code/docgen/extract-docs.sh
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.daily/15-get-statistics
--- a/deploy/config/cron.daily/15-get-statistics	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.daily/15-get-statistics	Thu Aug 17 16:04:36 2017 +0100
@@ -1,2 +1,4 @@
 #!/bin/bash
+
 sudo -u code sh -c "cd /var/www/code ; ./script/rails runner -e production extra/soundsoftware/get-statistics.rb >> log/statistics.log"
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.daily/20-check-end-of-external-repo-log
--- a/deploy/config/cron.daily/20-check-end-of-external-repo-log	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.daily/20-check-end-of-external-repo-log	Thu Aug 17 16:04:36 2017 +0100
@@ -1,2 +1,3 @@
 #!/bin/bash
+
 tail -2 /var/log/external-repos.log
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.hourly/00-drupal-cron
--- a/deploy/config/cron.hourly/00-drupal-cron	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.hourly/00-drupal-cron	Thu Aug 17 16:04:36 2017 +0100
@@ -1,3 +1,6 @@
 #!/bin/bash
-/usr/bin/wget -O - -q -t 1 http://www.soundsoftware.ac.uk/cron.php
 
+## No longer used - this site is now static
+
+# /usr/bin/wget -O - -q -t 1 http://www.soundsoftware.ac.uk/cron.php
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.hourly/10-redmine-fetch-changesets
--- a/deploy/config/cron.hourly/10-redmine-fetch-changesets	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.hourly/10-redmine-fetch-changesets	Thu Aug 17 16:04:36 2017 +0100
@@ -1,3 +1,5 @@
 #!/bin/bash
+
 sudo -u code sh -c "cd /var/www/code ; ./script/rails runner \"Repository.fetch_changesets\" -e production 2>&1 | grep -v 'Not trusting' | grep -v 'svn:' | grep -v 'working copy' | grep -v 'deprecated' | grep -v 'version_requirements'"
 exit 0
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.hourly/20-convert-external-repos
--- a/deploy/config/cron.hourly/20-convert-external-repos	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.hourly/20-convert-external-repos	Thu Aug 17 16:04:36 2017 +0100
@@ -1,2 +1,4 @@
 #!/bin/bash
+
 sudo -H -u www-data /var/www/code/reposman/run-external.sh
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.hourly/30-expire-explore-cache
--- a/deploy/config/cron.hourly/30-expire-explore-cache	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.hourly/30-expire-explore-cache	Thu Aug 17 16:04:36 2017 +0100
@@ -1,2 +1,3 @@
 #!/bin/sh
+
 rm -f /var/www/code/tmp/cache/*/*/views*explore*
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.hourly/40-export-git
--- a/deploy/config/cron.hourly/40-export-git	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.hourly/40-export-git	Thu Aug 17 16:04:36 2017 +0100
@@ -1,3 +1,6 @@
 #!/bin/bash
+
 logfile="/var/www/code/log/export-git.log"
+
 sudo -u code sh -c "cd /tmp ; /var/www/code/extra/soundsoftware/export-git.sh production /var/hg /var/files/git-mirror >> $logfile 2>&1"
+
diff -r 94669513c53c -r c18460da6620 deploy/config/cron.minutely/00-redmine-repositories
--- a/deploy/config/cron.minutely/00-redmine-repositories	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/cron.minutely/00-redmine-repositories	Thu Aug 17 16:04:36 2017 +0100
@@ -1,3 +1,4 @@
 #!/bin/bash
+
 sudo -u www-data /var/www/code/reposman/run-reposman.sh
 
diff -r 94669513c53c -r c18460da6620 deploy/config/logrotate.conf
--- a/deploy/config/logrotate.conf	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/config/logrotate.conf	Thu Aug 17 16:04:36 2017 +0100
@@ -2,6 +2,10 @@
 # rotate log files weekly
 weekly
 
+# use the syslog group by default, since this is the owning group
+# of /var/log/syslog.
+su root syslog
+
 # keep 4 weeks worth of backlogs
 rotate 4
 
diff -r 94669513c53c -r c18460da6620 deploy/config/run-external.sh.in
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/config/run-external.sh.in	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,18 @@
+#!/bin/bash
+logfile=/var/log/external-repos.log
+( 
+flock -s 200 
+echo >> $logfile
+date >> $logfile
+/var/www/code/reposman/convert-external-repos.rb \
+	-s /var/hg \
+	-r https://code.soundsoftware.ac.uk/ \
+	-k INSERT_API_KEY_HERE \
+	-v \
+	--http-user=user \
+	--http-pass=password \
+        -c "/var/www/code/reposman/update-external-repo.sh" \
+	>> $logfile 2>&1
+date >> $logfile
+echo Done >> $logfile
+)200>>$logfile
diff -r 94669513c53c -r c18460da6620 deploy/config/run-reposman.sh.in
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/config/run-reposman.sh.in	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,17 @@
+#!/bin/bash
+logfile=/var/log/reposman.log
+( 
+flock -s 200 
+echo >> $logfile
+/var/www/code/reposman/reposman-soundsoftware.rb \
+	-s /var/hg \
+	-r https://code.soundsoftware.ac.uk/ \
+	-k INSERT_API_KEY_HERE \
+	--http-user=user \
+	--http-pass=password \
+	-o www-data \
+	-g code \
+	-c "/var/www/code/reposman/run-hginit.sh" \
+	--scm=Mercurial \
+	>> $logfile 2>&1
+)200>>$logfile
diff -r 94669513c53c -r c18460da6620 deploy/docker/start.sh
--- a/deploy/docker/start.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/docker/start.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -48,7 +48,7 @@
 done
 
 provisioning_commands=$(
-    for x in "$deploydir"/provision.d/[0-9]*; do
+    for x in "$deploydir"/provision.d/[0-9]*.sh; do
         echo "RUN /bin/bash /var/www/code/deploy/provision.d/$(basename $x)"
     done | sed 's/$/\\n/' | fmt -2000 | sed 's/ RUN/RUN/g' )
 
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/000-system-packages.sh
--- a/deploy/provision.d/000-system-packages.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/provision.d/000-system-packages.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -43,7 +43,6 @@
             ruby-dev \
             sudo
 
-apt-get clean && rm -rf /var/lib/apt/lists/*
 locale-gen en_US.UTF-8
 
 
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/020-users.sh
--- a/deploy/provision.d/020-users.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/provision.d/020-users.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -2,11 +2,14 @@
 
 set -e
 
-# The "code" user (in group www-data) owns the site and repo
-# directories.
+# The webapp directory is owned and run by the code user, in group
+# www-data. The repos and other things served directly are the other
+# way around -- owned by the www-data user, in group code.
 
-if ! grep -q '^code:' /etc/passwd ; then
-    groupadd code
-    useradd -g code -G www-data code
-fi
+for user in code docgen ; do
+    if ! grep -q "^$user:" /etc/passwd ; then
+        groupadd "$user"
+        useradd -g "$user" -G www-data "$user"
+    fi
+done
 
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/040-hg-dir.sh
--- a/deploy/provision.d/040-hg-dir.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/provision.d/040-hg-dir.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -15,6 +15,6 @@
 
 chmod +x /var/hg/index.cgi
 
-chown -R code.www-data /var/hg
+chown -R www-data.code /var/hg
 find /var/hg -type d -exec chmod g+s \{\} \;
 
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/110-hg-testdir.sh
--- a/deploy/provision.d/110-hg-testdir.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/provision.d/110-hg-testdir.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -10,5 +10,5 @@
     echo "Cloning vamp-plugin-sdk repo for testing..."
     cd /var/hg
     hg clone https://code.soundsoftware.ac.uk/hg/vamp-plugin-sdk
-    chown -R code.www-data vamp-plugin-sdk
+    chown -R www-data.code vamp-plugin-sdk
 fi
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/120-docgen.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/provision.d/120-docgen.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+# Copy docgen scripts to the place they actually live. This is
+# particularly badly managed, since the target location is actually
+# within the repo already
+
+cd /var/www/code
+
+mkdir -p docgen
+
+for file in \
+    doxysafe.pl \
+    extract-docs.sh \
+    extract-doxygen.sh \
+    extract-javadoc.sh \
+    extract-matlabdocs.sh \
+    matlab-docs.conf \
+    matlab-docs-credit.html \
+    matlab-docs.pl ; do
+    if [ ! -f docgen/"$file" ]; then
+        cp extra/soundsoftware/"$file" docgen/
+    fi
+done
+
+chown code.www-data docgen/*
+chmod +x docgen/*.sh
+
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/130-reposman.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/provision.d/130-reposman.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+# Copy reposman scripts to the place they actually live. Like docgen,
+# this is particularly badly managed, since the target location is
+# actually within the repo already. At least in this case some of the
+# scripts have to be edited to insert the server's API key, so there
+# is a bit of logic there
+
+cd /var/www/code
+
+mkdir -p reposman
+
+for file in \
+    convert-external-repos.rb \
+    reposman-soundsoftware.rb \
+    run-hginit.sh \
+    update-external-repo.sh ; do
+    if [ ! -f reposman/"$file" ]; then
+        cp extra/soundsoftware/"$file" reposman/
+    fi
+done
+
+for file in \
+    run-external.sh \
+    run-reposman.sh ; do
+    if [ ! -f reposman/"$file" ]; then
+        ##!!! TODO: actually insert API key
+        cat deploy/config/"$file".in > reposman/"$file"
+    fi
+done
+
+chown code.www-data reposman/*
+chmod +x reposman/*.sh
+chmod +x reposman/*.rb
+
+touch /var/log/reposman.log
+touch /var/log/external-repos.log
+chown www-data.code /var/log/reposman.log
+chown www-data.code /var/log/external-repos.log
+
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/140-cron.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/provision.d/140-cron.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+# Initialise directories used as targets for cron activity (if they
+# don't already exist)
+
+for dir in \
+    /var/files/backups \
+    /var/doc \
+    /var/files/git-mirror ; do
+    if [ ! -d "$dir" ]; then
+        mkdir -p "$dir"
+        chown -R code.www-data "$dir"
+        chmod g+s "$dir"
+    fi
+done
+
+# Copy cron scripts to the appropriate destinations
+
+cd /var/www/code
+
+if [ ! -d /etc/cron.minutely ]; then
+    mkdir -p /etc/cron.minutely
+    echo '*  *    * * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.minutely )' >> /etc/crontab
+fi
+
+for t in minutely hourly daily monthly; do
+    for s in deploy/config/cron.$t/[0-9]* ; do
+        name=$(basename $s)
+        dest="/etc/cron.$t/$name"
+        if [ ! -f "$dest" ]; then
+            cp "$s" "$dest"
+            chmod +x "$dest"
+        fi
+    done
+done
+
+
+             
diff -r 94669513c53c -r c18460da6620 deploy/provision.d/150-logrotate.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/provision.d/150-logrotate.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /var/www/code
+cp deploy/config/logrotate.conf /etc/logrotate.conf
diff -r 94669513c53c -r c18460da6620 deploy/vagrant/vagrant-provision.sh
--- a/deploy/vagrant/vagrant-provision.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/deploy/vagrant/vagrant-provision.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -1,16 +1,13 @@
 #!/bin/bash
 
 #!!! still not covered:
-# * cron jobs
 # * https
-# * web fonts
-# * reposman scripts (and their API key setup, etc)
-# * docgen script install
-# * logrotate config (check against system one)
+# * http auth for API (/sys) and /admin interfaces
+# * API keys and http auth for reposman and docgen
 
 set -e
 
-for f in /code-to-deploy/deploy/provision.d/[0-9]* ; do
+for f in /code-to-deploy/deploy/provision.d/[0-9]*.sh ; do
     case "$f" in
         *~) ;;
         *) echo "Running provision script: $f"
diff -r 94669513c53c -r c18460da6620 extra/soundsoftware/extract-docs.sh
--- a/extra/soundsoftware/extract-docs.sh	Thu Aug 17 13:56:15 2017 +0100
+++ b/extra/soundsoftware/extract-docs.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -7,9 +7,9 @@
 
 hgdir="/var/hg"
 docdir="/var/doc"
-logfile="/var/www/test-cannam/log/extract-docs.log"
+logfile="/var/www/code/log/extract-docs.log"
 
-redgrp="redmine"
+redgrp="code"
 
 apikey=""
 apischeme="https"
diff -r 94669513c53c -r c18460da6620 extra/soundsoftware/run-hginit.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/extra/soundsoftware/run-hginit.sh	Thu Aug 17 16:04:36 2017 +0100
@@ -0,0 +1,3 @@
+#!/bin/sh
+location="$1"
+hg init "$location" && mkdir "$location/.hg/store/data"