Mercurial > hg > soundsoftware-site
changeset 1590:c18460da6620 dockerise
Numerous deployment updates
line wrap: on
line diff
--- a/deploy/config/code.conf.in Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/code.conf.in Thu Aug 17 16:04:36 2017 +0100 @@ -18,6 +18,20 @@ RailsSpawnMethod smart ExpiresDefault "access plus 1 minute" +# <Location /sys> +# AuthType Basic +# AuthUserFile "/etc/apache2/auth/user.htpasswd" +# AuthName "code.soundsoftware.ac.uk" +# Require user user +# </Location> + +# <Location /admin> +# AuthType Digest +# AuthUserFile "/etc/apache2/auth/admin.htdigest" +# AuthName "code.soundsoftware.ac.uk admin interface" +# Require user admin +# </Location> + <DirectoryMatch "^/.*/\.svn/"> Order allow,deny Deny from all
--- a/deploy/config/cron.daily/00-backup-db Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.daily/00-backup-db Thu Aug 17 16:04:36 2017 +0100 @@ -1,6 +1,10 @@ #!/bin/sh + outfile="/var/files/backups/postgres-dumpall-`date +%Y%m%d%H%M`" + oldmask=`umask` umask 0277 + su postgres -c /usr/bin/pg_dumpall > "$outfile" && bzip2 "$outfile" + umask "$oldmask"
--- a/deploy/config/cron.daily/10-extract-docs Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.daily/10-extract-docs Thu Aug 17 16:04:36 2017 +0100 @@ -1,3 +1,6 @@ #!/bin/bash + cd /tmp + /var/www/code/docgen/extract-docs.sh +
--- a/deploy/config/cron.daily/15-get-statistics Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.daily/15-get-statistics Thu Aug 17 16:04:36 2017 +0100 @@ -1,2 +1,4 @@ #!/bin/bash + sudo -u code sh -c "cd /var/www/code ; ./script/rails runner -e production extra/soundsoftware/get-statistics.rb >> log/statistics.log" +
--- a/deploy/config/cron.daily/20-check-end-of-external-repo-log Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.daily/20-check-end-of-external-repo-log Thu Aug 17 16:04:36 2017 +0100 @@ -1,2 +1,3 @@ #!/bin/bash + tail -2 /var/log/external-repos.log
--- a/deploy/config/cron.hourly/00-drupal-cron Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.hourly/00-drupal-cron Thu Aug 17 16:04:36 2017 +0100 @@ -1,3 +1,6 @@ #!/bin/bash -/usr/bin/wget -O - -q -t 1 http://www.soundsoftware.ac.uk/cron.php +## No longer used - this site is now static + +# /usr/bin/wget -O - -q -t 1 http://www.soundsoftware.ac.uk/cron.php +
--- a/deploy/config/cron.hourly/10-redmine-fetch-changesets Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.hourly/10-redmine-fetch-changesets Thu Aug 17 16:04:36 2017 +0100 @@ -1,3 +1,5 @@ #!/bin/bash + sudo -u code sh -c "cd /var/www/code ; ./script/rails runner \"Repository.fetch_changesets\" -e production 2>&1 | grep -v 'Not trusting' | grep -v 'svn:' | grep -v 'working copy' | grep -v 'deprecated' | grep -v 'version_requirements'" exit 0 +
--- a/deploy/config/cron.hourly/20-convert-external-repos Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.hourly/20-convert-external-repos Thu Aug 17 16:04:36 2017 +0100 @@ -1,2 +1,4 @@ #!/bin/bash + sudo -H -u www-data /var/www/code/reposman/run-external.sh +
--- a/deploy/config/cron.hourly/30-expire-explore-cache Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.hourly/30-expire-explore-cache Thu Aug 17 16:04:36 2017 +0100 @@ -1,2 +1,3 @@ #!/bin/sh + rm -f /var/www/code/tmp/cache/*/*/views*explore*
--- a/deploy/config/cron.hourly/40-export-git Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.hourly/40-export-git Thu Aug 17 16:04:36 2017 +0100 @@ -1,3 +1,6 @@ #!/bin/bash + logfile="/var/www/code/log/export-git.log" + sudo -u code sh -c "cd /tmp ; /var/www/code/extra/soundsoftware/export-git.sh production /var/hg /var/files/git-mirror >> $logfile 2>&1" +
--- a/deploy/config/cron.minutely/00-redmine-repositories Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/cron.minutely/00-redmine-repositories Thu Aug 17 16:04:36 2017 +0100 @@ -1,3 +1,4 @@ #!/bin/bash + sudo -u www-data /var/www/code/reposman/run-reposman.sh
--- a/deploy/config/logrotate.conf Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/config/logrotate.conf Thu Aug 17 16:04:36 2017 +0100 @@ -2,6 +2,10 @@ # rotate log files weekly weekly +# use the syslog group by default, since this is the owning group +# of /var/log/syslog. +su root syslog + # keep 4 weeks worth of backlogs rotate 4
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/run-external.sh.in Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,18 @@ +#!/bin/bash +logfile=/var/log/external-repos.log +( +flock -s 200 +echo >> $logfile +date >> $logfile +/var/www/code/reposman/convert-external-repos.rb \ + -s /var/hg \ + -r https://code.soundsoftware.ac.uk/ \ + -k INSERT_API_KEY_HERE \ + -v \ + --http-user=user \ + --http-pass=password \ + -c "/var/www/code/reposman/update-external-repo.sh" \ + >> $logfile 2>&1 +date >> $logfile +echo Done >> $logfile +)200>>$logfile
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/run-reposman.sh.in Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,17 @@ +#!/bin/bash +logfile=/var/log/reposman.log +( +flock -s 200 +echo >> $logfile +/var/www/code/reposman/reposman-soundsoftware.rb \ + -s /var/hg \ + -r https://code.soundsoftware.ac.uk/ \ + -k INSERT_API_KEY_HERE \ + --http-user=user \ + --http-pass=password \ + -o www-data \ + -g code \ + -c "/var/www/code/reposman/run-hginit.sh" \ + --scm=Mercurial \ + >> $logfile 2>&1 +)200>>$logfile
--- a/deploy/docker/start.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/docker/start.sh Thu Aug 17 16:04:36 2017 +0100 @@ -48,7 +48,7 @@ done provisioning_commands=$( - for x in "$deploydir"/provision.d/[0-9]*; do + for x in "$deploydir"/provision.d/[0-9]*.sh; do echo "RUN /bin/bash /var/www/code/deploy/provision.d/$(basename $x)" done | sed 's/$/\\n/' | fmt -2000 | sed 's/ RUN/RUN/g' )
--- a/deploy/provision.d/000-system-packages.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/provision.d/000-system-packages.sh Thu Aug 17 16:04:36 2017 +0100 @@ -43,7 +43,6 @@ ruby-dev \ sudo -apt-get clean && rm -rf /var/lib/apt/lists/* locale-gen en_US.UTF-8
--- a/deploy/provision.d/020-users.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/provision.d/020-users.sh Thu Aug 17 16:04:36 2017 +0100 @@ -2,11 +2,14 @@ set -e -# The "code" user (in group www-data) owns the site and repo -# directories. +# The webapp directory is owned and run by the code user, in group +# www-data. The repos and other things served directly are the other +# way around -- owned by the www-data user, in group code. -if ! grep -q '^code:' /etc/passwd ; then - groupadd code - useradd -g code -G www-data code -fi +for user in code docgen ; do + if ! grep -q "^$user:" /etc/passwd ; then + groupadd "$user" + useradd -g "$user" -G www-data "$user" + fi +done
--- a/deploy/provision.d/040-hg-dir.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/provision.d/040-hg-dir.sh Thu Aug 17 16:04:36 2017 +0100 @@ -15,6 +15,6 @@ chmod +x /var/hg/index.cgi -chown -R code.www-data /var/hg +chown -R www-data.code /var/hg find /var/hg -type d -exec chmod g+s \{\} \;
--- a/deploy/provision.d/110-hg-testdir.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/provision.d/110-hg-testdir.sh Thu Aug 17 16:04:36 2017 +0100 @@ -10,5 +10,5 @@ echo "Cloning vamp-plugin-sdk repo for testing..." cd /var/hg hg clone https://code.soundsoftware.ac.uk/hg/vamp-plugin-sdk - chown -R code.www-data vamp-plugin-sdk + chown -R www-data.code vamp-plugin-sdk fi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/120-docgen.sh Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +# Copy docgen scripts to the place they actually live. This is +# particularly badly managed, since the target location is actually +# within the repo already + +cd /var/www/code + +mkdir -p docgen + +for file in \ + doxysafe.pl \ + extract-docs.sh \ + extract-doxygen.sh \ + extract-javadoc.sh \ + extract-matlabdocs.sh \ + matlab-docs.conf \ + matlab-docs-credit.html \ + matlab-docs.pl ; do + if [ ! -f docgen/"$file" ]; then + cp extra/soundsoftware/"$file" docgen/ + fi +done + +chown code.www-data docgen/* +chmod +x docgen/*.sh +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/130-reposman.sh Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +# Copy reposman scripts to the place they actually live. Like docgen, +# this is particularly badly managed, since the target location is +# actually within the repo already. At least in this case some of the +# scripts have to be edited to insert the server's API key, so there +# is a bit of logic there + +cd /var/www/code + +mkdir -p reposman + +for file in \ + convert-external-repos.rb \ + reposman-soundsoftware.rb \ + run-hginit.sh \ + update-external-repo.sh ; do + if [ ! -f reposman/"$file" ]; then + cp extra/soundsoftware/"$file" reposman/ + fi +done + +for file in \ + run-external.sh \ + run-reposman.sh ; do + if [ ! -f reposman/"$file" ]; then + ##!!! TODO: actually insert API key + cat deploy/config/"$file".in > reposman/"$file" + fi +done + +chown code.www-data reposman/* +chmod +x reposman/*.sh +chmod +x reposman/*.rb + +touch /var/log/reposman.log +touch /var/log/external-repos.log +chown www-data.code /var/log/reposman.log +chown www-data.code /var/log/external-repos.log +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/140-cron.sh Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Initialise directories used as targets for cron activity (if they +# don't already exist) + +for dir in \ + /var/files/backups \ + /var/doc \ + /var/files/git-mirror ; do + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + chown -R code.www-data "$dir" + chmod g+s "$dir" + fi +done + +# Copy cron scripts to the appropriate destinations + +cd /var/www/code + +if [ ! -d /etc/cron.minutely ]; then + mkdir -p /etc/cron.minutely + echo '* * * * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.minutely )' >> /etc/crontab +fi + +for t in minutely hourly daily monthly; do + for s in deploy/config/cron.$t/[0-9]* ; do + name=$(basename $s) + dest="/etc/cron.$t/$name" + if [ ! -f "$dest" ]; then + cp "$s" "$dest" + chmod +x "$dest" + fi + done +done + + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/150-logrotate.sh Thu Aug 17 16:04:36 2017 +0100 @@ -0,0 +1,4 @@ +#!/bin/bash + +cd /var/www/code +cp deploy/config/logrotate.conf /etc/logrotate.conf
--- a/deploy/vagrant/vagrant-provision.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/deploy/vagrant/vagrant-provision.sh Thu Aug 17 16:04:36 2017 +0100 @@ -1,16 +1,13 @@ #!/bin/bash #!!! still not covered: -# * cron jobs # * https -# * web fonts -# * reposman scripts (and their API key setup, etc) -# * docgen script install -# * logrotate config (check against system one) +# * http auth for API (/sys) and /admin interfaces +# * API keys and http auth for reposman and docgen set -e -for f in /code-to-deploy/deploy/provision.d/[0-9]* ; do +for f in /code-to-deploy/deploy/provision.d/[0-9]*.sh ; do case "$f" in *~) ;; *) echo "Running provision script: $f"
--- a/extra/soundsoftware/extract-docs.sh Thu Aug 17 13:56:15 2017 +0100 +++ b/extra/soundsoftware/extract-docs.sh Thu Aug 17 16:04:36 2017 +0100 @@ -7,9 +7,9 @@ hgdir="/var/hg" docdir="/var/doc" -logfile="/var/www/test-cannam/log/extract-docs.log" +logfile="/var/www/code/log/extract-docs.log" -redgrp="redmine" +redgrp="code" apikey="" apischeme="https"