Mercurial > hg > soundsoftware-site

changeset 1593:83412a0a2389 dockerise

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Handle API keys etc, + tidying handling of generated files
author Chris Cannam
date Fri, 18 Aug 2017 15:02:20 +0100
parents 72d9219f2f19
children 69aee698921b
files .hgignore deploy/any/prepare.sh deploy/config/code.conf.in deploy/config/database.yml.in deploy/config/run-external.sh.in deploy/config/run-reposman.sh.in deploy/docker/start.sh deploy/provision.d/050-webapp-db.sh deploy/provision.d/100-apache-config.sh deploy/provision.d/120-docgen.sh deploy/provision.d/130-reposman.sh deploy/vagrant/start.sh deploy/vagrant/vagrant-provision.sh extra/soundsoftware/extract-docs.sh
diffstat 14 files changed, 118 insertions(+), 132 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Fri Aug 18 14:46:06 2017 +0100
+++ b/.hgignore	Fri Aug 18 15:02:20 2017 +0100
@@ -42,6 +42,4 @@
 *.pyc
 *-console.log
 postgres-dumpall
-deploy/config/code.conf
-deploy/config/database.yml
-deploy/docker/Dockerfile
+*.gen
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/deploy/any/prepare.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# To be sourced into a container-specific start.sh file, not run
+# standalone
+
+usage() {
+    echo "Usage: $0 <database-password> <api-key> <api-httpauth-password>" 1>&2
+    exit 2
+}
+
+dbpass="$1"
+if [ -z "$dbpass" ]; then
+    usage
+fi
+
+apikey="$2"
+if [ -z "$apikey" ]; then
+    usage
+fi
+
+apipass="$3"
+if [ -z "$apipass" ]; then
+    usage
+fi
+
+set -eu -o pipefail
+
+rootdir="$mydir/../.."
+
+deploydir="$rootdir"/deploy
+if [ ! -d "$deploydir" ]; then
+    echo "ERROR: Unexpected repository layout - expected directory at $deploydir"
+    exit 2
+fi
+
+managerdir="$deploydir/vagrant"
+if [ ! -d "$managerdir" ]; then
+    echo "ERROR: Required directory $managerdir not found"
+    exit 2
+fi
+
+configdir="$deploydir/config"
+if [ ! -d "$configdir" ]; then
+    echo "ERROR: Required directory $configdir not found"
+    exit 2
+fi
+
+if [ ! -f "$rootdir/postgres-dumpall" ]; then
+    echo "ERROR: I expect to find a Postgres SQL multi-db dump file in $rootdir/postgres-dumpall"
+    exit 2
+fi
+
+fontdir="$rootdir"/public/themes/soundsoftware/stylesheets/fonts
+if [ ! -f "$fontdir/24BC0E_0_0.woff" ]; then
+    echo "ERROR: I expect to find necessary webfonts in $fontdir"
+    exit 2
+fi
+
+apischeme=http
+apihost=localhost
+
+#apischeme=https
+#apihost=code.soundsoftware.ac.uk
+
+for f in "$configdir"/*.in "$rootdir"/extra/soundsoftware/extract-docs.sh ; do
+    out="$configdir"/$(basename "$f" .in).gen
+    cat "$f" | sed \
+                   -e 's/INSERT_DATABASE_PASSWORD_HERE/'"$dbpass"'/g' \
+                   -e 's/INSERT_API_KEY_HERE/'"$apikey"'/g' \
+                   -e 's/INSERT_API_SCHEME_HERE/'"$apischeme"'/g' \
+                   -e 's/INSERT_API_HOST_HERE/'"$apihost"'/g' \
+                   -e 's/INSERT_API_USER_HERE/user/g' \
+                   -e 's/INSERT_API_PASSWORD_HERE/'"$apipass"'/g' \
+                   > "$out"
+done
--- a/deploy/config/code.conf.in	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/config/code.conf.in	Fri Aug 18 15:02:20 2017 +0100
@@ -2,6 +2,9 @@
 # A test Apache config. Lacks SSL, lacks a desirable extra layer of
 # authentication for admin interface paths. Do not deploy this.
 
+# Note this has been updated for Apache 2.4, which introduced a number
+# of (welcome) changes to access control directives.
+
 PerlLoadModule Apache::Authn::SoundSoftware
 
 <VirtualHost *:80>
@@ -33,21 +36,15 @@
 #	</Location>
 
         <DirectoryMatch "^/.*/\.svn/">
-                Order allow,deny
-                Deny from all
-                Satisfy All
+                Require all denied
         </DirectoryMatch>
 
         <DirectoryMatch "^/.*/\.hg/">
-                Order allow,deny
-                Deny from all
-                Satisfy All
+                Require all denied
         </DirectoryMatch>
 
         <DirectoryMatch "^/.*/\.git/">
-                Order allow,deny
-                Deny from all
-                Satisfy All
+                Require all denied
         </DirectoryMatch>
 
         <Directory /var/www/code/public>
@@ -73,7 +70,7 @@
 		PerlSetVar HTTPS "on"
 		SoundSoftwareDSN "dbi:Pg:database=code;host=localhost"
     		SoundSoftwareDbUser "code"
-     		SoundSoftwareDbPass "INSERT_POSTGRES_PASSWORD_HERE"
+     		SoundSoftwareDbPass "INSERT_DATABASE_PASSWORD_HERE"
 		SoundSoftwareRepoPrefix "/var/hg/"
                 #!!! "on" in production please!:
                 SoundSoftwareSslRequired "off"
@@ -86,16 +83,13 @@
 
 	<Directory "/var/files/git-mirror">
 		Options -Indexes +FollowSymLinks
-                Order allow,deny
-                Allow from all
+                Require all granted
 	</Directory>
 	<Directory ~ "/var/files/git-mirror/.*\.workdir">
-		Order allow,deny
-		Deny from all
+                Require all denied
 	</Directory>
 	<Directory ~ "/var/files/git-mirror/__.*">
-                Order allow,deny
-                Deny from all
+                Require all denied
 	</Directory>
 
 	ErrorLog /var/log/apache2/code-error.log
--- a/deploy/config/database.yml.in	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/config/database.yml.in	Fri Aug 18 15:02:20 2017 +0100
@@ -3,5 +3,5 @@
   database: code
   host: localhost
   username: code
-  password: "INSERT_POSTGRES_PASSWORD_HERE"
+  password: "INSERT_DATABASE_PASSWORD_HERE"
 
--- a/deploy/config/run-external.sh.in	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/config/run-external.sh.in	Fri Aug 18 15:02:20 2017 +0100
@@ -6,11 +6,11 @@
 date >> $logfile
 /var/www/code/reposman/convert-external-repos.rb \
 	-s /var/hg \
-	-r https://code.soundsoftware.ac.uk/ \
+	-r INSERT_API_SCHEME_HERE://INSERT_API_HOST_HERE/ \
 	-k INSERT_API_KEY_HERE \
 	-v \
-	--http-user=user \
-	--http-pass=password \
+	--http-user=INSERT_API_USER_HERE \
+	--http-pass=INSERT_API_PASSWORD_HERE \
         -c "/var/www/code/reposman/update-external-repo.sh" \
 	>> $logfile 2>&1
 date >> $logfile
--- a/deploy/config/run-reposman.sh.in	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/config/run-reposman.sh.in	Fri Aug 18 15:02:20 2017 +0100
@@ -5,10 +5,10 @@
 echo >> $logfile
 /var/www/code/reposman/reposman-soundsoftware.rb \
 	-s /var/hg \
-	-r https://code.soundsoftware.ac.uk/ \
+	-r INSERT_API_SCHEME_HERE://INSERT_API_HOST_HERE/ \
 	-k INSERT_API_KEY_HERE \
-	--http-user=user \
-	--http-pass=password \
+	--http-user=INSERT_API_USER_HERE \
+	--http-pass=INSERT_API_PASSWORD_HERE \
 	-o www-data \
 	-g code \
 	-c "/var/www/code/reposman/run-hginit.sh" \
--- a/deploy/docker/start.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/docker/start.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -1,51 +1,7 @@
 #!/bin/bash
 
 mydir=$(dirname "$0")
-
-dbpwd="$1"
-if [ -z "$dbpwd" ]; then
-    echo "Usage: $0 <database-password>" 1>&2
-    exit 2
-fi
-
-set -eu -o pipefail
-
-rootdir="$mydir/../.."
-
-deploydir="$rootdir"/deploy
-if [ ! -d "$deploydir" ]; then
-    echo "ERROR: Unexpected repository layout - expected directory at $deploydir"
-    exit 2
-fi
-
-managerdir="$deploydir/docker"
-if [ ! -d "$managerdir" ]; then
-    echo "ERROR: Required directory $managerdir not found"
-    exit 2
-fi
-
-configdir="$deploydir/config"
-if [ ! -d "$configdir" ]; then
-    echo "ERROR: Required directory $configdir not found"
-    exit 2
-fi
-
-if [ ! -f "$rootdir/postgres-dumpall" ]; then
-    echo "ERROR: I expect to find a Postgres SQL multi-db dump file in $rootdir/postgres-dumpall"
-    exit 2
-fi
-
-fontdir="$rootdir"/public/themes/soundsoftware/stylesheets/fonts
-if [ ! -f "$fontdir/24BC0E_0_0.woff" ]; then
-    echo "ERROR: I expect to find necessary webfonts in $fontdir"
-    exit 2
-fi
-
-for f in database.yml code.conf ; do
-    cat "$configdir/$f.in" |
-        sed 's/INSERT_POSTGRES_PASSWORD_HERE/'"$dbpwd"'/g' > \
-            "$configdir/$f"
-done
+. "$mydir"/../any/prepare.sh
 
 provisioning_commands=$(
     for x in "$deploydir"/provision.d/[0-9]*.sh; do
@@ -59,12 +15,12 @@
 
 cat "$managerdir/Dockerfile.in" |
     sed 's,INSERT_PROVISIONING_HERE,'"$provisioning_commands"',' >> \
-        "$managerdir/Dockerfile"
+        "$managerdir/Dockerfile.gen"
 
 cd "$rootdir"
 
 dockertag="cannam/soundsoftware-site"
 
-sudo docker build -t "$dockertag" -f "deploy/docker/Dockerfile" .
+sudo docker build -t "$dockertag" -f "deploy/docker/Dockerfile.gen" .
 sudo docker run -p 8080:80 -d "$dockertag"
 
--- a/deploy/provision.d/050-webapp-db.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/provision.d/050-webapp-db.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -5,7 +5,7 @@
 # Copy across the database config file (the source file has presumably
 # been generated from a skeleton, earlier in provisioning)
 
-infile=/var/www/code/deploy/config/database.yml
+infile=/var/www/code/deploy/config/database.yml.gen
 outfile=/var/www/code/config/database.yml
 
 if [ ! -f "$outfile" ]; then
--- a/deploy/provision.d/100-apache-config.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/provision.d/100-apache-config.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -6,10 +6,10 @@
 
 cd /var/www/code
 
-codeconffile=/var/www/code/deploy/config/code.conf
+codeconffile=/var/www/code/deploy/config/code.conf.gen
 
 if [ ! -f "$codeconffile" ]; then
-    echo "ERROR: Apache config file $codeconffile not found - has the database secret been interpolated from $codeconffile.in correctly?"
+    echo "ERROR: Apache config file $codeconffile not found - has the database secret been interpolated from its input file correctly?"
     exit 2
 fi
 
--- a/deploy/provision.d/120-docgen.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/provision.d/120-docgen.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -12,7 +12,6 @@
 
 for file in \
     doxysafe.pl \
-    extract-docs.sh \
     extract-doxygen.sh \
     extract-javadoc.sh \
     extract-matlabdocs.sh \
@@ -24,6 +23,13 @@
     fi
 done
 
+for file in \
+    extract-docs.sh ; do
+    if [ ! -f docgen/"$file" ]; then
+        cp deploy/config/"$file".gen docgen/"$file"
+    fi
+done
+
 chown code.www-data docgen/*
 chmod +x docgen/*.sh
 
--- a/deploy/provision.d/130-reposman.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/provision.d/130-reposman.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -26,8 +26,7 @@
     run-external.sh \
     run-reposman.sh ; do
     if [ ! -f reposman/"$file" ]; then
-        ##!!! TODO: actually insert API key
-        cat deploy/config/"$file".in > reposman/"$file"
+        cp deploy/config/"$file".gen reposman/"$file"
     fi
 done
 
--- a/deploy/vagrant/start.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/vagrant/start.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -1,53 +1,8 @@
 #!/bin/bash
 
 mydir=$(dirname "$0")
-
-dbpwd="$1"
-if [ -z "$dbpwd" ]; then
-    echo "Usage: $0 <database-password>" 1>&2
-    exit 2
-fi
-
-set -eu -o pipefail
-
-rootdir="$mydir/../.."
-
-deploydir="$rootdir"/deploy
-if [ ! -d "$deploydir" ]; then
-    echo "ERROR: Unexpected repository layout - expected directory at $deploydir"
-    exit 2
-fi
-
-managerdir="$deploydir/vagrant"
-if [ ! -d "$managerdir" ]; then
-    echo "ERROR: Required directory $managerdir not found"
-    exit 2
-fi
-
-configdir="$deploydir/config"
-if [ ! -d "$configdir" ]; then
-    echo "ERROR: Required directory $configdir not found"
-    exit 2
-fi
-
-if [ ! -f "$rootdir/postgres-dumpall" ]; then
-    echo "ERROR: I expect to find a Postgres SQL multi-db dump file in $rootdir/postgres-dumpall"
-    exit 2
-fi
-
-fontdir="$rootdir"/public/themes/soundsoftware/stylesheets/fonts
-if [ ! -f "$fontdir/24BC0E_0_0.woff" ]; then
-    echo "ERROR: I expect to find necessary webfonts in $fontdir"
-    exit 2
-fi
-
-for f in database.yml code.conf ; do
-    cat "$configdir/$f.in" |
-        sed 's/INSERT_POSTGRES_PASSWORD_HERE/'"$dbpwd"'/g' > \
-            "$configdir/$f"
-done
+. "$mydir"/../any/prepare.sh
 
 cd "$managerdir"
-
 vagrant up
 
--- a/deploy/vagrant/vagrant-provision.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/deploy/vagrant/vagrant-provision.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -3,15 +3,16 @@
 #!!! still not covered:
 # * https
 # * http auth for API (/sys) and /admin interfaces
-# * API keys and http auth for reposman and docgen
+# * sending email
 
 set -e
 
 for f in /code-to-deploy/deploy/provision.d/[0-9]*.sh ; do
     case "$f" in
         *~) ;;
-        *) echo "Running provision script: $f"
+        *) echo "Running provisioning script: $f"
            /bin/bash "$f";;
     esac
 done
 
+echo "All provisioning scripts complete"
--- a/extra/soundsoftware/extract-docs.sh	Fri Aug 18 14:46:06 2017 +0100
+++ b/extra/soundsoftware/extract-docs.sh	Fri Aug 18 15:02:20 2017 +0100
@@ -11,11 +11,13 @@
 
 redgrp="code"
 
-apikey=""
-apischeme="https"
-apihost=""
-apiuser=""
-apipass=""
+apikey="INSERT_API_KEY_HERE"
+apischeme="INSERT_API_SCHEME_HERE"
+apihost="INSERT_API_HOST_HERE"
+
+# HTTP auth username/password for /sys api calls
+apiuser="INSERT_API_USER_HERE"
+apipass="INSERT_API_PASSWORD_HERE"
 
 progdir=$(dirname $0)
 case "$progdir" in