SoundSoftware Code Site

#!/bin/bash

set -e

# Install necessary system packages. This assumes we are deploying on

# Ubuntu 16.04.

# We aim to make all of these provisioning scripts non-destructive if

# run more than once. In this case, running the script again will

# install any outstanding updates.

apt-get update && \

    apt-get dist-upgrade -y && \

    apt-get install -y \

            ack-grep \

            apache2 \

            apache2-dev \

            apt-utils \

            build-essential \

            cron \

            curl \

            doxygen \

            exim4 \

            git \

            graphviz \

            imagemagick \

            libapache-dbi-perl \

            libapache2-mod-perl2 \

            libapr1-dev \

            libaprutil1-dev \

            libauthen-simple-ldap-perl \

            libcurl4-openssl-dev \

            libdbd-pg-perl \

            libpq-dev \

            libmagickwand-dev \

            libio-socket-ssl-perl \

            logrotate \

            lynx \

            mercurial \

            mercurial-git \

            openjdk-9-jdk-headless \

            postgresql \

            rsync \

            ruby \

            ruby-dev \

            sudo

locale-gen en_US.UTF-8

#!/bin/bash

set -e

# Phusion Passenger as application server.

# This gets installed through gem, not apt, and we ask for a specific

# version (the last in the 4.0.x line).

if [ ! -f /var/lib/gems/2.3.0/gems/passenger-4.0.60/buildout/apache2/mod_passenger.so ]; then

    gem install passenger -v 4.0.60 --no-rdoc --no-ri

    passenger-install-apache2-module --languages=ruby

fi

#!/bin/bash

set -e

# The webapp directory is owned and run by the code user, in group

# www-data. The repos and other things served directly are the other

# way around -- owned by the www-data user, in group code.

for user in code docgen ; do

    if ! grep -q "^$user:" /etc/passwd ; then

        groupadd "$user"

        useradd -g "$user" -G www-data "$user"

    fi

done

#!/bin/bash

set -e

# We might be running in one of two ways:

#

# 1. The code directory is already at /var/www/code, either because a

# previous provisioning step has imported it there or because this

# script has been run before -- in this situation all we do is

# re-check the ownership and permissions. OR

#

# 2. The code directory has not yet been copied to /var/www/code, in

# which case we expect to find it at /code-to-deploy, e.g. as a

# Vagrant shared folder, and we copy it over from there. (We don't

# deploy directly from shared folders as we might not be able to

# manipulate ownership and permissions properly there.)

if [ ! -d /var/www/code ]; then

    if [ ! -d /code-to-deploy ]; then

        echo "ERROR: Expected to find code tree at /var/www/code or /code-to-deploy: is the deployment script being invoked correctly?"

        exit 2

    fi

    cp -a /code-to-deploy /var/www/code

fi

chown -R code.www-data /var/www/code

chmod 755 /var/www/code

find /var/www/code -type d -exec chmod g+s \{\} \;

#!/bin/bash

set -e

# In a real deployment, /var/hg is probably mounted from somewhere

# else. But in an empty deployment we need to create it, and in both

# cases we set up the config files with their current versions here.

if [ ! -f /var/hg/index.cgi ]; then

    mkdir -p /var/hg

fi

cp /var/www/code/deploy/config/index.cgi /var/hg/

cp /var/www/code/deploy/config/hgweb.config /var/hg/

chmod +x /var/hg/index.cgi

chown -R www-data.code /var/hg

find /var/hg -type d -exec chmod g+s \{\} \;

#!/bin/bash

set -e

# Copy across the database config file (the source file has presumably

# been generated from a skeleton, earlier in provisioning)

infile=/var/www/code/deploy/config/database.yml.gen

outfile=/var/www/code/config/database.yml

if [ ! -f "$outfile" ]; then

    if [ ! -f "$infile" ]; then

        echo "ERROR: Database config file $infile not found - has the database secret been interpolated from $infile.in correctly?"

        exit 2

    fi

    cp "$infile" "$outfile"

fi

#!/bin/bash

set -e

# Install Ruby gems for the web app.

# We aim to make all of these provisioning scripts non-destructive if

# run more than once. In this case, running the script again will

# install any outstanding updates.

cd /var/www/code

gem install bundler

bundle install

#!/bin/bash

set -e

# Create a session token if it hasn't already been created.

cd /var/www/code

if [ ! -f config/initializers/secret_token.rb ]; then

    bundle exec rake generate_secret_token

fi

#!/bin/bash

set -e

# Start the database and if a dump file is found, load it. The dump

# file is then deleted so that the db won't be overwritten on

# subsequent runs. (The original repo contains no dump file, so it

# should exist only if you have provided some data to load.)

/etc/init.d/postgresql start

dumpdir="/code-to-deploy"

if [ ! -d "$dumpdir" ]; then

    dumpdir=/var/www/code

fi

cd "$dumpdir"

if [ -f postgres-dumpall ]; then

    chmod ugo+r postgres-dumpall

    sudo -u postgres psql -f postgres-dumpall postgres

    rm postgres-dumpall

fi

#!/bin/bash

set -e

# Install the Apache mod_perl module used for hg repo access control

if [ ! -f /usr/local/lib/site_perl/Apache/Authn/SoundSoftware.pm ]; then

    mkdir -p /usr/local/lib/site_perl/Apache/Authn/

    cp /var/www/code/extra/soundsoftware/SoundSoftware.pm \

       /usr/local/lib/site_perl/Apache/Authn/

fi

#!/bin/bash

set -e

# Install Apache config files and module loaders

cd /var/www/code

codeconf=/var/www/code/deploy/config/code.conf.gen

codeconfssl=/var/www/code/deploy/config/code-ssl.conf.gen

staticconf=/var/www/code/deploy/config/soundsoftware-static.conf

if [ ! -f "$codeconf" ]; then

    echo "ERROR: Apache config file $codeconf not found - has the database secret been interpolated from its input file correctly?"

    exit 2

fi

if [ ! -f /etc/apache2/sites-enabled/10-code.conf ]; then

    rm -f /etc/apache2/sites-enabled/000-default.conf

    cp deploy/config/passenger.conf /etc/apache2/mods-available/

    cp deploy/config/passenger.load /etc/apache2/mods-available/

    cp deploy/config/perl.conf      /etc/apache2/mods-available/

    ln -s ../mods-available/passenger.conf   /etc/apache2/mods-enabled/

    ln -s ../mods-available/passenger.load   /etc/apache2/mods-enabled/

    ln -s ../mods-available/perl.conf        /etc/apache2/mods-enabled/

    ln -s ../mods-available/expires.load     /etc/apache2/mods-enabled/

    ln -s ../mods-available/rewrite.load     /etc/apache2/mods-enabled/

    ln -s ../mods-available/cgi.load         /etc/apache2/mods-enabled/

    ln -s ../mods-available/ssl.load         /etc/apache2/mods-enabled/

    ln -s ../mods-available/auth_digest.load /etc/apache2/mods-enabled/

    cp "$codeconf" /etc/apache2/sites-available/code.conf

    cp "$codeconfssl" /etc/apache2/sites-available/code-ssl.conf

    cp "$staticconf" /etc/apache2/sites-available/soundsoftware-static.conf

    ln -s ../sites-available/code.conf /etc/apache2/sites-enabled/10-code.conf

    apache2ctl configtest

fi

#!/bin/bash

set -e

# In case we are running without a properly mounted /var/hg directory,

# check for the existence of one repo and, if absent, attempt to clone

# it so that we have something we can serve for test purposes.

if [ ! -d /var/hg/vamp-plugin-sdk ]; then

    echo "Cloning vamp-plugin-sdk repo for testing..."

    cd /var/hg

    hg clone https://code.soundsoftware.ac.uk/hg/vamp-plugin-sdk

    chown -R www-data.code vamp-plugin-sdk

fi

#!/bin/bash

set -e

# Initialise directories used as targets for cron activity (if they

# don't already exist)

# Reminder: the webapp directory is owned and run by the code user, in

# group www-data. The repos and other things served directly are

# usually the other way around -- owned by the www-data user, in group

# code. I don't recall whether there is a good reason for this.

for dir in \

    /var/files/backups \

    /var/doc \

    /var/files/code \

    /var/files/git-mirror ; do

    if [ ! -d "$dir" ]; then

        mkdir -p "$dir"

        chown -R code.www-data "$dir"

        chmod g+s "$dir"

    fi

done

for dir in \

    /var/mirror ; do

    if [ ! -d "$dir" ]; then

        mkdir -p "$dir"

        chown -R www-data.code "$dir"

        chmod g+s "$dir"

    fi

done

if [ ! -e /var/www/code/files ]; then

    ln -s /var/files/code /var/www/code/files

fi

#!/bin/bash

set -e

# Copy docgen scripts, including the generated scripts with

# interpolated API key etc, to the directory they will be run from.

# These are run from cron jobs to do the (currently daily) update of

# extracted documentation from Doxygen, Javadoc, and MATLAB, and to

# enable displaying them with the redmine_embedded plugin. (The API

# key is needed to automatically switch on the embedded module for a

# project the first time its docs are extracted.)

cd /var/www/code

mkdir -p docgen

for file in \

    doxysafe.pl \

    extract-doxygen.sh \

    extract-javadoc.sh \

    extract-matlabdocs.sh \

    matlab-docs.conf \

    matlab-docs-credit.html \

    matlab-docs.pl ; do

    if [ ! -f docgen/"$file" ]; then

        cp extra/soundsoftware/"$file" docgen/

    fi

done

for file in \

    extract-docs.sh ; do

    if [ ! -f docgen/"$file" ]; then

        cp deploy/config/"$file".gen docgen/"$file"

    fi

done

chown code.www-data docgen/*

chmod +x docgen/*.sh

#!/bin/bash

set -e

# Copy reposman (repository manager) scripts, including the generated

# scripts with interpolated API key etc, to the directory they will be

# run from.

# There are two sets of scripts here:

#

# 1. The reposman script that plods through all the projects that have

# repositories defined, creates those repositories on disc, and

# registers their locations with the projects. This happens often,

# currently every minute.

#

# 2. The external repo management script that plods through all the

# projects that have external repositories defined, clones or updates

# those external repos to their local locations, and if necessary

# registers them with the projects. This happens less often, currently

# every hour.

cd /var/www/code

mkdir -p reposman

for file in \

    convert-external-repos.rb \

    reposman-soundsoftware.rb \

    run-hginit.sh \

    update-external-repo.sh ; do

    if [ ! -f reposman/"$file" ]; then

        cp extra/soundsoftware/"$file" reposman/

    fi

done

for file in \

    run-external.sh \

    run-reposman.sh ; do

    if [ ! -f reposman/"$file" ]; then

        cp deploy/config/"$file".gen reposman/"$file"

    fi

done

chown code.www-data reposman/*

chmod +x reposman/*.sh

chmod +x reposman/*.rb

touch /var/log/reposman.log

touch /var/log/external-repos.log

chown www-data.code /var/log/reposman.log

chown www-data.code /var/log/external-repos.log

#!/bin/bash

set -e

# Copy cron scripts to the appropriate destinations

cd /var/www/code

if [ ! -d /etc/cron.minutely ]; then

    mkdir -p /etc/cron.minutely

    echo '*  *    * * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.minutely )' >> /etc/crontab

fi

for t in minutely hourly daily monthly; do

    for s in deploy/config/cron.$t/[0-9]* ; do

        name=$(basename $s)

        dest="/etc/cron.$t/$name"

        if [ ! -f "$dest" ]; then

            cp "$s" "$dest"

            chmod +x "$dest"

        fi

    done

done

#!/bin/bash

cd /var/www/code

cp deploy/config/logrotate.conf /etc/logrotate.conf

#!/bin/bash

set -e

# Print reminders of the things that we haven't covered in the deploy

# scripts

cat <<EOF

*** APACHE SSL CONFIGURATION

    The provisioning scripts set up a simple HTTP site only. Refer to

    code-ssl.conf for an example HTTPS configuration (you will of

    course need to provide the key/cert files).

*** CRON SCRIPTS

    A number of cron scripts have been installed. It might be no bad

    thing to prime and test them by running them all once now. Some of

    the services tested by the smoke test script (below) may depend on

    their having run. Use deploy/any/run-cron-scripts.sh for this.

*** SMOKE TEST

    There is a smoke test script in the deploy/test directory. That

    is, a quick automated acceptance test that checks that basic

    services are returning successful HTTP codes. Consider running it

    against this server from another host, i.e. not just localhost.

*** EMAIL

    Outgoing email is required for notifications, but has not been

    configured as part of this provisioning setup. You'll need to set

    up the server's outgoing mail support and also edit the application

    email settings in config/configuration.yml.

*** STATIC FRONT PAGE

    We have set up only the code/repository site -- if you want a

    separate front page, remember to configure that!

EOF

#!/bin/bash

set -e

# Last action: check & start the webserver

apache2ctl configtest

apache2ctl restart