SoundSoftware Code Site

#!/bin/bash

set -e

# Install necessary system packages. This assumes we are deploying on

# Ubuntu 16.04.

# We aim to make all of these provisioning scripts non-destructive if

# run more than once. In this case, running the script again will

# install any outstanding updates.

apt-get update && \

    apt-get dist-upgrade -y && \

    apt-get install -y \

            ack-grep \

            apache2 \

            apache2-dev \

            apt-utils \

            build-essential \

            cron \

            curl \

            doxygen \

            exim4 \

            git \

            graphviz \

            imagemagick \

            libapache-dbi-perl \

            libapache2-mod-perl2 \

            libapr1-dev \

            libaprutil1-dev \

            libauthen-simple-ldap-perl \

            libcurl4-openssl-dev \

            libdbd-pg-perl \

            libpq-dev \

            libmagickwand-dev \

            libio-socket-ssl-perl \

            logrotate \

            mercurial \

            mercurial-git \

            openjdk-9-jdk-headless \

            postgresql \

            rsync \

            ruby \

            ruby-dev \

            sudo

locale-gen en_US.UTF-8

#!/bin/bash

set -e

# Phusion Passenger as application server.

# This gets installed through gem, not apt, and we ask for a specific

# version (the last in the 4.0.x line).

if [ ! -f /var/lib/gems/2.3.0/gems/passenger-4.0.60/buildout/apache2/mod_passenger.so ]; then

    gem install passenger -v 4.0.60 --no-rdoc --no-ri

    passenger-install-apache2-module --languages=ruby

fi

#!/bin/bash

set -e

# The webapp directory is owned and run by the code user, in group

# www-data. The repos and other things served directly are the other

# way around -- owned by the www-data user, in group code.

for user in code docgen ; do

    if ! grep -q "^$user:" /etc/passwd ; then

        groupadd "$user"

        useradd -g "$user" -G www-data "$user"

    fi

done

#!/bin/bash

set -e

# We might be running in one of two ways:

#

# 1. The code directory is already at /var/www/code, either because a

# previous provisioning step has imported it there or because this

# script has been run before -- in this situation all we do is

# re-check the ownership and permissions. OR

#

# 2. The code directory has not yet been copied to /var/www/code, in

# which case we expect to find it at /code-to-deploy, e.g. as a

# Vagrant shared folder, and we copy it over from there. (We don't

# deploy directly from shared folders as we might not be able to

# manipulate ownership and permissions properly there.)

if [ ! -d /var/www/code ]; then

    if [ ! -d /code-to-deploy ]; then

        echo "ERROR: Expected to find code tree at /var/www/code or /code-to-deploy: is the deployment script being invoked correctly?"

        exit 2

    fi

    cp -a /code-to-deploy /var/www/code

fi

chown -R code.www-data /var/www/code

chmod 755 /var/www/code

find /var/www/code -type d -exec chmod g+s \{\} \;

#!/bin/bash

set -e

# In a real deployment, /var/hg is probably mounted from somewhere

# else. But in an empty deployment we need to create it, and in both

# cases we set up the config files with their current versions here.

if [ ! -f /var/hg/index.cgi ]; then

    mkdir -p /var/hg

fi

cp /var/www/code/deploy/config/index.cgi /var/hg/

cp /var/www/code/deploy/config/hgweb.config /var/hg/

chmod +x /var/hg/index.cgi

chown -R www-data.code /var/hg

find /var/hg -type d -exec chmod g+s \{\} \;

#!/bin/bash

set -e

# Copy across the database config file (the source file has presumably

# been generated from a skeleton, earlier in provisioning)

infile=/var/www/code/deploy/config/database.yml.gen

outfile=/var/www/code/config/database.yml

if [ ! -f "$outfile" ]; then

    if [ ! -f "$infile" ]; then

        echo "ERROR: Database config file $infile not found - has the database secret been interpolated from $infile.in correctly?"

        exit 2

    fi

    cp "$infile" "$outfile"

fi

#!/bin/bash

set -e

# Install Ruby gems for the web app.

# We aim to make all of these provisioning scripts non-destructive if

# run more than once. In this case, running the script again will

# install any outstanding updates.

cd /var/www/code

gem install bundler

bundle install

#!/bin/bash

set -e

# Create a session token if it hasn't already been created.

cd /var/www/code

if [ ! -f config/initializers/secret_token.rb ]; then

    bundle exec rake generate_secret_token

fi

#!/bin/bash

set -e

# Start the database and if a dump file is found, load it. The dump

# file is then deleted so that the db won't be overwritten on

# subsequent runs. (The original repo contains no dump file, so it

# should exist only if you have provided some data to load.)

/etc/init.d/postgresql start

cd /var/www/code

if [ -f postgres-dumpall ]; then

    chmod ugo+r postgres-dumpall

    sudo -u postgres psql -f postgres-dumpall postgres

    rm postgres-dumpall

fi

#!/bin/bash

set -e

# Install the Apache mod_perl module used for hg repo access control

if [ ! -f /usr/local/lib/site_perl/Apache/Authn/SoundSoftware.pm ]; then

    mkdir -p /usr/local/lib/site_perl/Apache/Authn/

    cp /var/www/code/extra/soundsoftware/SoundSoftware.pm \

       /usr/local/lib/site_perl/Apache/Authn/

fi

#!/bin/bash

set -e

# Install Apache config files and module loaders

cd /var/www/code

codeconf=/var/www/code/deploy/config/code.conf.gen

codeconfssl=/var/www/code/deploy/config/code-ssl.conf.gen

staticconf=/var/www/code/deploy/config/soundsoftware-static.conf

if [ ! -f "$codeconf" ]; then

    echo "ERROR: Apache config file $codeconf not found - has the database secret been interpolated from its input file correctly?"

    exit 2

fi

if [ ! -f /etc/apache2/sites-enabled/10-code.conf ]; then

    rm -f /etc/apache2/sites-enabled/000-default.conf

    cp deploy/config/passenger.conf /etc/apache2/mods-available/

    cp deploy/config/passenger.load /etc/apache2/mods-available/

    cp deploy/config/perl.conf      /etc/apache2/mods-available/

    ln -s ../mods-available/passenger.conf  /etc/apache2/mods-enabled/

    ln -s ../mods-available/passenger.load  /etc/apache2/mods-enabled/

    ln -s ../mods-available/perl.conf       /etc/apache2/mods-enabled/

    ln -s ../mods-available/expires.load    /etc/apache2/mods-enabled/

    ln -s ../mods-available/rewrite.load    /etc/apache2/mods-enabled/

    ln -s ../mods-available/cgi.load        /etc/apache2/mods-enabled/

    ln -s ../mods-available/ssl.load        /etc/apache2/mods-enabled/

    cp "$codeconf" /etc/apache2/sites-available/code.conf

    cp "$codeconfssl" /etc/apache2/sites-available/code-ssl.conf

    cp "$staticconf" /etc/apache2/sites-available/soundsoftware-static.conf

    ln -s ../sites-available/code.conf /etc/apache2/sites-enabled/10-code.conf

    apache2ctl configtest

fi

#!/bin/bash

set -e

# In case we are running without a properly mounted /var/hg directory,

# check for the existence of one repo and, if absent, attempt to clone

# it so that we have something we can serve for test purposes.

if [ ! -d /var/hg/vamp-plugin-sdk ]; then

    echo "Cloning vamp-plugin-sdk repo for testing..."

    cd /var/hg

    hg clone https://code.soundsoftware.ac.uk/hg/vamp-plugin-sdk

    chown -R www-data.code vamp-plugin-sdk

fi

#!/bin/bash

set -e

# Initialise directories used as targets for cron activity (if they

# don't already exist)

# Reminder: the webapp directory is owned and run by the code user, in

# group www-data. The repos and other things served directly are

# usually the other way around -- owned by the www-data user, in group

# code. I don't recall whether there is a good reason for this.

for dir in \

    /var/files/backups \

    /var/doc \

    /var/files/git-mirror ; do

    if [ ! -d "$dir" ]; then

        mkdir -p "$dir"

        chown -R code.www-data "$dir"

        chmod g+s "$dir"

    fi

done

for dir in \

    /var/mirror ; do

    if [ ! -d "$dir" ]; then

        mkdir -p "$dir"

        chown -R www-data.code "$dir"

        chmod g+s "$dir"

    fi

done

#!/bin/bash

set -e

# Copy docgen scripts, including the generated scripts with

# interpolated API key etc, to the directory they will be run from.

# These are run from cron jobs to do the (currently daily) update of

# extracted documentation from Doxygen, Javadoc, and MATLAB, and to

# enable displaying them with the redmine_embedded plugin. (The API

# key is needed to automatically switch on the embedded module for a

# project the first time its docs are extracted.)

cd /var/www/code

mkdir -p docgen

for file in \

    doxysafe.pl \

    extract-doxygen.sh \

    extract-javadoc.sh \

    extract-matlabdocs.sh \

    matlab-docs.conf \

    matlab-docs-credit.html \

    matlab-docs.pl ; do

    if [ ! -f docgen/"$file" ]; then

        cp extra/soundsoftware/"$file" docgen/

    fi

done

for file in \

    extract-docs.sh ; do

    if [ ! -f docgen/"$file" ]; then

        cp deploy/config/"$file".gen docgen/"$file"

    fi

done

chown code.www-data docgen/*

chmod +x docgen/*.sh

#!/bin/bash

set -e

# Copy reposman (repository manager) scripts, including the generated

# scripts with interpolated API key etc, to the directory they will be

# run from.

# There are two sets of scripts here:

#

# 1. The reposman script that plods through all the projects that have

# repositories defined, creates those repositories on disc, and

# registers their locations with the projects. This happens often,

# currently every minute.

#

# 2. The external repo management script that plods through all the

# projects that have external repositories defined, clones or updates

# those external repos to their local locations, and if necessary

# registers them with the projects. This happens less often, currently

# every hour.

cd /var/www/code

mkdir -p reposman

for file in \

    convert-external-repos.rb \

    reposman-soundsoftware.rb \

    run-hginit.sh \

    update-external-repo.sh ; do

    if [ ! -f reposman/"$file" ]; then

        cp extra/soundsoftware/"$file" reposman/

    fi

done

for file in \

    run-external.sh \

    run-reposman.sh ; do

    if [ ! -f reposman/"$file" ]; then

        cp deploy/config/"$file".gen reposman/"$file"

    fi

done

chown code.www-data reposman/*

chmod +x reposman/*.sh

chmod +x reposman/*.rb

touch /var/log/reposman.log

touch /var/log/external-repos.log

chown www-data.code /var/log/reposman.log

chown www-data.code /var/log/external-repos.log

#!/bin/bash

set -e

# Copy cron scripts to the appropriate destinations

cd /var/www/code

if [ ! -d /etc/cron.minutely ]; then

    mkdir -p /etc/cron.minutely

    echo '*  *    * * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.minutely )' >> /etc/crontab

fi

for t in minutely hourly daily monthly; do

    for s in deploy/config/cron.$t/[0-9]* ; do

        name=$(basename $s)

        dest="/etc/cron.$t/$name"

        if [ ! -f "$dest" ]; then

            cp "$s" "$dest"

            chmod +x "$dest"

        fi

    done

done

#!/bin/bash

cd /var/www/code

cp deploy/config/logrotate.conf /etc/logrotate.conf

#!/bin/bash

set -e

# Print reminders of the things that we haven't covered in the deploy

# scripts

cat <<EOF

*** APACHE SSL CONFIGURATION

    The provisioning scripts set up a simple HTTP site only. Refer to

    code-ssl.conf for an example HTTPS configuration (you will of

    course need to provide the key/cert files).

*** SMOKE TEST

    There is a smoke test script in the deploy/test directory. That

    is, a quick automated acceptance test that checks that basic

    services are returning successful HTTP codes. Consider running it

    against this server from another host, i.e. not just localhost.

*** EMAIL

    Outgoing email is required for notifications, but has not been

    configured as part of this provisioning setup.

*** STATIC FRONT PAGE

    We have set up only the code/repository site -- if you want a

    separate front page, remember to configure that!

EOF

#!/bin/bash

set -e

# Last action: check & start the webserver

apache2ctl configtest

apache2ctl restart