# HG changeset patch # User Chris Cannam # Date 1504606790 -3600 # Node ID 96bcc0ebd102636beaf121946c49194297e185bc # Parent bc47b68a9487a677b86a90438fbbf6cf282315cc# Parent 90bed4e10cc8e809ac15b8ad4600441ffe5ab9bd Merge from branch deploy diff -r bc47b68a9487 -r 96bcc0ebd102 .dockerignore --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.dockerignore Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,6 @@ +.dockerignore +.hg +.hgignore +deploy/docker/Dockerfile +deploy/docker/start.sh +*.log diff -r bc47b68a9487 -r 96bcc0ebd102 .hgignore --- a/.hgignore Tue Aug 01 13:30:33 2017 +0100 +++ b/.hgignore Tue Sep 05 11:19:50 2017 +0100 @@ -1,5 +1,4 @@ syntax: glob - .project .loadpath .powrc @@ -35,7 +34,13 @@ *~ public/themes/soundsoftware/stylesheets/fonts/* .bundle -Gemfile.lock Gemfile.local - +*.interpolated re:^config\.ru$ +.vagrant +*.orig +*.pyc +*-console.log +postgres-dumpall +*.gen +deploy/test/output diff -r bc47b68a9487 -r 96bcc0ebd102 Gemfile --- a/Gemfile Tue Aug 01 13:30:33 2017 +0100 +++ b/Gemfile Tue Sep 05 11:19:50 2017 +0100 @@ -9,10 +9,6 @@ gem "mime-types" gem "awesome_nested_set", "2.1.6" -#cc -- CiteProc v1.0.0 broke our citations (CiteProc.process returns nil). -# Until I've managed to work out what's up and fix that... -gem "citeproc", "0.0.6" - # Optional gem for LDAP authentication group :ldap do gem "net-ldap", "~> 0.3.1" diff -r bc47b68a9487 -r 96bcc0ebd102 Gemfile.lock --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Gemfile.lock Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,175 @@ +GEM + remote: https://rubygems.org/ + specs: + actionmailer (3.2.22.5) + actionpack (= 3.2.22.5) + mail (~> 2.5.4) + actionpack (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) + builder (~> 3.0.0) + erubis (~> 2.7.0) + journey (~> 1.0.4) + rack (~> 1.4.5) + rack-cache (~> 1.2) + rack-test (~> 0.6.1) + sprockets (~> 2.2.1) + activemodel (3.2.22.5) + activesupport (= 3.2.22.5) + builder (~> 3.0.0) + activerecord (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) + arel (~> 3.0.2) + tzinfo (~> 0.3.29) + activeresource (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) + activesupport (3.2.22.5) + i18n (~> 0.6, >= 0.6.4) + multi_json (~> 1.0) + acts-as-taggable-on (2.3.3) + rails (~> 3.0) + arel (3.0.3) + awesome_nested_set (2.1.6) + activerecord (>= 3.0.0) + bibtex-ruby (4.4.4) + latex-decode (~> 0.0) + builder (3.0.0) + capybara (2.1.0) + mime-types (>= 1.16) + nokogiri (>= 1.3.3) + rack (>= 1.0.0) + rack-test (>= 0.5.4) + xpath (~> 2.0) + childprocess (0.7.1) + ffi (~> 1.0, >= 1.0.11) + citeproc (1.0.5) + namae (~> 0.8) + citeproc-ruby (1.1.7) + citeproc (>= 1.0.4, < 2.0) + csl (~> 1.4) + coderay (1.1.1) + csl (1.4.5) + namae (~> 0.7) + csl-styles (1.0.1.8) + csl (~> 1.0) + erubis (2.7.0) + fastercsv (1.5.5) + ffi (1.9.18) + hike (1.2.3) + i18n (0.8.6) + journey (1.0.4) + jquery-rails (2.0.3) + railties (>= 3.1.0, < 5.0) + thor (~> 0.14) + json (1.8.6) + latex-decode (0.2.2) + unicode (~> 0.4) + mail (2.5.5) + mime-types (~> 1.16) + treetop (~> 1.4.8) + metaclass (0.0.4) + mime-types (1.25.1) + mini_portile2 (2.2.0) + mocha (1.0.0) + metaclass (~> 0.0.1) + multi_json (1.12.1) + namae (0.11.3) + net-ldap (0.3.1) + nokogiri (1.8.0) + mini_portile2 (~> 2.2.0) + pg (0.21.0) + polyglot (0.3.5) + rack (1.4.7) + rack-cache (1.7.0) + rack (>= 0.4) + rack-openid (1.4.2) + rack (>= 1.1.0) + ruby-openid (>= 2.1.8) + rack-ssl (1.3.4) + rack + rack-test (0.6.3) + rack (>= 1.0) + rails (3.2.22.5) + actionmailer (= 3.2.22.5) + actionpack (= 3.2.22.5) + activerecord (= 3.2.22.5) + activeresource (= 3.2.22.5) + activesupport (= 3.2.22.5) + bundler (~> 1.0) + railties (= 3.2.22.5) + railties (3.2.22.5) + actionpack (= 3.2.22.5) + activesupport (= 3.2.22.5) + rack-ssl (~> 1.3.2) + rake (>= 0.8.7) + rdoc (~> 3.4) + thor (>= 0.14.6, < 2.0) + rake (10.1.1) + rdoc (3.12.2) + json (~> 1.4) + redcarpet (2.3.0) + rmagick (2.16.0) + ruby-openid (2.3.0) + rubyzip (1.2.1) + selenium-webdriver (3.5.1) + childprocess (~> 0.5) + rubyzip (~> 1.0) + shoulda (3.3.2) + shoulda-context (~> 1.0.1) + shoulda-matchers (~> 1.4.1) + shoulda-context (1.0.2) + shoulda-matchers (1.4.1) + activesupport (>= 3.0.0) + sprockets (2.2.3) + hike (~> 1.2) + multi_json (~> 1.0) + rack (~> 1.0) + tilt (~> 1.1, != 1.3.0) + thor (0.19.4) + tilt (1.4.1) + treetop (1.4.15) + polyglot + polyglot (>= 0.3.1) + tzinfo (0.3.53) + unicode (0.4.4.4) + xpath (2.1.0) + nokogiri (~> 1.3) + yard (0.9.9) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord-jdbc-adapter (~> 1.3.2) + activerecord-jdbcpostgresql-adapter + acts-as-taggable-on (= 2.3.3) + awesome_nested_set (= 2.1.6) + bibtex-ruby + builder (= 3.0.0) + capybara (~> 2.1.0) + citeproc + citeproc-ruby + coderay (~> 1.1.0) + csl-styles + fastercsv (~> 1.5.0) + jquery-rails (~> 2.0.2) + mime-types + mocha (~> 1.0.0) + net-ldap (~> 0.3.1) + nokogiri + pg (>= 0.11.0) + rack-openid + rails (~> 3.2.22) + rake (~> 10.1.1) + rdoc (>= 2.4.2) + redcarpet (~> 2.3.0) + rmagick (>= 2.0.0) + ruby-openid (~> 2.3.0) + selenium-webdriver + shoulda (~> 3.3.2) + yard + +BUNDLED WITH + 1.15.3 diff -r bc47b68a9487 -r 96bcc0ebd102 app/views/projects/explore.html.erb --- a/app/views/projects/explore.html.erb Tue Aug 01 13:30:33 2017 +0100 +++ b/app/views/projects/explore.html.erb Tue Sep 05 11:19:50 2017 +0100 @@ -11,11 +11,12 @@

<%= l(:label_explore_projects) %>

-
-

<%=l(:label_project_tags_all)%>

- <% cache(:action => 'explore', :action_suffix => 'tags') do %> - <%= render :partial => 'projects/tagcloud' %> +
+

<%=l(:label_projects_busy)%>

+ <% cache(:action => 'explore', :action_suffix => 'busy_projects') do %> + <%= render :partial => 'activities/busy' %> <% end %> + <%= link_to l(:label_overall_activity), { :controller => 'activities', :action => 'index' }, :class => 'more' %>
@@ -27,26 +28,6 @@
-
-
-

<%=l(:label_institutions_busy)%>

- <% cache(:action => 'explore', :action_suffix => 'busy_institutions') do %> - <%= render :partial => 'activities/busy_institution' %> - <% end %> - <%= link_to l(:label_overall_activity), { :controller => 'activities', :action => 'index' }, :class => 'more' %> -
-
- -
-
-

<%=l(:label_projects_busy)%>

- <% cache(:action => 'explore', :action_suffix => 'busy_projects') do %> - <%= render :partial => 'activities/busy' %> - <% end %> - <%= link_to l(:label_overall_activity), { :controller => 'activities', :action => 'index' }, :class => 'more' %> -
-
-

<%=l(:label_projects_mature)%>

@@ -57,4 +38,12 @@
+
+

<%=l(:label_project_tags_all)%>

+ <% cache(:action => 'explore', :action_suffix => 'tags') do %> + <%= render :partial => 'projects/tagcloud' %> + <% end %> +
+ + <% html_title(l(:label_explore_projects)) -%> diff -r bc47b68a9487 -r 96bcc0ebd102 app/views/repositories/_dir_list.html.erb --- a/app/views/repositories/_dir_list.html.erb Tue Aug 01 13:30:33 2017 +0100 +++ b/app/views/repositories/_dir_list.html.erb Tue Sep 05 11:19:50 2017 +0100 @@ -1,4 +1,4 @@ -
+
diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/README --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/README Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,56 @@ + +Deploying the SoundSoftware site +================================ + +These scripts can be used for test or staging deployments reproducing +much of the configuration of the live site. Currently it's assumed +that you are providing a database dump to load -- there is no +provisioning step to initialise a new database. + + +You will need +------------- + +Required: + + * A database dump to load. This should be left in a file called + postgres-dumpall in the soundsoftware-site root + + * The database password and /sys API key for the target site. (This + can be queried from the db: settings table, name "sys_api_key". You + can change it in the admin UI; grep API config/*.in to see the + files you'll need to update if you change it) + + * The (copyrighted) web font files used in our deployment. Leave + these in /public/themes/soundsoftware/stylesheets/fonts/ + +Optional (or required for proper deployments): + + * HTTPS key/cert files + + +Three ways to deploy +-------------------- + + 1. Using Vagrant to set up a development VM: Run ./vagrant/start.sh + + 2. Using Docker to set up a development container: Run ./docker/start.sh + + 3. On a "real" VM or server: + + * Ensure the soundsoftware-site repo is checked out at /code-to-deploy + * Run /code-to-deploy/deploy/any/run-provisioning.sh as root + + But be very careful with this! You could screw up a dev box -- or + an existing live server! -- if you accidentally provision the site + directly onto it when you should have used Vagrant or a container. + + +After deployment +---------------- + +There is a smoke test script at test/smoketest.sh which checks that +the home page, a project page, a repo page etc can be retrieved. Some +of the pages it tries to retrieve are dependent on their generating +cron scripts having run at least once since the server was set up. + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/any/prepare.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/any/prepare.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,75 @@ +#!/bin/bash + +# To be sourced into a container-specific start.sh file, not run +# standalone + +usage() { + echo "Usage: $0 " 1>&2 + exit 2 +} + +dbpass="$1" +if [ -z "$dbpass" ]; then + usage +fi + +apikey="$2" +if [ -z "$apikey" ]; then + usage +fi + +apipass="$3" +if [ -z "$apipass" ]; then + usage +fi + +set -eu -o pipefail + +rootdir="$mydir/../.." + +deploydir="$rootdir"/deploy +if [ ! -d "$deploydir" ]; then + echo "ERROR: Unexpected repository layout - expected directory at $deploydir" + exit 2 +fi + +managerdir="$deploydir/vagrant" +if [ ! -d "$managerdir" ]; then + echo "ERROR: Required directory $managerdir not found" + exit 2 +fi + +configdir="$deploydir/config" +if [ ! -d "$configdir" ]; then + echo "ERROR: Required directory $configdir not found" + exit 2 +fi + +if [ ! -f "$rootdir/postgres-dumpall" ]; then + echo "ERROR: I expect to find a Postgres SQL multi-db dump file in $rootdir/postgres-dumpall. Create an empty file there if you don't want to load a database." + exit 2 +fi + +fontdir="$rootdir"/public/themes/soundsoftware/stylesheets/fonts +if [ ! -f "$fontdir/24BC0E_0_0.woff" ]; then + echo "ERROR: I expect to find necessary webfonts in $fontdir" + exit 2 +fi + +#apischeme=http +#apihost=localhost + +apischeme=https +apihost=code.soundsoftware.ac.uk + +for f in "$configdir"/*.in "$rootdir"/extra/soundsoftware/extract-docs.sh ; do + out="$configdir"/$(basename "$f" .in).gen + cat "$f" | sed \ + -e 's/INSERT_DATABASE_PASSWORD_HERE/'"$dbpass"'/g' \ + -e 's/INSERT_API_KEY_HERE/'"$apikey"'/g' \ + -e 's/INSERT_API_SCHEME_HERE/'"$apischeme"'/g' \ + -e 's/INSERT_API_HOST_HERE/'"$apihost"'/g' \ + -e 's/INSERT_API_USER_HERE/user/g' \ + -e 's/INSERT_API_PASSWORD_HERE/'"$apipass"'/g' \ + > "$out" +done diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/any/run-cron-scripts.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/any/run-cron-scripts.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +cd /var/www/code + +for t in minutely hourly daily monthly; do + for s in deploy/config/cron.$t/[0-9]* ; do + name=$(basename $s) + actual="/etc/cron.$t/$name" + echo "Running cron script $actual..." + "$actual" + done +done diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/any/run-provisioning.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/any/run-provisioning.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,27 @@ +#!/bin/bash + +mydir=$(dirname "$0") +case "$mydir" in + /*) ;; + *) mydir=$(echo "$(pwd)/$mydir" | sed 's,/\./,/,g') +esac + +if [ "$mydir" != "/code-to-deploy/deploy/any" ]; then + echo "ERROR: Expected repository to be at /code-to-deploy prior to provisioning" + echo " (My directory is $mydir, expected /code-to-deploy/deploy/any)" + exit 2 +fi + +set -e + +. "$mydir"/prepare.sh + +for f in "$mydir"/../provision.d/[0-9]*.sh ; do + case "$f" in + *~) ;; + *) echo "Running provisioning script: $f" + /bin/bash "$f" + ServerName code.soundsoftware.ac.uk + ServerAdmin chris.cannam@soundsoftware.ac.uk + + DocumentRoot /var/www/code/public + PassengerRestartDir restart_files + PassengerHighPerformance on + PassengerMaxRequests 50000 + PassengerStatThrottleRate 5 + PassengerFriendlyErrorPages off + RailsSpawnMethod smart + ExpiresDefault "access plus 1 minute" + + # Redirect all activity to secure site + Redirect seeother / "https://code.soundsoftware.ac.uk/" + + + Require all denied + + + + Require all denied + + + + Require all denied + + + + Options -MultiViews + + + ErrorLog /var/log/apache2/code-error.log + CustomLog /var/log/apache2/code-access.log vhost_combined + + LogLevel warn + ServerSignature Off + + + + ServerName code.soundsoftware.ac.uk + ServerAdmin chris.cannam@soundsoftware.ac.uk + + SSLEngine on + SSLCertificateFile /etc/apache2/certs/code.soundsoftware.ac.uk.crt + SSLCertificateKeyFile /etc/apache2/certs/code.soundsoftware.ac.uk.key + SSLCertificateChainFile /etc/apache2/certs/code.soundsoftware.ac.uk.ca-bundle + SSLVerifyClient none + SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM:+LOW + + DocumentRoot /var/www/code/public + PassengerRestartDir restart_files + PassengerHighPerformance on + PassengerMaxRequests 50000 + PassengerStatThrottleRate 5 + PassengerStartTimeout 60 + PassengerFriendlyErrorPages off + RailsSpawnMethod smart + ExpiresDefault "access plus 1 minute" + + + AuthType Basic + AuthUserFile "/etc/apache2/auth/user.htpasswd" + AuthName "code.soundsoftware.ac.uk" + Require user user + + + + AuthType Digest + AuthUserFile "/etc/apache2/auth/admin.htdigest" + AuthName "code.soundsoftware.ac.uk admin interface" + Require user admin + + + + Require all denied + + + + Require all denied + + + + Require all denied + + + + Options -MultiViews + + + + # Avoid other sites embedding our fonts + RewriteEngine on + RewriteCond %{HTTP_REFERER} !^$ + RewriteCond %{HTTP_REFERER} !^http(s)?://code.soundsoftware.ac.uk/.*$ [NC] + RewriteRule \.(ttf|woff|eot|otf|svg|zip|gz|html|txt)$ - [F] + + + ScriptAlias /hg "/var/hg/index.cgi" + + + AuthName "Mercurial" + AuthType Basic + Require valid-user + PerlAccessHandler Apache::Authn::SoundSoftware::access_handler + PerlAuthenHandler Apache::Authn::SoundSoftware::authen_handler + PerlSetVar HTTPS "on" + SoundSoftwareDSN "dbi:Pg:database=code;host=localhost" + SoundSoftwareDbUser "code" + SoundSoftwareDbPass "INSERT_DATABASE_PASSWORD_HERE" + SoundSoftwareRepoPrefix "/var/hg/" + SoundSoftwareSslRequired "on" + Options +ExecCGI + AddHandler cgi-script .cgi + ExpiresDefault now + + + Alias /git "/var/files/git-mirror" + + + Options -Indexes +FollowSymLinks + Require all granted + + + Require all denied + + + Require all denied + + + ErrorLog /var/log/apache2/code-error.log + CustomLog /var/log/apache2/code-access.log vhost_combined + + LogLevel warn + ServerSignature Off + + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/code.conf.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/code.conf.in Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,102 @@ + +# A test Apache config. Lacks SSL, lacks a desirable extra layer of +# authentication for admin interface paths. Do not deploy this. + +# Note this has been updated for Apache 2.4, which introduced a number +# of (welcome) changes to access control directives. + +PerlLoadModule Apache::Authn::SoundSoftware + + + ServerName code.soundsoftware.ac.uk + ServerAdmin chris.cannam@soundsoftware.ac.uk + + DocumentRoot /var/www/code/public + PassengerRestartDir restart_files + PassengerHighPerformance on + PassengerMaxRequests 50000 + PassengerStatThrottleRate 5 + PassengerStartTimeout 60 + PassengerFriendlyErrorPages on + RailsSpawnMethod smart + ExpiresDefault "access plus 1 minute" + +# +# AuthType Basic +# AuthUserFile "/etc/apache2/auth/user.htpasswd" +# AuthName "code.soundsoftware.ac.uk" +# Require user user +# + +# +# AuthType Digest +# AuthUserFile "/etc/apache2/auth/admin.htdigest" +# AuthName "code.soundsoftware.ac.uk admin interface" +# Require user admin +# + + + Require all denied + + + + Require all denied + + + + Require all denied + + + + Options -MultiViews + + + + # Avoid other sites embedding our fonts + RewriteEngine on + RewriteCond %{HTTP_REFERER} !^$ + RewriteCond %{HTTP_REFERER} !^http(s)?://code.soundsoftware.ac.uk/.*$ [NC] + RewriteRule \.(ttf|woff|eot|otf|svg|zip|gz|html|txt)$ - [F] + + + ScriptAlias /hg "/var/hg/index.cgi" + + + AuthName "Mercurial" + AuthType Basic + Require valid-user + PerlAccessHandler Apache::Authn::SoundSoftware::access_handler + PerlAuthenHandler Apache::Authn::SoundSoftware::authen_handler + PerlSetVar HTTPS "off" + SoundSoftwareDSN "dbi:Pg:database=code;host=localhost" + SoundSoftwareDbUser "code" + SoundSoftwareDbPass "INSERT_DATABASE_PASSWORD_HERE" + SoundSoftwareRepoPrefix "/var/hg/" + #!!! "on" in production please!: + SoundSoftwareSslRequired "off" + Options +ExecCGI + AddHandler cgi-script .cgi + ExpiresDefault now + + + Alias /git "/var/files/git-mirror" + + + Options -Indexes +FollowSymLinks + Require all granted + + + Require all denied + + + Require all denied + + + ErrorLog /var/log/apache2/code-error.log + CustomLog /var/log/apache2/code-access.log vhost_combined + + LogLevel warn + ServerSignature Off + + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.daily/00-backup-db --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.daily/00-backup-db Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,10 @@ +#!/bin/sh + +outfile="/var/files/backups/postgres-dumpall-`date +%Y%m%d%H%M`" + +oldmask=`umask` +umask 0277 + +su postgres -c /usr/bin/pg_dumpall > "$outfile" && bzip2 "$outfile" + +umask "$oldmask" diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.daily/10-extract-docs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.daily/10-extract-docs Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,6 @@ +#!/bin/bash + +cd /tmp + +/var/www/code/docgen/extract-docs.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.daily/15-get-statistics --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.daily/15-get-statistics Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo -u code sh -c "cd /var/www/code ; ./script/rails runner -e production extra/soundsoftware/get-statistics.rb >> log/statistics.log" + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.daily/20-check-end-of-external-repo-log --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.daily/20-check-end-of-external-repo-log Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,3 @@ +#!/bin/bash + +tail -2 /var/log/external-repos.log diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.hourly/00-drupal-cron --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.hourly/00-drupal-cron Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,6 @@ +#!/bin/bash + +## No longer used - this site is now static + +# /usr/bin/wget -O - -q -t 1 http://www.soundsoftware.ac.uk/cron.php + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.hourly/10-redmine-fetch-changesets --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.hourly/10-redmine-fetch-changesets Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo -u code sh -c "cd /var/www/code ; ./script/rails runner \"Repository.fetch_changesets\" -e production 2>&1 | grep -v 'Not trusting' | grep -v 'svn:' | grep -v 'working copy' | grep -v 'deprecated' | grep -v 'version_requirements'" +exit 0 + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.hourly/20-convert-external-repos --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.hourly/20-convert-external-repos Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo -H -u www-data /var/www/code/reposman/run-external.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.hourly/30-expire-explore-cache --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.hourly/30-expire-explore-cache Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,3 @@ +#!/bin/sh + +rm -f /var/www/code/tmp/cache/*/*/views*explore* diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.hourly/40-export-git --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.hourly/40-export-git Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,6 @@ +#!/bin/bash + +logfile="/var/www/code/log/export-git.log" + +sudo -u code sh -c "cd /tmp ; /var/www/code/extra/soundsoftware/export-git.sh production /var/hg /var/files/git-mirror >> $logfile 2>&1" + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.minutely/00-redmine-repositories --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.minutely/00-redmine-repositories Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo -u www-data /var/www/code/reposman/run-reposman.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/cron.monthly/00-backup-files --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/cron.monthly/00-backup-files Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,10 @@ +#!/bin/sh +cd / +for location in var/www etc/apache2 etc/cron.*; do + target="/var/files/backups/`echo $location | sed 's,/,_,g'`-`date +%Y%m%d%H%M`" + oldmask=`umask` + umask 0277 + tar cjf "$target".tar.bz2 "$location" + umask "$oldmask" +done + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/database.yml.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/database.yml.in Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,7 @@ +production: + adapter: postgresql + database: code + host: localhost + username: code + password: "INSERT_DATABASE_PASSWORD_HERE" + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/hgweb.config --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/hgweb.config Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,6 @@ +[paths] +/ = /var/hg/* + +[web] +allow_archive = gz, zip, bz2 +allow_push = * diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/index.cgi --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/index.cgi Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# An example CGI script to export multiple hgweb repos, edit as necessary + +# adjust python path if not a system-wide install: +#import sys +#sys.path.insert(0, "/path/to/python/lib") + +# enable importing on demand to reduce startup time +from mercurial import demandimport; demandimport.enable() + +# Uncomment to send python tracebacks to the browser if an error occurs: +import cgitb +cgitb.enable() + +# If you'd like to serve pages with UTF-8 instead of your default +# locale charset, you can do so by uncommenting the following lines. +# Note that this will cause your .hgrc files to be interpreted in +# UTF-8 and all your repo files to be displayed using UTF-8. +# +import os +os.environ["HGENCODING"] = "UTF-8" + +from mercurial.hgweb.hgwebdir_mod import hgwebdir +import mercurial.hgweb.wsgicgi as wsgicgi + +# The config file looks like this. You can have paths to individual +# repos, collections of repos in a directory tree, or both. +# +# [paths] +# virtual/path1 = /real/path1 +# virtual/path2 = /real/path2 +# virtual/root = /real/root/* +# / = /real/root2/* +# virtual/root2 = /real/root2/** +# +# [collections] +# /prefix/to/strip/off = /root/of/tree/full/of/repos +# +# paths example: +# +# * First two lines mount one repository into one virtual path, like +# '/real/path1' into 'virtual/path1'. +# +# * The third entry mounts every mercurial repository found in '/real/root' +# in 'virtual/root'. This format is preferred over the [collections] one, +# since using absolute paths as configuration keys is not supported on every +# platform (especially on Windows). +# +# * The fourth entry is a special case mounting all repositories in +# /'real/root2' in the root of the virtual directory. +# +# * The fifth entry recursively finds all repositories under the real root, +# and mounts them using their relative path (to given real root) under the +# virtual root. +# +# collections example: say directory tree /foo contains repos /foo/bar, +# /foo/quux/baz. Give this config section: +# [collections] +# /foo = /foo +# Then repos will list as bar and quux/baz. +# +# Alternatively you can pass a list of ('virtual/path', '/real/path') tuples +# or use a dictionary with entries like 'virtual/path': '/real/path' + +application = hgwebdir('hgweb.config') +wsgicgi.launch(application) diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/logrotate.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/logrotate.conf Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,69 @@ +# see "man logrotate" for details +# rotate log files weekly +weekly + +# use the syslog group by default, since this is the owning group +# of /var/log/syslog. +su root syslog + +# keep 4 weeks worth of backlogs +rotate 4 + +# create new (empty) log files after rotating old ones +create + +# uncomment this if you want your log files compressed +#compress + +# packages drop log rotation information into this directory +include /etc/logrotate.d + +# no packages own wtmp, or btmp -- we'll rotate them here +/var/log/wtmp { + missingok + monthly + create 0664 root utmp + rotate 1 +} + +/var/log/btmp { + missingok + monthly + create 0660 root utmp + rotate 1 +} + +# system-specific logs may be configured here +/var/www/code/log/*.log { + weekly + missingok + rotate 52 + compress + delaycompress + create 640 code code + sharedscripts + postrotate + touch /var/www/code/restart_files/restart.txt + endscript +} + +/var/log/reposman.log { + weekly + missingok + rotate 52 + compress + delaycompress + create 640 www-data code + sharedscripts +} + +/var/log/external-repos.log { + weekly + missingok + rotate 52 + compress + delaycompress + create 640 www-data code + sharedscripts +} + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/passenger.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/passenger.conf Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,2 @@ +PassengerMaxPoolSize 60 + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/passenger.load --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/passenger.load Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,3 @@ +LoadModule passenger_module /var/lib/gems/2.3.0/gems/passenger-4.0.60/buildout/apache2/mod_passenger.so +PassengerRoot /var/lib/gems/2.3.0/gems/passenger-4.0.60 +PassengerDefaultRuby /usr/bin/ruby2.3 diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/perl.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/perl.conf Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,4 @@ +# Apache::DBI is supposed to be a transparent replacement for Perl DBI with +# better performance when multiple connections are made with common DSN, user +# and password +PerlModule Apache::DBI diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/run-external.sh.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/run-external.sh.in Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,18 @@ +#!/bin/bash +logfile=/var/log/external-repos.log +( +flock -s 200 +echo >> $logfile +date >> $logfile +/var/www/code/reposman/convert-external-repos.rb \ + -s /var/hg \ + -r INSERT_API_SCHEME_HERE://INSERT_API_HOST_HERE/ \ + -k INSERT_API_KEY_HERE \ + -v \ + --http-user=INSERT_API_USER_HERE \ + --http-pass=INSERT_API_PASSWORD_HERE \ + -c "/var/www/code/reposman/update-external-repo.sh" \ + >> $logfile 2>&1 +date >> $logfile +echo Done >> $logfile +)200>>$logfile diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/run-reposman.sh.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/run-reposman.sh.in Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,17 @@ +#!/bin/bash +logfile=/var/log/reposman.log +( +flock -s 200 +echo >> $logfile +/var/www/code/reposman/reposman-soundsoftware.rb \ + -s /var/hg \ + -r INSERT_API_SCHEME_HERE://INSERT_API_HOST_HERE/ \ + -k INSERT_API_KEY_HERE \ + --http-user=INSERT_API_USER_HERE \ + --http-pass=INSERT_API_PASSWORD_HERE \ + -o www-data \ + -g code \ + -c "/var/www/code/reposman/run-hginit.sh" \ + --scm=Mercurial \ + >> $logfile 2>&1 +)200>>$logfile diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/config/soundsoftware-static.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/config/soundsoftware-static.conf Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,56 @@ + + ServerName soundsoftware.ac.uk + ServerAlias www.soundsoftware.ac.uk + ServerAdmin chris.cannam@eecs.qmul.ac.uk + + DocumentRoot /var/www/soundsoftware-static/soundsoftware.ac.uk + + ErrorLog /var/log/apache2/soundsoftware-error.log + CustomLog /var/log/apache2/soundsoftware-access.log vhost_combined + + + RewriteEngine on + RewriteCond %{REQUEST_FILENAME} !-d + RewriteCond %{REQUEST_FILENAME}\.html -f + RewriteRule ^(.*)$ $1.html + + + + Order Deny,Allow + deny from all + + + + Order allow,deny + Deny from all + Satisfy All + + + LogLevel warn + ServerSignature Off + + + + # We don't serve SSL: redirect to the code site + ServerName soundsoftware.ac.uk + ServerAlias www.soundsoftware.ac.uk + ServerAdmin chris.cannam@eecs.qmul.ac.uk + + SSLEngine on + SSLCertificateFile /etc/apache2/certs/code.soundsoftware.ac.uk.crt + SSLCertificateKeyFile /etc/apache2/certs/code.soundsoftware.ac.uk.key + SSLCertificateChainFile /etc/apache2/certs/code.soundsoftware.ac.uk.ca-bundle + SSLVerifyClient none + SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM:+LOW + + DocumentRoot /var/www/soundsoftware-static/soundsoftware.ac.uk + + Redirect permanent / https://code.soundsoftware.ac.uk/ + + ErrorLog /var/log/apache2/code-error.log + CustomLog /var/log/apache2/code-access.log vhost_combined + + LogLevel warn + ServerSignature Off + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/docker/Dockerfile.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/docker/Dockerfile.in Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,20 @@ + +FROM ubuntu:16.04 +MAINTAINER Chris Cannam + +COPY . /var/www/code + +WORKDIR /var/www/code + +INSERT_PROVISIONING_HERE + +# Start Postgres and foregrounded Apache + +RUN echo "#!/bin/bash" > container-run.sh +RUN echo "/etc/init.d/postgresql start" >> container-run.sh +RUN echo "apache2ctl -D FOREGROUND" >> container-run.sh +RUN chmod +x container-run.sh + +EXPOSE 80 +CMD ./container-run.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/docker/Dockerfile.inline --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/docker/Dockerfile.inline Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,139 @@ + +# For documentation and experimental purposes only. As a +# reconstruction of the machine image that runs this application, +# there are lots of things missing here; but as a good Docker +# configuration, it fails by mixing together rather a lot of concerns. + +FROM ubuntu:16.04 +MAINTAINER Chris Cannam + +RUN apt-get update && \ + apt-get install -y \ + apache2 \ + apache2-dev \ + apt-utils \ + build-essential \ + cron \ + curl \ + doxygen \ + exim4 \ + git \ + graphviz \ + imagemagick \ + libapache-dbi-perl \ + libapache2-mod-perl2 \ + libapr1-dev \ + libaprutil1-dev \ + libauthen-simple-ldap-perl \ + libcurl4-openssl-dev \ + libdbd-pg-perl \ + libpq-dev \ + libmagickwand-dev \ + libio-socket-ssl-perl \ + logrotate \ + mercurial \ + postgresql \ + rsync \ + ruby \ + ruby-dev \ + sudo + +# Also used on the live site, for javadoc extraction, but this is +# would be by far the biggest package here: let's omit it while we're +# not making use of it +# openjdk-9-jdk-headless + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + + +# Passenger gets installed through gem, not apt + +RUN gem install passenger -v 4.0.60 --no-rdoc --no-ri +RUN passenger-install-apache2-module --languages=ruby + + +# Copy across webapp, set up ownership + +COPY . /var/www/code + +RUN groupadd code +RUN useradd -g code -G www-data code +RUN chown -R code.www-data /var/www/code +RUN find /var/www/code -type d -exec chmod g+s \{\} \; + + +# Initialise /var/hg (in reality this would be mounted from somewhere) + +RUN mkdir -p /var/hg +RUN chown code.www-data /var/hg +RUN chmod g+s /var/hg +COPY extra/soundsoftware/scripted-deploy/config/index.cgi /var/hg/ +COPY extra/soundsoftware/scripted-deploy/config/hgweb.config /var/hg/ +RUN chmod +x /var/hg/index.cgi + + +# We're based in the code webapp directory from here on + +WORKDIR /var/www/code + + +# Set up database config etc + +RUN cp extra/soundsoftware/scripted-deploy/config/database.yml.interpolated config/database.yml + + +# Install Rails and dependencies (database.yml must be populated before this) + +RUN gem install bundler +RUN bundle install + + +# Initialise Redmine token (bundler must be installed before this) + +RUN bundle exec rake generate_secret_token + + +# Import Postgres database from postgres-dumpall file + +RUN chown postgres postgres-dumpall +RUN /etc/init.d/postgresql start && sudo -u postgres psql -f postgres-dumpall postgres +RUN rm postgres-dumpall + + +# Install Perl auth module for Hg access + +RUN mkdir -p /usr/local/lib/site_perl/Apache/Authn/ +RUN cp extra/soundsoftware/SoundSoftware.pm /usr/local/lib/site_perl/Apache/Authn/ + + +# Set up Apache config (todo: insert variables) + +RUN rm -f /etc/apache2/sites-enabled/000-default.conf + +RUN cp extra/soundsoftware/scripted-deploy/config/passenger.conf /etc/apache2/mods-available/ +RUN cp extra/soundsoftware/scripted-deploy/config/passenger.load /etc/apache2/mods-available/ +RUN cp extra/soundsoftware/scripted-deploy/config/perl.conf /etc/apache2/mods-available/ + +RUN ln -s ../mods-available/passenger.conf /etc/apache2/mods-enabled/ +RUN ln -s ../mods-available/passenger.load /etc/apache2/mods-enabled/ +RUN ln -s ../mods-available/perl.conf /etc/apache2/mods-enabled/ +RUN ln -s ../mods-available/expires.load /etc/apache2/mods-enabled/ +RUN ln -s ../mods-available/rewrite.load /etc/apache2/mods-enabled/ +RUN ln -s ../mods-available/cgi.load /etc/apache2/mods-enabled/ + +RUN cp extra/soundsoftware/scripted-deploy/config/code.conf.interpolated /etc/apache2/sites-available/code.conf +RUN ln -s ../sites-available/code.conf /etc/apache2/sites-enabled/10-code.conf + +RUN apache2ctl configtest + + +# Start Postgres and foregrounded Apache + +RUN echo "#!/bin/bash" > container-run.sh +RUN echo "/etc/init.d/postgresql start" >> container-run.sh +RUN echo "apache2ctl -D FOREGROUND" >> container-run.sh +RUN chmod +x container-run.sh + +EXPOSE 80 +CMD ./container-run.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/docker/start.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/docker/start.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,26 @@ +#!/bin/bash + +mydir=$(dirname "$0") +. "$mydir"/../any/prepare.sh + +provisioning_commands=$( + for x in "$deploydir"/provision.d/[0-9]*.sh; do + echo "RUN /bin/bash /var/www/code/deploy/provision.d/$(basename $x)" + done | sed 's/$/\\n/' | fmt -2000 | sed 's/ RUN/RUN/g' ) + +( echo + echo "### DO NOT EDIT THIS FILE - it is generated from Dockerfile.in" + echo +) > "$managerdir/Dockerfile" + +cat "$managerdir/Dockerfile.in" | + sed 's,INSERT_PROVISIONING_HERE,'"$provisioning_commands"',' >> \ + "$managerdir/Dockerfile.gen" + +cd "$rootdir" + +dockertag="cannam/soundsoftware-site" + +sudo docker build -t "$dockertag" -f "deploy/docker/Dockerfile.gen" . +sudo docker run -p 8080:80 -d "$dockertag" + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/000-system-packages.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/000-system-packages.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e + +# Install necessary system packages. This assumes we are deploying on +# Ubuntu 16.04. + +# We aim to make all of these provisioning scripts non-destructive if +# run more than once. In this case, running the script again will +# install any outstanding updates. + +apt-get update && \ + apt-get dist-upgrade -y && \ + apt-get install -y \ + ack-grep \ + apache2 \ + apache2-dev \ + apt-utils \ + build-essential \ + cron \ + curl \ + doxygen \ + exim4 \ + git \ + graphviz \ + imagemagick \ + libapache-dbi-perl \ + libapache2-mod-perl2 \ + libapr1-dev \ + libaprutil1-dev \ + libauthen-simple-ldap-perl \ + libcurl4-openssl-dev \ + libdbd-pg-perl \ + libpq-dev \ + libmagickwand-dev \ + libio-socket-ssl-perl \ + logrotate \ + lynx \ + mercurial \ + mercurial-git \ + openjdk-9-jdk-headless \ + postgresql \ + rsync \ + ruby \ + ruby-dev \ + sudo + +locale-gen en_US.UTF-8 + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/010-passenger.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/010-passenger.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Phusion Passenger as application server. +# This gets installed through gem, not apt, and we ask for a specific +# version (the last in the 4.0.x line). + +if [ ! -f /var/lib/gems/2.3.0/gems/passenger-4.0.60/buildout/apache2/mod_passenger.so ]; then + gem install passenger -v 4.0.60 --no-rdoc --no-ri + passenger-install-apache2-module --languages=ruby +fi + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/020-users.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/020-users.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# The webapp directory is owned and run by the code user, in group +# www-data. The repos and other things served directly are the other +# way around -- owned by the www-data user, in group code. + +for user in code docgen ; do + if ! grep -q "^$user:" /etc/passwd ; then + groupadd "$user" + useradd -g "$user" -G www-data "$user" + fi +done + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/030-webapp-dir.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/030-webapp-dir.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +# We might be running in one of two ways: +# +# 1. The code directory is already at /var/www/code, either because a +# previous provisioning step has imported it there or because this +# script has been run before -- in this situation all we do is +# re-check the ownership and permissions. OR +# +# 2. The code directory has not yet been copied to /var/www/code, in +# which case we expect to find it at /code-to-deploy, e.g. as a +# Vagrant shared folder, and we copy it over from there. (We don't +# deploy directly from shared folders as we might not be able to +# manipulate ownership and permissions properly there.) + +if [ ! -d /var/www/code ]; then + if [ ! -d /code-to-deploy ]; then + echo "ERROR: Expected to find code tree at /var/www/code or /code-to-deploy: is the deployment script being invoked correctly?" + exit 2 + fi + cp -a /code-to-deploy /var/www/code +fi + +chown -R code.www-data /var/www/code +chmod 755 /var/www/code +find /var/www/code -type d -exec chmod g+s \{\} \; + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/040-hg-dir.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/040-hg-dir.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# In a real deployment, /var/hg is probably mounted from somewhere +# else. But in an empty deployment we need to create it, and in both +# cases we set up the config files with their current versions here. + +if [ ! -f /var/hg/index.cgi ]; then + mkdir -p /var/hg +fi + +cp /var/www/code/deploy/config/index.cgi /var/hg/ +cp /var/www/code/deploy/config/hgweb.config /var/hg/ + +chmod +x /var/hg/index.cgi + +chown -R www-data.code /var/hg +find /var/hg -type d -exec chmod g+s \{\} \; + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/050-webapp-db.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/050-webapp-db.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +# Copy across the database config file (the source file has presumably +# been generated from a skeleton, earlier in provisioning) + +infile=/var/www/code/deploy/config/database.yml.gen +outfile=/var/www/code/config/database.yml + +if [ ! -f "$outfile" ]; then + if [ ! -f "$infile" ]; then + echo "ERROR: Database config file $infile not found - has the database secret been interpolated from $infile.in correctly?" + exit 2 + fi + cp "$infile" "$outfile" +fi + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/060-bundler.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/060-bundler.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# Install Ruby gems for the web app. + +# We aim to make all of these provisioning scripts non-destructive if +# run more than once. In this case, running the script again will +# install any outstanding updates. + +cd /var/www/code +gem install bundler +bundle install + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/070-secret-token.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/070-secret-token.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Create a session token if it hasn't already been created. + +cd /var/www/code + +if [ ! -f config/initializers/secret_token.rb ]; then + bundle exec rake generate_secret_token +fi + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/080-database-load.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/080-database-load.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Start the database and if a dump file is found, load it. The dump +# file is then deleted so that the db won't be overwritten on +# subsequent runs. (The original repo contains no dump file, so it +# should exist only if you have provided some data to load.) + +/etc/init.d/postgresql start + +dumpdir="/code-to-deploy" +if [ ! -d "$dumpdir" ]; then + dumpdir=/var/www/code +fi + +cd "$dumpdir" + +if [ -f postgres-dumpall ]; then + chmod ugo+r postgres-dumpall + sudo -u postgres psql -f postgres-dumpall postgres + rm postgres-dumpall +fi + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/090-perl-auth-module.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/090-perl-auth-module.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Install the Apache mod_perl module used for hg repo access control + +if [ ! -f /usr/local/lib/site_perl/Apache/Authn/SoundSoftware.pm ]; then + mkdir -p /usr/local/lib/site_perl/Apache/Authn/ + cp /var/www/code/extra/soundsoftware/SoundSoftware.pm \ + /usr/local/lib/site_perl/Apache/Authn/ +fi + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/100-apache-config.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/100-apache-config.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +# Install Apache config files and module loaders + +cd /var/www/code + +codeconf=/var/www/code/deploy/config/code.conf.gen +codeconfssl=/var/www/code/deploy/config/code-ssl.conf.gen +staticconf=/var/www/code/deploy/config/soundsoftware-static.conf + +if [ ! -f "$codeconf" ]; then + echo "ERROR: Apache config file $codeconf not found - has the database secret been interpolated from its input file correctly?" + exit 2 +fi + +if [ ! -f /etc/apache2/sites-enabled/10-code.conf ]; then + + rm -f /etc/apache2/sites-enabled/000-default.conf + + cp deploy/config/passenger.conf /etc/apache2/mods-available/ + cp deploy/config/passenger.load /etc/apache2/mods-available/ + cp deploy/config/perl.conf /etc/apache2/mods-available/ + + ln -s ../mods-available/passenger.conf /etc/apache2/mods-enabled/ + ln -s ../mods-available/passenger.load /etc/apache2/mods-enabled/ + ln -s ../mods-available/perl.conf /etc/apache2/mods-enabled/ + ln -s ../mods-available/expires.load /etc/apache2/mods-enabled/ + ln -s ../mods-available/rewrite.load /etc/apache2/mods-enabled/ + ln -s ../mods-available/cgi.load /etc/apache2/mods-enabled/ + ln -s ../mods-available/ssl.load /etc/apache2/mods-enabled/ + ln -s ../mods-available/auth_digest.load /etc/apache2/mods-enabled/ + + cp "$codeconf" /etc/apache2/sites-available/code.conf + cp "$codeconfssl" /etc/apache2/sites-available/code-ssl.conf + cp "$staticconf" /etc/apache2/sites-available/soundsoftware-static.conf + ln -s ../sites-available/code.conf /etc/apache2/sites-enabled/10-code.conf + + apache2ctl configtest + +fi + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/110-hg-testdir.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/110-hg-testdir.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# In case we are running without a properly mounted /var/hg directory, +# check for the existence of one repo and, if absent, attempt to clone +# it so that we have something we can serve for test purposes. + +if [ ! -d /var/hg/vamp-plugin-sdk ]; then + echo "Cloning vamp-plugin-sdk repo for testing..." + cd /var/hg + hg clone https://code.soundsoftware.ac.uk/hg/vamp-plugin-sdk + chown -R www-data.code vamp-plugin-sdk +fi diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/115-other-dirs.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/115-other-dirs.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# Initialise directories used as targets for cron activity (if they +# don't already exist) + +# Reminder: the webapp directory is owned and run by the code user, in +# group www-data. The repos and other things served directly are +# usually the other way around -- owned by the www-data user, in group +# code. I don't recall whether there is a good reason for this. + +for dir in \ + /var/files/backups \ + /var/doc \ + /var/files/code \ + /var/files/git-mirror ; do + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + chown -R code.www-data "$dir" + chmod g+s "$dir" + fi +done + +for dir in \ + /var/mirror ; do + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + chown -R www-data.code "$dir" + chmod g+s "$dir" + fi +done + +if [ ! -e /var/www/code/files ]; then + ln -s /var/files/code /var/www/code/files +fi diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/120-docgen.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/120-docgen.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Copy docgen scripts, including the generated scripts with +# interpolated API key etc, to the directory they will be run from. + +# These are run from cron jobs to do the (currently daily) update of +# extracted documentation from Doxygen, Javadoc, and MATLAB, and to +# enable displaying them with the redmine_embedded plugin. (The API +# key is needed to automatically switch on the embedded module for a +# project the first time its docs are extracted.) + +cd /var/www/code + +mkdir -p docgen + +for file in \ + doxysafe.pl \ + extract-doxygen.sh \ + extract-javadoc.sh \ + extract-matlabdocs.sh \ + matlab-docs.conf \ + matlab-docs-credit.html \ + matlab-docs.pl ; do + if [ ! -f docgen/"$file" ]; then + cp extra/soundsoftware/"$file" docgen/ + fi +done + +for file in \ + extract-docs.sh ; do + if [ ! -f docgen/"$file" ]; then + cp deploy/config/"$file".gen docgen/"$file" + fi +done + +chown code.www-data docgen/* +chmod +x docgen/*.sh + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/130-reposman.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/130-reposman.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +# Copy reposman (repository manager) scripts, including the generated +# scripts with interpolated API key etc, to the directory they will be +# run from. + +# There are two sets of scripts here: +# +# 1. The reposman script that plods through all the projects that have +# repositories defined, creates those repositories on disc, and +# registers their locations with the projects. This happens often, +# currently every minute. +# +# 2. The external repo management script that plods through all the +# projects that have external repositories defined, clones or updates +# those external repos to their local locations, and if necessary +# registers them with the projects. This happens less often, currently +# every hour. + +cd /var/www/code + +mkdir -p reposman + +for file in \ + convert-external-repos.rb \ + reposman-soundsoftware.rb \ + run-hginit.sh \ + update-external-repo.sh ; do + if [ ! -f reposman/"$file" ]; then + cp extra/soundsoftware/"$file" reposman/ + fi +done + +for file in \ + run-external.sh \ + run-reposman.sh ; do + if [ ! -f reposman/"$file" ]; then + cp deploy/config/"$file".gen reposman/"$file" + fi +done + +chown code.www-data reposman/* +chmod +x reposman/*.sh +chmod +x reposman/*.rb + +touch /var/log/reposman.log +touch /var/log/external-repos.log +chown www-data.code /var/log/reposman.log +chown www-data.code /var/log/external-repos.log + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/140-cron.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/140-cron.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Copy cron scripts to the appropriate destinations + +cd /var/www/code + +if [ ! -d /etc/cron.minutely ]; then + mkdir -p /etc/cron.minutely + echo '* * * * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.minutely )' >> /etc/crontab +fi + +for t in minutely hourly daily monthly; do + for s in deploy/config/cron.$t/[0-9]* ; do + name=$(basename $s) + dest="/etc/cron.$t/$name" + if [ ! -f "$dest" ]; then + cp "$s" "$dest" + chmod +x "$dest" + fi + done +done + + + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/150-logrotate.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/150-logrotate.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,4 @@ +#!/bin/bash + +cd /var/www/code +cp deploy/config/logrotate.conf /etc/logrotate.conf diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/provision.d/190-reminders.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/provision.d/190-reminders.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +# Print reminders of the things that we haven't covered in the deploy +# scripts + +cat <&2 + echo "Usage: $0 " 1>&2 + echo 1>&2 + echo " e.g. $0 https://code.soundsoftware.ac.uk" 1>&2 + echo " or $0 http://localhost:8080" 1>&2 + echo 1>&2 + exit 2 +} + +uribase="$1" +if [ -z "$uribase" ]; then + usage +fi + +set -eu + +# A project known to exist, be public, and have a repository +project_with_repo=vamp-plugin-sdk + +# A project known to exist, be public, and have embedded documentation +project_with_docs=vamp-plugin-sdk + +# A project known to exist, be public, and have a bibliography +project_with_biblio=sonic-visualiser + +# A project known not to exist +nonexistent_project=nonexistent-project + +# A file for download known to exist +file_for_download=/attachments/download/2210/vamp-plugin-sdk-2.7.1-binaries-osx.tar.gz + +tried=0 +succeeded=0 + +mydir=$(dirname "$0") + +try() { + mkdir -p "$mydir/output" + origin=$(pwd) + cd "$mydir/output" + path="$1" + description="$2" + expected="$3" + url="$uribase$path" + echo + echo "Trying \"$description\" [$url]..." + echo + if wget "$url" ; then + echo "+++ Succeeded" + succeeded=$(($succeeded + 1)) + else + returned="$?" + if [ "$returned" = "$expected" ]; then + echo "+++ Succeeded [returned expected code $expected]" + succeeded=$(($succeeded + 1)) + else + echo "--- FAILED with return code $returned" + fi + fi + tried=$(($tried + 1)) + cd "$origin" +} + +assert() { + try "$1" "$2" 0 +} + +fail() { + try "$1" "$2" "$3" +} + +assert "/" "Front page" +assert "/projects/$project_with_repo" "Project page" +assert "/projects/$project_with_biblio" "Project page with bibliography" +assert "/projects/$project_with_repo/repository" "Repository page" +assert "/hg/$project_with_repo" "Mercurial repo" +assert "/projects/$project_with_docs/embedded" "Project documentation page (from docgen cron script)" +assert "/git/$project_with_repo/info/refs" "Git repo mirror" +assert "$file_for_download" "File for download" + +# we expect this to return an http auth requirement, not a 404 - the +# value 6 is wget's return code for auth failure +fail "/hg/$nonexistent_project" "Mercurial repo" 6 + +echo +echo "Passed $succeeded of $tried" +echo + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/vagrant/Vagrantfile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/vagrant/Vagrantfile Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,9 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "ubuntu/xenial64" + config.vm.network "forwarded_port", guest: 80, host: 8080 + config.vm.synced_folder "../..", "/code-to-deploy" + config.vm.provision :shell, path: "vagrant-provision.sh" +end diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/vagrant/start.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/vagrant/start.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,8 @@ +#!/bin/bash + +mydir=$(dirname "$0") +. "$mydir"/../any/prepare.sh + +cd "$managerdir" +vagrant up + diff -r bc47b68a9487 -r 96bcc0ebd102 deploy/vagrant/vagrant-provision.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/deploy/vagrant/vagrant-provision.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +for f in /code-to-deploy/deploy/provision.d/[0-9]*.sh ; do + case "$f" in + *~) ;; + *) echo "Running provisioning script: $f" + /bin/bash "$f";; + esac +done + +echo "All provisioning scripts complete" diff -r bc47b68a9487 -r 96bcc0ebd102 extra/soundsoftware/SoundSoftware.pm --- a/extra/soundsoftware/SoundSoftware.pm Tue Aug 01 13:30:33 2017 +0100 +++ b/extra/soundsoftware/SoundSoftware.pm Tue Sep 05 11:19:50 2017 +0100 @@ -32,7 +32,8 @@ Debian/ubuntu: apt-get install libapache-dbi-perl libapache2-mod-perl2 \ - libdbd-mysql-perl libauthen-simple-ldap-perl libio-socket-ssl-perl + libdbd-mysql-perl libdbd-pg-perl libio-socket-ssl-perl \ + libauthen-simple-ldap-perl Note that LDAP support is hardcoded "on" in this script (it is optional in the original Redmine.pm). @@ -179,7 +180,7 @@ my $method = $r->method; print STDERR "SoundSoftware.pm:$$: Method: $method, uri " . $r->uri . ", location " . $r->location . "\n"; - print STDERR "SoundSoftware.pm:$$: Accept: " . $r->headers_in->{Accept} . "\n"; +# print STDERR "SoundSoftware.pm:$$: Accept: " . $r->headers_in->{Accept} . "\n"; my $dbh = connect_database($r); unless ($dbh) { @@ -248,6 +249,16 @@ # case we can decide for certain to accept in this function print STDERR "SoundSoftware.pm:$$: Method is read-only, no restriction here\n"; $r->set_handlers(PerlAuthenHandler => [\&OK]); + if (!defined $r->user or $r->user eq '') { + # Apache 2.4+ requires auth module to set user if no + # auth was needed. Note that this actually tells + # apache that user has been identified, so authen + # handler will never be called (i.e. we must not do + # this unless we are actually approving the auth-free + # access). If we don't do this, we get a 500 error + # here after the set_handlers call above + $r->user('*anon*'); + } return OK; } diff -r bc47b68a9487 -r 96bcc0ebd102 extra/soundsoftware/extract-docs.sh --- a/extra/soundsoftware/extract-docs.sh Tue Aug 01 13:30:33 2017 +0100 +++ b/extra/soundsoftware/extract-docs.sh Tue Sep 05 11:19:50 2017 +0100 @@ -7,15 +7,17 @@ hgdir="/var/hg" docdir="/var/doc" -logfile="/var/www/test-cannam/log/extract-docs.log" +logfile="/var/www/code/log/extract-docs.log" -redgrp="redmine" +redgrp="code" -apikey="" -apischeme="https" -apihost="" -apiuser="" -apipass="" +apikey="INSERT_API_KEY_HERE" +apischeme="INSERT_API_SCHEME_HERE" +apihost="INSERT_API_HOST_HERE" + +# HTTP auth username/password for /sys api calls +apiuser="INSERT_API_USER_HERE" +apipass="INSERT_API_PASSWORD_HERE" progdir=$(dirname $0) case "$progdir" in diff -r bc47b68a9487 -r 96bcc0ebd102 extra/soundsoftware/run-hginit.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/extra/soundsoftware/run-hginit.sh Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,3 @@ +#!/bin/sh +location="$1" +hg init "$location" && mkdir "$location/.hg/store/data" diff -r bc47b68a9487 -r 96bcc0ebd102 extra/soundsoftware/update-external-repo.sh --- a/extra/soundsoftware/update-external-repo.sh Tue Aug 01 13:30:33 2017 +0100 +++ b/extra/soundsoftware/update-external-repo.sh Tue Sep 05 11:19:50 2017 +0100 @@ -125,7 +125,7 @@ "$hg" init "$local_repo" fi if [ -d "$project_repo_mirror/.git" ]; then - ( cd "$local_repo" && "$hg" --config extensions.hggit= pull "$project_repo_mirror" ) && echo "$remote_repo" > "$successfile" + ( cd "$local_repo" && "$hg" --config extensions.git= pull "$project_repo_mirror" ) && echo "$remote_repo" > "$successfile" else ( cd "$local_repo" && "$hg" pull "$project_repo_mirror" ) && echo "$remote_repo" > "$successfile" fi diff -r bc47b68a9487 -r 96bcc0ebd102 plugins/redmine_bibliography/Gemfile --- a/plugins/redmine_bibliography/Gemfile Tue Aug 01 13:30:33 2017 +0100 +++ b/plugins/redmine_bibliography/Gemfile Tue Sep 05 11:19:50 2017 +0100 @@ -1,3 +1,6 @@ gem 'bibtex-ruby' gem 'nokogiri' -gem 'citeproc-ruby' \ No newline at end of file +gem 'csl' +gem 'csl-styles' +gem 'citeproc' +gem 'citeproc-ruby' diff -r bc47b68a9487 -r 96bcc0ebd102 plugins/redmine_bibliography/app/models/publication.rb --- a/plugins/redmine_bibliography/app/models/publication.rb Tue Aug 01 13:30:33 2017 +0100 +++ b/plugins/redmine_bibliography/app/models/publication.rb Tue Sep 05 11:19:50 2017 +0100 @@ -101,7 +101,12 @@ end if style == :ieee - CiteProc.process(bib.to_citeproc, :style => :ieee, :format => :html) + cite = bib.to_citeproc + cite_id = cite["id"] + cp = CiteProc::Processor.new style: 'ieee', format: 'html' + cp.import [cite] + texts = cp.render :bibliography, id: cite_id + texts[0] else bibtex = bib.to_s :include => :meta_content bibtex.strip! diff -r bc47b68a9487 -r 96bcc0ebd102 plugins/redmine_bibliography/init.rb --- a/plugins/redmine_bibliography/init.rb Tue Aug 01 13:30:33 2017 +0100 +++ b/plugins/redmine_bibliography/init.rb Tue Sep 05 11:19:50 2017 +0100 @@ -2,6 +2,8 @@ require 'bibtex' require 'citeproc' +require 'citeproc/ruby' +require 'csl/styles' # Patches to the Redmine core. ActionDispatch::Callbacks.to_prepare do diff -r bc47b68a9487 -r 96bcc0ebd102 plugins/redmine_embedded/Gemfile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/plugins/redmine_embedded/Gemfile Tue Sep 05 11:19:50 2017 +0100 @@ -0,0 +1,1 @@ +gem 'iconv' diff -r bc47b68a9487 -r 96bcc0ebd102 public/stylesheets/application.css --- a/public/stylesheets/application.css Tue Aug 01 13:30:33 2017 +0100 +++ b/public/stylesheets/application.css Tue Sep 05 11:19:50 2017 +0100 @@ -515,6 +515,8 @@ p.pagination {margin-top:8px; font-size: 90%} +#browser-autoscroll { clear: right } + /***** Tabular forms ******/ .tabular p{ margin: 0;