SoundSoftware Code Site

    -exec egrep '^ *package +[a-zA-Z][a-zA-Z0-9\._-]*;.*$' \{\} /dev/null \; |

    sed -e 's/\/[^\/]*: *package */:/' -e 's/;.*$//' |

# Read an Apache log file in SoundSoftware site format from stdin and

# produce some per-project stats.

#

# Invoke with e.g.

#

# cat /var/log/apache2/code-access.log | \

#   script/runner -e production extra/soundsoftware/get-apache-log-stats.rb

# Use the ApacheLogRegex parser, a neat thing

# See http://www.simonecarletti.com/blog/2009/02/apache-log-regex-a-lightweight-ruby-apache-log-parser/

require 'apachelogregex'

# This is the format defined in our httpd.conf

vhost_combined_format = '%v:%p %h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"'

parser = ApacheLogRegex.new(vhost_combined_format)

# project name -> count of hg clones

clones = Hash.new(0)

# project name -> count of hg pulls

pulls = Hash.new(0)

# project name -> count of hg pushes

pushes = Hash.new(0)

# project name -> count of hg archive requests (i.e. Download as Zip)

zips = Hash.new(0)

# project name -> count of hits to pages under /projects/projectname

hits = Hash.new(0)

# project name -> Project object

@projects = Hash.new

parseable = 0

unparseable = 0

def is_public_project?(project)

  if !project

    false

  elsif project =~ /^\d+$/

    # ignore numerical project ids, they are only used when editing projects

    false

  elsif @projects.key?(project)

    @projects[project].is_public? 

  else

    pobj = Project.find_by_identifier(project)

    if pobj

      @projects[project] = pobj

      pobj.is_public?

    else

      print "Project not found: ", project, "\n"

      false

    end

  end

end

def print_stats(h)

  h.keys.sort { |a,b| h[b] <=> h[a] }.each do |p|

    if h[p] > 0

      print h[p], " ", @projects[p].name, " [", p, "]\n"

    end

  end

end

STDIN.each do |line|

  record = parser.parse(line)

  # most annoyingly, the parser can't handle the comma-separated list

  # in X-Forwarded-For where it has more than one element. If it has

  # failed, remove any IP addresses or the word "unknown" with

  # trailing commas and try again

  if not record

    filtered = line.gsub(/(unknown|([0-9]+\.){3}[0-9]+),\s*/, "")

    record = parser.parse(filtered)

  end

  # discard, but count, unparseable lines

  if not record

    print "Line not parseable: ", line, "\n"

    unparseable += 1

    next

  end

  # discard everything that isn't a 200 OK response

  next if record["%>s"] != "200"

  # discard anything apparently requested by a crawler

  next if record["%{User-Agent}i"] =~ /(bot|slurp|crawler|spider|Redmine)\b/i

  # pull out request e.g. GET / HTTP/1.0

  request = record["%r"]

  # split into method, path, protocol

  if not request =~ /^[^\s]+ ([^\s]+) [^\s]+$/

    print "Line not parseable (bad method, path, protocol): ", line, "\n"

    unparseable += 1

    next

  end

  # get the path e.g. /projects/weevilmatic and split on /

  path = $~[1]

  components = path.split("/")

  # should have at least two elements unless path is "/"; first should

  # be empty (begins with /)

  if path != "/" and (components.size < 2 or components[0] != "")

    print "Line not parseable (degenerate path): ", line, "\n"

    unparseable += 1

    next

  end

  if components[1] == "hg"

    # path is /hg/project?something or /hg/project/something

    project = components[2].split("?")[0]

    if not is_public_project?(project)

      next

    end

    if components[2] =~ /&roots=00*$/

      clones[project] += 1

    elsif components[2] =~ /cmd=capabilities/

      pulls[project] += 1

    elsif components[2] =~ /cmd=unbundle/

      pushes[project] += 1

    elsif components[3] == "archive"

      zips[project] += 1

    end

  elsif components[1] == "projects"

    # path is /projects/project or /projects/project/something

    project = components[2]

    project = project.split("?")[0] if project

    if not is_public_project?(project)

      next

    end

    hits[project] += 1

  end

  parseable += 1

end

# Each clone is also a pull; deduct it from the pulls hash, because we

# want that to contain only non-clone pulls

clones.keys.each do |project|

  pulls[project] -= 1

end

print parseable, " parseable\n"

print unparseable, " unparseable\n"

print "\nMercurial clones:\n"

print_stats clones

print "\nMercurial pulls (excluding clones):\n"

print_stats pulls

print "\nMercurial pushes:\n"

print_stats pushes

print "\nMercurial archive (zip file) downloads:\n"

print_stats zips

print "\nProject page hits (excluding crawlers):\n"

print_stats hits

# this script will get stats from the repo and print them to stdout

# USAGE: 

# ./script/runner -e production extra/soundsoftware/get-statistics.rb 

#

d1 = Date.parse("20100701") # => 1 Jul 2010

d2 = Date.today

def delta_array (iarray)

  # returns an array with the deltas

  ## prepends a zero and drops the last element

  deltas = [0] + iarray

  deltas = deltas.first(deltas.size - 1)

  return iarray.zip(deltas).map { |x, y| x - y }

end

def months_between(d1, d2)

   months = []

   start_date = Date.civil(d1.year, d1.month, 1)

   end_date = Date.civil(d2.year, d2.month, 1)

   raise ArgumentError unless d1 <= d2

   while (start_date < end_date)

     months << start_date

     start_date = start_date >>1

   end

   months << end_date

end

def weeks_between(d1, d2)

   weeks = []

   start_date = Date.civil(d1.year, d1.month, d1.day)

   end_date = Date.civil(d2.year, d2.month, d2.day)

   raise ArgumentError unless d1 <= d2

   while (start_date < end_date)

     weeks << start_date

     start_date = start_date + 2.week

   end

   weeks << end_date

end

def get_user_project_evol_stats()

  # dates = months_between(d1, d2)

  dates = months_between(d1, d2)

  # number of users 

  n_users = []

  n_projects = []

  qm_users = []

  dates.each do |date|

    users =  User.find_by_sql ["SELECT * FROM users WHERE users.status = '1' AND users.created_on <= ?;", date]

    projects =  Project.find_by_sql ["SELECT * FROM projects WHERE projects.created_on <= ?;", date]

    qm_users_list = User.find_by_sql ["SELECT * FROM users,ssamr_user_details WHERE users.status = '1' AND ssamr_user_details.user_id = users.id AND (users.mail like '%qmul%' OR ssamr_user_details.institution_id = '99') AND users.created_on <= ?;", date ]

    qm_users << qm_users_list.count

    n_users << users.count

    n_projects << projects.count

    #  private_projects =  Project.find(:all, :conditions => {:created_on  => d1..date, is_public => false})

  end

  user_deltas = delta_array(n_users)

  proj_deltas = delta_array(n_projects)

  qm_user_deltas = delta_array(qm_users)

  puts "Date Users D_Users QM_Users D_QM_users Projects D_Projects"

  dates.zip(n_users, user_deltas, qm_users, qm_user_deltas, n_projects, proj_deltas).each do |a, b, c, d, e, f, g|

    puts "#{a} #{b} #{c} #{d} #{e} #{f} #{g}"

  end

end

def get_project_status()

  date = "20121101"

   all_projects = Project.find(:all, :conditions => ["created_on < ?", date])

  #  all_projects = Project.find(:all, :conditions => ["is_public = ? AND created_on < ?", true, date])

#  all_projects = Project.find(:all, :conditions => ["is_public = ? AND created_on < ?", false, date])

  collab = []

  users_per_proj = []

  #  puts "Public Users Institutions"

  all_projects.each do |proj| 

    insts = []

    proj.users.each do |u|  

      if u.institution == "" || u.institution == "No Institution Set"

        if u.mail.include?("qmul.ac.uk") || u.mail.include?("andrewrobertson77")

          insts << "Queen Mary, University of London"          

        else

          insts << u.mail

        end

      else

        insts << u.institution

      end

    end

    users_per_proj << proj.users.count

    collab << insts.uniq.count

  end

  #  freq = collab.inject(Hash.new(0)) { |h,v| h[v] += 1; h }

  #  freq = freq.sort_by {|key, value| value}

  #  puts freq.inspect.sort

  puts "Projects: #{all_projects.count} UpP: #{users_per_proj.sum / users_per_proj.size.to_f} Users1+: #{users_per_proj.count{|x| x> 1}} Users2+: #{users_per_proj.count{|x| x> 2}} Collab1+: #{collab.count{|x| x > 1}} Collab2+: #{collab.count{|x| x > 2}} IpP: #{collab.sum / collab.size.to_f}"

end

def get_user_projects_ratios()

  user_projects = User.find(:all, :conditions=> {:status => 1})

  pub_proj_user = user_projects.map{|u| u.projects.find(:all, :conditions=>{:is_public => true}).count}

  user_projects.zip(pub_proj_user).each do |u, pub|

      puts "#{u.projects.count} #{pub}"

  end

end

def get_inst_list()

  users = User.find(:all, :conditions => {:status => 1})

  inst_list = users.map{|user| user.institution}

  freq = inst_list.inject(Hash.new(0)) { |h,v| h[v] += 1; h }

end

# get_user_projects_ratios()

# get_user_project_evol_stats()

get_project_status()