Chris@978: 
Chris@980: # Read an Apache log file in SoundSoftware site format from stdin and
Chris@980: # produce some per-project stats.
Chris@978: #
Chris@978: # Invoke with e.g.
Chris@978: #
Chris@978: # cat /var/log/apache2/code-access.log | \
Chris@978: #   script/runner -e production extra/soundsoftware/get-apache-log-stats.rb
Chris@978: 
Chris@975: 
Chris@975: # Use the ApacheLogRegex parser, a neat thing
Chris@975: # See http://www.simonecarletti.com/blog/2009/02/apache-log-regex-a-lightweight-ruby-apache-log-parser/
Chris@975: require 'apachelogregex'
Chris@975: 
Chris@975: # This is the format defined in our httpd.conf
Chris@975: vhost_combined_format = '%v:%p %h %{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"'
Chris@975: 
Chris@975: parser = ApacheLogRegex.new(vhost_combined_format)
Chris@975: 
Chris@975: # project name -> count of hg clones
Chris@975: clones = Hash.new(0)
Chris@975: 
Chris@975: # project name -> count of hg pulls
Chris@975: pulls = Hash.new(0)
Chris@975: 
Chris@978: # project name -> count of hg pushes
Chris@978: pushes = Hash.new(0)
Chris@975: 
Chris@975: # project name -> count of hg archive requests (i.e. Download as Zip)
Chris@975: zips = Hash.new(0)
Chris@975: 
Chris@975: # project name -> count of hits to pages under /projects/projectname
Chris@975: hits = Hash.new(0)
Chris@975: 
Chris@978: # project name -> Project object
Chris@978: @projects = Hash.new
Chris@978: 
Chris@975: parseable = 0
Chris@975: unparseable = 0
Chris@975: 
Chris@979: def is_public_project?(project)
Chris@978:   if !project
Chris@978:     false
Chris@983:   elsif project =~ /^\d+$/
Chris@983:     # ignore numerical project ids, they are only used when editing projects
Chris@983:     false
Chris@978:   elsif @projects.key?(project)
Chris@979:     @projects[project].is_public? 
Chris@978:   else
Chris@978:     pobj = Project.find_by_identifier(project)
Chris@978:     if pobj
Chris@978:       @projects[project] = pobj
Chris@979:       pobj.is_public?
Chris@978:     else
Chris@979:       print "Project not found: ", project, "\n"
Chris@978:       false
Chris@978:     end
Chris@978:   end
Chris@978: end
Chris@978: 
Chris@980: def print_stats(h)
Chris@980:   h.keys.sort { |a,b| h[b] <=> h[a] }.each do |p|
Chris@982:     if h[p] > 0
Chris@983:       print h[p], " ", @projects[p].name, " [", p, "]\n"
Chris@982:     end
Chris@980:   end
Chris@980: end
Chris@980: 
Chris@979: STDIN.each do |line|
Chris@975: 
Chris@975:   record = parser.parse(line)
Chris@975: 
Chris@975:   # most annoyingly, the parser can't handle the comma-separated list
Chris@975:   # in X-Forwarded-For where it has more than one element. If it has
Chris@983:   # failed, remove any IP addresses or the word "unknown" with
Chris@983:   # trailing commas and try again
Chris@975:   if not record
Chris@983:     filtered = line.gsub(/(unknown|([0-9]+\.){3}[0-9]+),\s*/, "")
Chris@975:     record = parser.parse(filtered)
Chris@975:   end
Chris@975: 
Chris@975:   # discard, but count, unparseable lines
Chris@975:   if not record
Chris@979:     print "Line not parseable: ", line, "\n"
Chris@975:     unparseable += 1
Chris@975:     next
Chris@975:   end
Chris@975: 
Chris@975:   # discard everything that isn't a 200 OK response
Chris@975:   next if record["%>s"] != "200"
Chris@975: 
Chris@975:   # discard anything apparently requested by a crawler
Chris@975:   next if record["%{User-Agent}i"] =~ /(bot|slurp|crawler|spider|Redmine)\b/i
Chris@975: 
Chris@975:   # pull out request e.g. GET / HTTP/1.0
Chris@975:   request = record["%r"]
Chris@975: 
Chris@975:   # split into method, path, protocol
Chris@975:   if not request =~ /^[^\s]+ ([^\s]+) [^\s]+$/
Chris@979:     print "Line not parseable (bad method, path, protocol): ", line, "\n"
Chris@975:     unparseable += 1
Chris@975:     next
Chris@975:   end
Chris@975: 
Chris@975:   # get the path e.g. /projects/weevilmatic and split on /
Chris@975:   path = $~[1]
Chris@975:   components = path.split("/")
Chris@975:   
Chris@975:   # should have at least two elements unless path is "/"; first should
Chris@975:   # be empty (begins with /)
Chris@975:   if path != "/" and (components.size < 2 or components[0] != "")
Chris@979:     print "Line not parseable (degenerate path): ", line, "\n"
Chris@975:     unparseable += 1
Chris@975:     next
Chris@975:   end
Chris@975: 
Chris@975:   if components[1] == "hg"
Chris@975:     
Chris@975:     # path is /hg/project?something or /hg/project/something
Chris@975: 
Chris@975:     project = components[2].split("?")[0]
Chris@979:     if not is_public_project?(project)
Chris@978:       next
Chris@978:     end
Chris@975: 
Chris@975:     if components[2] =~ /&roots=00*$/
Chris@975:       clones[project] += 1
Chris@975:     elsif components[2] =~ /cmd=capabilities/
Chris@975:       pulls[project] += 1
Chris@978:     elsif components[2] =~ /cmd=unbundle/
Chris@978:       pushes[project] += 1
Chris@975:     elsif components[3] == "archive"
Chris@975:       zips[project] += 1
Chris@975:     end
Chris@975: 
Chris@975:   elsif components[1] == "projects"
Chris@975: 
Chris@975:     # path is /projects/project or /projects/project/something
Chris@975: 
Chris@975:     project = components[2]
Chris@979:     project = project.split("?")[0] if project
Chris@979:     if not is_public_project?(project)
Chris@978:       next
Chris@975:     end
Chris@975: 
Chris@978:     hits[project] += 1
Chris@978: 
Chris@975:   end
Chris@975: 
Chris@975:   parseable += 1
Chris@975: end
Chris@975: 
Chris@975: # Each clone is also a pull; deduct it from the pulls hash, because we
Chris@975: # want that to contain only non-clone pulls
Chris@975: 
Chris@975: clones.keys.each do |project|
Chris@975:   pulls[project] -= 1
Chris@975: end
Chris@975: 
Chris@982: print parseable, " parseable\n"
Chris@982: print unparseable, " unparseable\n"
Chris@982: 
Chris@982: 
Chris@980: print "\nMercurial clones:\n"
Chris@980: print_stats clones
Chris@980: 
Chris@980: print "\nMercurial pulls (excluding clones):\n"
Chris@980: print_stats pulls
Chris@980: 
Chris@980: print "\nMercurial pushes:\n"
Chris@980: print_stats pushes
Chris@980: 
Chris@980: print "\nMercurial archive (zip file) downloads:\n"
Chris@980: print_stats zips
Chris@980: 
Chris@982: print "\nProject page hits (excluding crawlers):\n"
Chris@980: print_stats hits
Chris@975: 
Chris@975: