Chris@0: module CodeRay
Chris@0: 
Chris@0:   require 'coderay/helpers/plugin'
Chris@0: 
Chris@0:   # = Scanners
Chris@0:   #
Chris@0:   # This module holds the Scanner class and its subclasses.
Chris@0:   # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
Chris@0:   # can be found in coderay/scanners/ruby.
Chris@0:   #
Chris@0:   # Scanner also provides methods and constants for the register
Chris@0:   # mechanism and the [] method that returns the Scanner class
Chris@0:   # belonging to the given lang.
Chris@0:   #
Chris@0:   # See PluginHost.
Chris@0:   module Scanners
Chris@0:     extend PluginHost
Chris@0:     plugin_path File.dirname(__FILE__), 'scanners'
Chris@0: 
Chris@0:     require 'strscan'
Chris@0: 
Chris@0:     # = Scanner
Chris@0:     #
Chris@0:     # The base class for all Scanners.
Chris@0:     #
Chris@0:     # It is a subclass of Ruby's great +StringScanner+, which
Chris@0:     # makes it easy to access the scanning methods inside.
Chris@0:     #
Chris@0:     # It is also +Enumerable+, so you can use it like an Array of
Chris@0:     # Tokens:
Chris@0:     #
Chris@0:     #   require 'coderay'
Chris@0:     #   
Chris@0:     #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
Chris@0:     #   
Chris@0:     #   for text, kind in c_scanner
Chris@0:     #     puts text if kind == :operator
Chris@0:     #   end
Chris@0:     #   
Chris@0:     #   # prints: (*==)++;
Chris@0:     #
Chris@0:     # OK, this is a very simple example :)
Chris@0:     # You can also use +map+, +any?+, +find+ and even +sort_by+,
Chris@0:     # if you want.
Chris@0:     class Scanner < StringScanner
Chris@0:       
Chris@0:       extend Plugin
Chris@0:       plugin_host Scanners
Chris@0: 
Chris@0:       # Raised if a Scanner fails while scanning
Chris@0:       ScanError = Class.new(Exception)
Chris@0: 
Chris@0:       require 'coderay/helpers/word_list'
Chris@0: 
Chris@0:       # The default options for all scanner classes.
Chris@0:       #
Chris@0:       # Define @default_options for subclasses.
Chris@0:       DEFAULT_OPTIONS = { :stream => false }
Chris@0:       
Chris@0:       KINDS_NOT_LOC = [:comment, :doctype]
Chris@0: 
Chris@0:       class << self
Chris@0: 
Chris@0:         # Returns if the Scanner can be used in streaming mode.
Chris@0:         def streamable?
Chris@0:           is_a? Streamable
Chris@0:         end
Chris@0: 
Chris@0:         def normify code
Chris@0:           code = code.to_s
Chris@0:           if code.respond_to? :force_encoding
Chris@0:             debug, $DEBUG = $DEBUG, false
Chris@0:             begin
Chris@0:               code.force_encoding 'utf-8'
Chris@0:               code[/\z/]  # raises an ArgumentError when code contains a non-UTF-8 char
Chris@0:             rescue ArgumentError
Chris@0:               code.force_encoding 'binary'
Chris@0:             ensure
Chris@0:               $DEBUG = debug
Chris@0:             end
Chris@0:           end
Chris@0:           code.to_unix
Chris@0:         end
Chris@0:         
Chris@0:         def file_extension extension = nil
Chris@0:           if extension
Chris@0:             @file_extension = extension.to_s
Chris@0:           else
Chris@0:             @file_extension ||= plugin_id.to_s
Chris@0:           end
Chris@0:         end
Chris@0: 
Chris@0:       end
Chris@0: 
Chris@0: =begin
Chris@0: ## Excluded for speed reasons; protected seems to make methods slow.
Chris@0: 
Chris@0:   # Save the StringScanner methods from being called.
Chris@0:   # This would not be useful for highlighting.
Chris@0:   strscan_public_methods =
Chris@0:     StringScanner.instance_methods -
Chris@0:     StringScanner.ancestors[1].instance_methods
Chris@0:   protected(*strscan_public_methods)
Chris@0: =end
Chris@0: 
Chris@0:       # Create a new Scanner.
Chris@0:       #
Chris@0:       # * +code+ is the input String and is handled by the superclass
Chris@0:       #   StringScanner.
Chris@0:       # * +options+ is a Hash with Symbols as keys.
Chris@0:       #   It is merged with the default options of the class (you can
Chris@0:       #   overwrite default options here.)
Chris@0:       # * +block+ is the callback for streamed highlighting.
Chris@0:       #
Chris@0:       # If you set :stream to +true+ in the options, the Scanner uses a
Chris@0:       # TokenStream with the +block+ as callback to handle the tokens.
Chris@0:       #
Chris@0:       # Else, a Tokens object is used.
Chris@0:       def initialize code='', options = {}, &block
Chris@0:         raise "I am only the basic Scanner class. I can't scan "\
Chris@0:           "anything. :( Use my subclasses." if self.class == Scanner
Chris@0:         
Chris@0:         @options = self.class::DEFAULT_OPTIONS.merge options
Chris@0: 
Chris@0:         super Scanner.normify(code)
Chris@0: 
Chris@0:         @tokens = options[:tokens]
Chris@0:         if @options[:stream]
Chris@0:           warn "warning in CodeRay::Scanner.new: :stream is set, "\
Chris@0:             "but no block was given" unless block_given?
Chris@0:           raise NotStreamableError, self unless kind_of? Streamable
Chris@0:           @tokens ||= TokenStream.new(&block)
Chris@0:         else
Chris@0:           warn "warning in CodeRay::Scanner.new: Block given, "\
Chris@0:             "but :stream is #{@options[:stream]}" if block_given?
Chris@0:           @tokens ||= Tokens.new
Chris@0:         end
Chris@0:         @tokens.scanner = self
Chris@0: 
Chris@0:         setup
Chris@0:       end
Chris@0: 
Chris@0:       def reset
Chris@0:         super
Chris@0:         reset_instance
Chris@0:       end
Chris@0: 
Chris@0:       def string= code
Chris@0:         code = Scanner.normify(code)
Chris@0:         super code
Chris@0:         reset_instance
Chris@0:       end
Chris@0: 
Chris@0:       # More mnemonic accessor name for the input string.
Chris@0:       alias code string
Chris@0:       alias code= string=
Chris@0: 
Chris@0:       # Returns the Plugin ID for this scanner.
Chris@0:       def lang
Chris@0:         self.class.plugin_id
Chris@0:       end
Chris@0: 
Chris@0:       # Scans the code and returns all tokens in a Tokens object.
Chris@0:       def tokenize new_string=nil, options = {}
Chris@0:         options = @options.merge(options)
Chris@0:         self.string = new_string if new_string
Chris@0:         @cached_tokens =
Chris@0:           if @options[:stream]  # :stream must have been set already
Chris@0:             reset unless new_string
Chris@0:             scan_tokens @tokens, options
Chris@0:             @tokens
Chris@0:           else
Chris@0:             scan_tokens @tokens, options
Chris@0:           end
Chris@0:       end
Chris@0: 
Chris@0:       def tokens
Chris@0:         @cached_tokens ||= tokenize
Chris@0:       end
Chris@0:       
Chris@0:       # Whether the scanner is in streaming mode.
Chris@0:       def streaming?
Chris@0:         !!@options[:stream]
Chris@0:       end
Chris@0: 
Chris@0:       # Traverses the tokens.
Chris@0:       def each &block
Chris@0:         raise ArgumentError,
Chris@0:           'Cannot traverse TokenStream.' if @options[:stream]
Chris@0:         tokens.each(&block)
Chris@0:       end
Chris@0:       include Enumerable
Chris@0: 
Chris@0:       # The current line position of the scanner.
Chris@0:       #
Chris@0:       # Beware, this is implemented inefficiently. It should be used
Chris@0:       # for debugging only.
Chris@0:       def line
Chris@0:         string[0..pos].count("\n") + 1
Chris@0:       end
Chris@0:       
Chris@0:       def column pos = self.pos
Chris@0:         return 0 if pos <= 0
Chris@0:         string = string()
Chris@0:         if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
Chris@0:           @bin_string ||= string.dup.force_encoding('binary')
Chris@0:           string = @bin_string
Chris@0:         end
Chris@0:         pos - (string.rindex(?\n, pos) || 0)
Chris@0:       end
Chris@0:       
Chris@0:       def marshal_dump
Chris@0:         @options
Chris@0:       end
Chris@0:       
Chris@0:       def marshal_load options
Chris@0:         @options = options
Chris@0:       end
Chris@0: 
Chris@0:     protected
Chris@0: 
Chris@0:       # Can be implemented by subclasses to do some initialization
Chris@0:       # that has to be done once per instance.
Chris@0:       #
Chris@0:       # Use reset for initialization that has to be done once per
Chris@0:       # scan.
Chris@0:       def setup
Chris@0:       end
Chris@0: 
Chris@0:       # This is the central method, and commonly the only one a
Chris@0:       # subclass implements.
Chris@0:       #
Chris@0:       # Subclasses must implement this method; it must return +tokens+
Chris@0:       # and must only use Tokens#<< for storing scanned tokens!
Chris@0:       def scan_tokens tokens, options
Chris@0:         raise NotImplementedError,
Chris@0:           "#{self.class}#scan_tokens not implemented."
Chris@0:       end
Chris@0: 
Chris@0:       def reset_instance
Chris@0:         @tokens.clear unless @options[:keep_tokens]
Chris@0:         @cached_tokens = nil
Chris@0:         @bin_string = nil if defined? @bin_string
Chris@0:       end
Chris@0: 
Chris@0:       # Scanner error with additional status information
Chris@0:       def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
Chris@0:         raise ScanError, <<-EOE % [
Chris@0: 
Chris@0: 
Chris@0: ***ERROR in %s: %s (after %d tokens)
Chris@0: 
Chris@0: tokens:
Chris@0: %s
Chris@0: 
Chris@0: current line: %d  column: %d  pos: %d
Chris@0: matched: %p  state: %p
Chris@0: bol? = %p,  eos? = %p
Chris@0: 
Chris@0: surrounding code:
Chris@0: %p  ~~  %p
Chris@0: 
Chris@0: 
Chris@0: ***ERROR***
Chris@0: 
Chris@0:         EOE
Chris@0:           File.basename(caller[0]),
Chris@0:           msg,
Chris@0:           tokens.size,
Chris@0:           tokens.last(10).map { |t| t.inspect }.join("\n"),
Chris@0:           line, column, pos,
Chris@0:           matched, state, bol?, eos?,
Chris@0:           string[pos - ambit, ambit],
Chris@0:           string[pos, ambit],
Chris@0:         ]
Chris@0:       end
Chris@0: 
Chris@0:     end
Chris@0: 
Chris@0:   end
Chris@0: end
Chris@0: 
Chris@0: class String
Chris@0:   # I love this hack. It seems to silence all dos/unix/mac newline problems.
Chris@0:   def to_unix
Chris@0:     if index ?\r
Chris@0:       gsub(/\r\n?/, "\n")
Chris@0:     else
Chris@0:       self
Chris@0:     end
Chris@0:   end
Chris@0: end