annotate vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb @ 855:7294e8db2515 bug_162

Close obsolete branch bug_162

module CodeRay

  require 'coderay/helpers/plugin'

  # = Scanners
  #
  # This module holds the Scanner class and its subclasses.
  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
  # can be found in coderay/scanners/ruby.
  #
  # Scanner also provides methods and constants for the register
  # mechanism and the [] method that returns the Scanner class
  # belonging to the given lang.
  #
  # See PluginHost.
  module Scanners
    extend PluginHost
    plugin_path File.dirname(__FILE__), 'scanners'

    require 'strscan'

    # = Scanner
    #
    # The base class for all Scanners.
    #
    # It is a subclass of Ruby's great +StringScanner+, which
    # makes it easy to access the scanning methods inside.
    #
    # It is also +Enumerable+, so you can use it like an Array of
    # Tokens:
    #
    #   require 'coderay'
    #   
    #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
    #   
    #   for text, kind in c_scanner
    #     puts text if kind == :operator
    #   end
    #   
    #   # prints: (*==)++;
    #
    # OK, this is a very simple example :)
    # You can also use +map+, +any?+, +find+ and even +sort_by+,
    # if you want.
    class Scanner < StringScanner
      
      extend Plugin
      plugin_host Scanners

      # Raised if a Scanner fails while scanning
      ScanError = Class.new(Exception)

      require 'coderay/helpers/word_list'

      # The default options for all scanner classes.
      #
      # Define @default_options for subclasses.
      DEFAULT_OPTIONS = { :stream => false }
      
      KINDS_NOT_LOC = [:comment, :doctype]

      class << self

        # Returns if the Scanner can be used in streaming mode.
        def streamable?
          is_a? Streamable
        end

        def normify code
          code = code.to_s
          if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
            code = code.dup
            original_encoding = code.encoding
            code.force_encoding 'Windows-1252'
            unless code.valid_encoding?
              code.force_encoding original_encoding
              if code.encoding.name == 'UTF-8'
                code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
              end
              code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
            end
          end
          code.to_unix
        end
        
        def file_extension extension = nil
          if extension
            @file_extension = extension.to_s
          else
            @file_extension ||= plugin_id.to_s
          end
        end

      end

=begin
## Excluded for speed reasons; protected seems to make methods slow.

  # Save the StringScanner methods from being called.
  # This would not be useful for highlighting.
  strscan_public_methods =
    StringScanner.instance_methods -
    StringScanner.ancestors[1].instance_methods
  protected(*strscan_public_methods)
=end

      # Create a new Scanner.
      #
      # * +code+ is the input String and is handled by the superclass
      #   StringScanner.
      # * +options+ is a Hash with Symbols as keys.
      #   It is merged with the default options of the class (you can
      #   overwrite default options here.)
      # * +block+ is the callback for streamed highlighting.
      #
      # If you set :stream to +true+ in the options, the Scanner uses a
      # TokenStream with the +block+ as callback to handle the tokens.
      #
      # Else, a Tokens object is used.
      def initialize code='', options = {}, &block
        raise "I am only the basic Scanner class. I can't scan "\
          "anything. :( Use my subclasses." if self.class == Scanner
        
        @options = self.class::DEFAULT_OPTIONS.merge options

        super Scanner.normify(code)

        @tokens = options[:tokens]
        if @options[:stream]
          warn "warning in CodeRay::Scanner.new: :stream is set, "\
            "but no block was given" unless block_given?
          raise NotStreamableError, self unless kind_of? Streamable
          @tokens ||= TokenStream.new(&block)
        else
          warn "warning in CodeRay::Scanner.new: Block given, "\
            "but :stream is #{@options[:stream]}" if block_given?
          @tokens ||= Tokens.new
        end
        @tokens.scanner = self

        setup
      end

      def reset
        super
        reset_instance
      end

      def string= code
        code = Scanner.normify(code)
        if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
          reset_state
          @string = code
        else
          super code
        end
        reset_instance
      end

      # More mnemonic accessor name for the input string.
      alias code string
      alias code= string=

      # Returns the Plugin ID for this scanner.
      def lang
        self.class.plugin_id
      end

      # Scans the code and returns all tokens in a Tokens object.
      def tokenize new_string=nil, options = {}
        options = @options.merge(options)
        self.string = new_string if new_string
        @cached_tokens =
          if @options[:stream]  # :stream must have been set already
            reset unless new_string
            scan_tokens @tokens, options
            @tokens
          else
            scan_tokens @tokens, options
          end
      end

      def tokens
        @cached_tokens ||= tokenize
      end
      
      # Whether the scanner is in streaming mode.
      def streaming?
        !!@options[:stream]
      end

      # Traverses the tokens.
      def each &block
        raise ArgumentError,
          'Cannot traverse TokenStream.' if @options[:stream]
        tokens.each(&block)
      end
      include Enumerable

      # The current line position of the scanner.
      #
      # Beware, this is implemented inefficiently. It should be used
      # for debugging only.
      def line
        string[0..pos].count("\n") + 1
      end
      
      def column pos = self.pos
        return 0 if pos <= 0
        string = string()
        if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
          @bin_string ||= string.dup.force_encoding('binary')
          string = @bin_string
        end
        pos - (string.rindex(?\n, pos) || 0)
      end
      
      def marshal_dump
        @options
      end
      
      def marshal_load options
        @options = options
      end

    protected

      # Can be implemented by subclasses to do some initialization
      # that has to be done once per instance.
      #
      # Use reset for initialization that has to be done once per
      # scan.
      def setup
      end

      # This is the central method, and commonly the only one a
      # subclass implements.
      #
      # Subclasses must implement this method; it must return +tokens+
      # and must only use Tokens#<< for storing scanned tokens!
      def scan_tokens tokens, options
        raise NotImplementedError,
          "#{self.class}#scan_tokens not implemented."
      end

      def reset_instance
        @tokens.clear unless @options[:keep_tokens]
        @cached_tokens = nil
        @bin_string = nil if defined? @bin_string
      end

      # Scanner error with additional status information
      def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
        raise ScanError, <<-EOE % [


***ERROR in %s: %s (after %d tokens)

tokens:
%s

current line: %d  column: %d  pos: %d
matched: %p  state: %p
bol? = %p,  eos? = %p

surrounding code:
%p  ~~  %p


***ERROR***

        EOE
          File.basename(caller[0]),
          msg,
          tokens.size,
          tokens.last(10).map { |t| t.inspect }.join("\n"),
          line, column, pos,
          matched, state, bol?, eos?,
          string[pos - ambit, ambit],
          string[pos, ambit],
        ]
      end

    end

  end
end

class String
  # I love this hack. It seems to silence all dos/unix/mac newline problems.
  def to_unix
    if index ?\r
      gsub(/\r\n?/, "\n")
    else
      self
    end
  end
end