Chris@0: module CodeRay
Chris@0: module Scanners
Chris@0:   
Chris@0:   # Bases on pygments' PythonLexer, see
Chris@0:   # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
Chris@0:   class Python < Scanner
Chris@0:     
Chris@0:     include Streamable
Chris@0:     
Chris@0:     register_for :python
Chris@0:     file_extension 'py'
Chris@0:     
Chris@0:     KEYWORDS = [
Chris@0:       'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
Chris@0:       'del', 'elif', 'else', 'except', 'finally', 'for',
Chris@0:       'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
Chris@0:       'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
Chris@0:       'nonlocal',  # new in Python 3
Chris@0:     ]
Chris@0:     
Chris@0:     OLD_KEYWORDS = [
Chris@0:       'exec', 'print',  # gone in Python 3
Chris@0:     ]
Chris@0:     
Chris@0:     PREDEFINED_METHODS_AND_TYPES = %w[
Chris@0:       __import__ abs all any apply basestring bin bool buffer
Chris@0:       bytearray bytes callable chr classmethod cmp coerce compile
Chris@0:       complex delattr dict dir divmod enumerate eval execfile exit
Chris@0:       file filter float frozenset getattr globals hasattr hash hex id
Chris@0:       input int intern isinstance issubclass iter len list locals
Chris@0:       long map max min next object oct open ord pow property range
Chris@0:       raw_input reduce reload repr reversed round set setattr slice
Chris@0:       sorted staticmethod str sum super tuple type unichr unicode
Chris@0:       vars xrange zip
Chris@0:     ]
Chris@0:     
Chris@0:     PREDEFINED_EXCEPTIONS = %w[
Chris@0:       ArithmeticError AssertionError AttributeError
Chris@0:       BaseException DeprecationWarning EOFError EnvironmentError
Chris@0:       Exception FloatingPointError FutureWarning GeneratorExit IOError
Chris@0:       ImportError ImportWarning IndentationError IndexError KeyError
Chris@0:       KeyboardInterrupt LookupError MemoryError NameError
Chris@0:       NotImplemented NotImplementedError OSError OverflowError
Chris@0:       OverflowWarning PendingDeprecationWarning ReferenceError
Chris@0:       RuntimeError RuntimeWarning StandardError StopIteration
Chris@0:       SyntaxError SyntaxWarning SystemError SystemExit TabError
Chris@0:       TypeError UnboundLocalError UnicodeDecodeError
Chris@0:       UnicodeEncodeError UnicodeError UnicodeTranslateError
Chris@0:       UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
Chris@0:     ]
Chris@0:     
Chris@0:     PREDEFINED_VARIABLES_AND_CONSTANTS = [
Chris@0:       'False', 'True', 'None', # "keywords" since Python 3
Chris@0:       'self', 'Ellipsis', 'NotImplemented',
Chris@0:     ]
Chris@0:     
Chris@0:     IDENT_KIND = WordList.new(:ident).
Chris@0:       add(KEYWORDS, :keyword).
Chris@0:       add(OLD_KEYWORDS, :old_keyword).
Chris@0:       add(PREDEFINED_METHODS_AND_TYPES, :predefined).
Chris@0:       add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
Chris@0:       add(PREDEFINED_EXCEPTIONS, :exception)
Chris@0:     
Chris@0:     NAME = / [^\W\d] \w* /x
Chris@0:     ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
Chris@0:     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
Chris@0:     
Chris@0:     OPERATOR = /
Chris@0:       \.\.\. |          # ellipsis
Chris@0:       \.(?!\d) |        # dot but not decimal point
Chris@0:       [,;:()\[\]{}] |   # simple delimiters
Chris@0:       \/\/=? | \*\*=? | # special math
Chris@0:       [-+*\/%&|^]=? |   # ordinary math and binary logic
Chris@0:       [~`] |            # binary complement and inspection
Chris@0:       <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
Chris@0:     /x
Chris@0:     
Chris@0:     STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
Chris@0:       h[delimiter] = Regexp.union delimiter
Chris@0:     end
Chris@0:     
Chris@0:     STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
Chris@0:       h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
Chris@0:     end
Chris@0:     
Chris@0:     DEF_NEW_STATE = WordList.new(:initial).
Chris@0:       add(%w(def), :def_expected).
Chris@0:       add(%w(import from), :include_expected).
Chris@0:       add(%w(class), :class_expected)
Chris@0:     
Chris@0:     DESCRIPTOR = /
Chris@0:       #{NAME}
Chris@0:       (?: \. #{NAME} )*
Chris@0:       | \*
Chris@0:     /x
Chris@0:     
Chris@0:     def scan_tokens tokens, options
Chris@0:       
Chris@0:       state = :initial
Chris@0:       string_delimiter = nil
Chris@0:       string_raw = false
Chris@0:       import_clause = class_name_follows = last_token_dot = false
Chris@0:       unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
Chris@0:       from_import_state = []
Chris@0:       
Chris@0:       until eos?
Chris@0:         
Chris@0:         kind = nil
Chris@0:         match = nil
Chris@0:         
Chris@0:         if state == :string
Chris@0:           if scan(STRING_DELIMITER_REGEXP[string_delimiter])
Chris@0:             tokens << [matched, :delimiter]
Chris@0:             tokens << [:close, :string]
Chris@0:             state = :initial
Chris@0:             next
Chris@0:           elsif string_delimiter.size == 3 && scan(/\n/)
Chris@0:             kind = :content
Chris@0:           elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
Chris@0:             kind = :content
Chris@0:           elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
Chris@0:             kind = :char
Chris@0:           elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
Chris@0:             kind = :char
Chris@0:           elsif scan(/ \\ . /x)
Chris@0:             kind = :content
Chris@0:           elsif scan(/ \\ | $ /x)
Chris@0:             tokens << [:close, :string]
Chris@0:             kind = :error
Chris@0:             state = :initial
Chris@0:           else
Chris@0:             raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
Chris@0:           end
Chris@0:         
Chris@0:         elsif match = scan(/ [ \t]+ | \\\n /x)
Chris@0:           tokens << [match, :space]
Chris@0:           next
Chris@0:         
Chris@0:         elsif match = scan(/\n/)
Chris@0:           tokens << [match, :space]
Chris@0:           state = :initial if state == :include_expected
Chris@0:           next
Chris@0:         
Chris@0:         elsif match = scan(/ \# [^\n]* /mx)
Chris@0:           tokens << [match, :comment]
Chris@0:           next
Chris@0:         
Chris@0:         elsif state == :initial
Chris@0:           
Chris@0:           if scan(/#{OPERATOR}/o)
Chris@0:             kind = :operator
Chris@0:           
Chris@0:           elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
Chris@0:             tokens << [:open, :string]
Chris@0:             string_delimiter = self[2]
Chris@0:             string_raw = false
Chris@0:             modifiers = self[1]
Chris@0:             unless modifiers.empty?
Chris@0:               string_raw = !!modifiers.index(?r)
Chris@0:               tokens << [modifiers, :modifier]
Chris@0:               match = string_delimiter
Chris@0:             end
Chris@0:             state = :string
Chris@0:             kind = :delimiter
Chris@0:           
Chris@0:           # TODO: backticks
Chris@0:           
Chris@0:           elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
Chris@0:             kind = IDENT_KIND[match]
Chris@0:             # TODO: keyword arguments
Chris@0:             kind = :ident if last_token_dot
Chris@0:             if kind == :old_keyword
Chris@0:               kind = check(/\(/) ? :ident : :keyword
Chris@0:             elsif kind == :predefined && check(/ *=/)
Chris@0:               kind = :ident
Chris@0:             elsif kind == :keyword
Chris@0:               state = DEF_NEW_STATE[match]
Chris@0:               from_import_state << match.to_sym if state == :include_expected
Chris@0:             end
Chris@0:           
Chris@0:           elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
Chris@0:             kind = :decorator
Chris@0:           
Chris@0:           elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
Chris@0:             kind = :hex
Chris@0:           
Chris@0:           elsif scan(/0[bB][01]+[lL]?/)
Chris@0:             kind = :bin
Chris@0:           
Chris@0:           elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
Chris@0:             kind = :float
Chris@0:             if scan(/[jJ]/)
Chris@0:               match << matched
Chris@0:               kind = :imaginary
Chris@0:             end
Chris@0:           
Chris@0:           elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
Chris@0:             kind = :oct
Chris@0:           
Chris@0:           elsif match = scan(/\d+([lL])?/)
Chris@0:             kind = :integer
Chris@0:             if self[1] == nil && scan(/[jJ]/)
Chris@0:               match << matched
Chris@0:               kind = :imaginary
Chris@0:             end
Chris@0:           
Chris@0:           else
Chris@0:             getch
Chris@0:             kind = :error
Chris@0:           
Chris@0:           end
Chris@0:             
Chris@0:         elsif state == :def_expected
Chris@0:           state = :initial
Chris@0:           if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
Chris@0:             kind = :method
Chris@0:           else
Chris@0:             next
Chris@0:           end
Chris@0:         
Chris@0:         elsif state == :class_expected
Chris@0:           state = :initial
Chris@0:           if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
Chris@0:             kind = :class
Chris@0:           else
Chris@0:             next
Chris@0:           end
Chris@0:           
Chris@0:         elsif state == :include_expected
Chris@0:           if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
Chris@0:             kind = :include
Chris@0:             if match == 'as'
Chris@0:               kind = :keyword
Chris@0:               from_import_state << :as
Chris@0:             elsif from_import_state.first == :from && match == 'import'
Chris@0:               kind = :keyword
Chris@0:               from_import_state << :import
Chris@0:             elsif from_import_state.last == :as
Chris@0:               # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
Chris@0:               kind = :ident
Chris@0:               from_import_state.pop
Chris@0:             elsif IDENT_KIND[match] == :keyword
Chris@0:               unscan
Chris@0:               match = nil
Chris@0:               state = :initial
Chris@0:               next
Chris@0:             end
Chris@0:           elsif match = scan(/,/)
Chris@0:             from_import_state.pop if from_import_state.last == :as
Chris@0:             kind = :operator
Chris@0:           else
Chris@0:             from_import_state = []
Chris@0:             state = :initial
Chris@0:             next
Chris@0:           end
Chris@0:           
Chris@0:         else
Chris@0:           raise_inspect 'Unknown state', tokens, state
Chris@0:           
Chris@0:         end
Chris@0:         
Chris@0:         match ||= matched
Chris@0:         if $CODERAY_DEBUG and not kind
Chris@0:           raise_inspect 'Error token %p in line %d' %
Chris@0:             [[match, kind], line], tokens, state
Chris@0:         end
Chris@0:         raise_inspect 'Empty token', tokens, state unless match
Chris@0:         
Chris@0:         last_token_dot = match == '.'
Chris@0:         
Chris@0:         tokens << [match, kind]
Chris@0:         
Chris@0:       end
Chris@0:       
Chris@0:       if state == :string
Chris@0:         tokens << [:close, :string]
Chris@0:       end
Chris@0:       
Chris@0:       tokens
Chris@0:     end
Chris@0:     
Chris@0:   end
Chris@0:   
Chris@0: end
Chris@0: end