SoundSoftware Code Site

module CodeRay

module Scanners

  # Bases on pygments' PythonLexer, see

  # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.

  class Python < Scanner

    include Streamable

    register_for :python

    file_extension 'py'

    KEYWORDS = [

      'and', 'as', 'assert', 'break', 'class', 'continue', 'def',

      'del', 'elif', 'else', 'except', 'finally', 'for',

      'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',

      'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',

      'nonlocal',  # new in Python 3

    ]

    OLD_KEYWORDS = [

      'exec', 'print',  # gone in Python 3

    ]

    PREDEFINED_METHODS_AND_TYPES = %w[

      __import__ abs all any apply basestring bin bool buffer

      bytearray bytes callable chr classmethod cmp coerce compile

      complex delattr dict dir divmod enumerate eval execfile exit

      file filter float frozenset getattr globals hasattr hash hex id

      input int intern isinstance issubclass iter len list locals

      long map max min next object oct open ord pow property range

      raw_input reduce reload repr reversed round set setattr slice

      sorted staticmethod str sum super tuple type unichr unicode

      vars xrange zip

    ]

    PREDEFINED_EXCEPTIONS = %w[

      ArithmeticError AssertionError AttributeError

      BaseException DeprecationWarning EOFError EnvironmentError

      Exception FloatingPointError FutureWarning GeneratorExit IOError

      ImportError ImportWarning IndentationError IndexError KeyError

      KeyboardInterrupt LookupError MemoryError NameError

      NotImplemented NotImplementedError OSError OverflowError

      OverflowWarning PendingDeprecationWarning ReferenceError

      RuntimeError RuntimeWarning StandardError StopIteration

      SyntaxError SyntaxWarning SystemError SystemExit TabError

      TypeError UnboundLocalError UnicodeDecodeError

      UnicodeEncodeError UnicodeError UnicodeTranslateError

      UnicodeWarning UserWarning ValueError Warning ZeroDivisionError

    ]

    PREDEFINED_VARIABLES_AND_CONSTANTS = [

      'False', 'True', 'None', # "keywords" since Python 3

      'self', 'Ellipsis', 'NotImplemented',

    ]

    IDENT_KIND = WordList.new(:ident).

      add(KEYWORDS, :keyword).

      add(OLD_KEYWORDS, :old_keyword).

      add(PREDEFINED_METHODS_AND_TYPES, :predefined).

      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).

      add(PREDEFINED_EXCEPTIONS, :exception)

    NAME = / [^\W\d] \w* /x

    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x

    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x

    OPERATOR = /

      \.\.\. |          # ellipsis

      \.(?!\d) |        # dot but not decimal point

      [,;:()\[\]{}] |   # simple delimiters

      \/\/=? | \*\*=? | # special math

      [-+*\/%&|^]=? |   # ordinary math and binary logic

      [~`] |            # binary complement and inspection

      <<=? | >>=? | [<>=]=? | !=  # comparison and assignment

    /x

    STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|

      h[delimiter] = Regexp.union delimiter

    end

    STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|

      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x

    end

    DEF_NEW_STATE = WordList.new(:initial).

      add(%w(def), :def_expected).

      add(%w(import from), :include_expected).

      add(%w(class), :class_expected)

    DESCRIPTOR = /

      #{NAME}

      (?: \. #{NAME} )*

      | \*

    /x

    def scan_tokens tokens, options

      state = :initial

      string_delimiter = nil

      string_raw = false

      import_clause = class_name_follows = last_token_dot = false

      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'

      from_import_state = []

      until eos?

        kind = nil

        match = nil

        if state == :string

          if scan(STRING_DELIMITER_REGEXP[string_delimiter])

            tokens << [matched, :delimiter]

            tokens << [:close, :string]

            state = :initial

            next

          elsif string_delimiter.size == 3 && scan(/\n/)

            kind = :content

          elsif scan(STRING_CONTENT_REGEXP[string_delimiter])

            kind = :content

          elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)

            kind = :char

          elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)

            kind = :char

          elsif scan(/ \\ . /x)

            kind = :content

          elsif scan(/ \\ | $ /x)

            tokens << [:close, :string]

            kind = :error

            state = :initial

          else

            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state

          end

        elsif match = scan(/ [ \t]+ | \\\n /x)

          tokens << [match, :space]

          next

        elsif match = scan(/\n/)

          tokens << [match, :space]

          state = :initial if state == :include_expected

          next

        elsif match = scan(/ \# [^\n]* /mx)

          tokens << [match, :comment]

          next

        elsif state == :initial

          if scan(/#{OPERATOR}/o)

            kind = :operator

          elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)

            tokens << [:open, :string]

            string_delimiter = self[2]

            string_raw = false

            modifiers = self[1]

            unless modifiers.empty?

              string_raw = !!modifiers.index(?r)

              tokens << [modifiers, :modifier]

              match = string_delimiter

            end

            state = :string

            kind = :delimiter

          # TODO: backticks

          elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            kind = IDENT_KIND[match]

            # TODO: keyword arguments

            kind = :ident if last_token_dot

            if kind == :old_keyword

              kind = check(/\(/) ? :ident : :keyword

            elsif kind == :predefined && check(/ *=/)

              kind = :ident

            elsif kind == :keyword

              state = DEF_NEW_STATE[match]

              from_import_state << match.to_sym if state == :include_expected

            end

          elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)

            kind = :decorator

          elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)

            kind = :hex

          elsif scan(/0[bB][01]+[lL]?/)

            kind = :bin

          elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)

            kind = :float

            if scan(/[jJ]/)

              match << matched

              kind = :imaginary

            end

          elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)

            kind = :oct

          elsif match = scan(/\d+([lL])?/)

            kind = :integer

            if self[1] == nil && scan(/[jJ]/)

              match << matched

              kind = :imaginary

            end

          else

            getch

            kind = :error

          end

        elsif state == :def_expected

          state = :initial

          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            kind = :method

          else

            next

          end

        elsif state == :class_expected

          state = :initial

          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            kind = :class

          else

            next

          end

        elsif state == :include_expected

          if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)

            kind = :include

            if match == 'as'

              kind = :keyword

              from_import_state << :as

            elsif from_import_state.first == :from && match == 'import'

              kind = :keyword

              from_import_state << :import

            elsif from_import_state.last == :as

              # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method

              kind = :ident

              from_import_state.pop

            elsif IDENT_KIND[match] == :keyword

              unscan

              match = nil

              state = :initial

              next

            end

          elsif match = scan(/,/)

            from_import_state.pop if from_import_state.last == :as

            kind = :operator

          else

            from_import_state = []

            state = :initial

            next

          end

        else

          raise_inspect 'Unknown state', tokens, state

        end

        match ||= matched

        if $CODERAY_DEBUG and not kind

          raise_inspect 'Error token %p in line %d' %

            [[match, kind], line], tokens, state

        end

        raise_inspect 'Empty token', tokens, state unless match

        last_token_dot = match == '.'

        tokens << [match, kind]

      end

      if state == :string

        tokens << [:close, :string]

      end

      tokens

    end

  end

end

end