SoundSoftware Code Site

module CodeRay

module Scanners

  # Scanner for Python. Supports Python 3.

  # 

  # Based on pygments' PythonLexer, see

  # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.

  class Python < Scanner

    register_for :python

    file_extension 'py'

    KEYWORDS = [

      'and', 'as', 'assert', 'break', 'class', 'continue', 'def',

      'del', 'elif', 'else', 'except', 'finally', 'for',

      'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',

      'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',

      'nonlocal',  # new in Python 3

    ]  # :nodoc:

    OLD_KEYWORDS = [

      'exec', 'print',  # gone in Python 3

    ]  # :nodoc:

    PREDEFINED_METHODS_AND_TYPES = %w[

      __import__ abs all any apply basestring bin bool buffer

      bytearray bytes callable chr classmethod cmp coerce compile

      complex delattr dict dir divmod enumerate eval execfile exit

      file filter float frozenset getattr globals hasattr hash hex id

      input int intern isinstance issubclass iter len list locals

      long map max min next object oct open ord pow property range

      raw_input reduce reload repr reversed round set setattr slice

      sorted staticmethod str sum super tuple type unichr unicode

      vars xrange zip

    ]  # :nodoc:

    PREDEFINED_EXCEPTIONS = %w[

      ArithmeticError AssertionError AttributeError

      BaseException DeprecationWarning EOFError EnvironmentError

      Exception FloatingPointError FutureWarning GeneratorExit IOError

      ImportError ImportWarning IndentationError IndexError KeyError

      KeyboardInterrupt LookupError MemoryError NameError

      NotImplemented NotImplementedError OSError OverflowError

      OverflowWarning PendingDeprecationWarning ReferenceError

      RuntimeError RuntimeWarning StandardError StopIteration

      SyntaxError SyntaxWarning SystemError SystemExit TabError

      TypeError UnboundLocalError UnicodeDecodeError

      UnicodeEncodeError UnicodeError UnicodeTranslateError

      UnicodeWarning UserWarning ValueError Warning ZeroDivisionError

    ]  # :nodoc:

    PREDEFINED_VARIABLES_AND_CONSTANTS = [

      'False', 'True', 'None',  # "keywords" since Python 3

      'self', 'Ellipsis', 'NotImplemented',

    ]  # :nodoc:

    IDENT_KIND = WordList.new(:ident).

      add(KEYWORDS, :keyword).

      add(OLD_KEYWORDS, :old_keyword).

      add(PREDEFINED_METHODS_AND_TYPES, :predefined).

      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :predefined_constant).

      add(PREDEFINED_EXCEPTIONS, :exception)  # :nodoc:

    NAME = / [^\W\d] \w* /x  # :nodoc:

    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:

    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x  # :nodoc:

    OPERATOR = /

      \.\.\. |          # ellipsis

      \.(?!\d) |        # dot but not decimal point

      [,;:()\[\]{}] |   # simple delimiters

      \/\/=? | \*\*=? | # special math

      [-+*\/%&|^]=? |   # ordinary math and binary logic

      [~`] |            # binary complement and inspection

      <<=? | >>=? | [<>=]=? | !=  # comparison and assignment

    /x  # :nodoc:

    STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter|

      h[delimiter] = Regexp.union delimiter  # :nodoc:

    }

    STRING_CONTENT_REGEXP = Hash.new { |h, delimiter|

      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x  # :nodoc:

    }

    DEF_NEW_STATE = WordList.new(:initial).

      add(%w(def), :def_expected).

      add(%w(import from), :include_expected).

      add(%w(class), :class_expected)  # :nodoc:

    DESCRIPTOR = /

      #{NAME}

      (?: \. #{NAME} )*

      | \*

    /x  # :nodoc:

    DOCSTRING_COMING = /

      [ \t]* u?r? ("""|''')

    /x  # :nodoc:

  protected

    def scan_tokens encoder, options

      state = :initial

      string_delimiter = nil

      string_raw = false

      string_type = nil

      docstring_coming = match?(/#{DOCSTRING_COMING}/o)

      last_token_dot = false

      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'

      from_import_state = []

      until eos?

        if state == :string

          if match = scan(STRING_DELIMITER_REGEXP[string_delimiter])

            encoder.text_token match, :delimiter

            encoder.end_group string_type

            string_type = nil

            state = :initial

            next

          elsif string_delimiter.size == 3 && match = scan(/\n/)

            encoder.text_token match, :content

          elsif match = scan(STRING_CONTENT_REGEXP[string_delimiter])

            encoder.text_token match, :content

          elsif !string_raw && match = scan(/ \\ #{ESCAPE} /ox)

            encoder.text_token match, :char

          elsif match = scan(/ \\ #{UNICODE_ESCAPE} /ox)

            encoder.text_token match, :char

          elsif match = scan(/ \\ . /x)

            encoder.text_token match, :content

          elsif match = scan(/ \\ | $ /x)

            encoder.end_group string_type

            string_type = nil

            encoder.text_token match, :error

            state = :initial

          else

            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state

          end

        elsif match = scan(/ [ \t]+ | \\?\n /x)

          encoder.text_token match, :space

          if match == "\n"

            state = :initial if state == :include_expected

            docstring_coming = true if match?(/#{DOCSTRING_COMING}/o)

          end

          next

        elsif match = scan(/ \# [^\n]* /mx)

          encoder.text_token match, :comment

          next

        elsif state == :initial

          if match = scan(/#{OPERATOR}/o)

            encoder.text_token match, :operator

          elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)

            string_delimiter = self[2]

            string_type = docstring_coming ? :docstring : :string

            docstring_coming = false if docstring_coming

            encoder.begin_group string_type

            string_raw = false

            modifiers = self[1]

            unless modifiers.empty?

              string_raw = !!modifiers.index(?r)

              encoder.text_token modifiers, :modifier

              match = string_delimiter

            end

            state = :string

            encoder.text_token match, :delimiter

          # TODO: backticks

          elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            kind = IDENT_KIND[match]

            # TODO: keyword arguments

            kind = :ident if last_token_dot

            if kind == :old_keyword

              kind = check(/\(/) ? :ident : :keyword

            elsif kind == :predefined && check(/ *=/)

              kind = :ident

            elsif kind == :keyword

              state = DEF_NEW_STATE[match]

              from_import_state << match.to_sym if state == :include_expected

            end

            encoder.text_token match, kind

          elsif match = scan(/@[a-zA-Z0-9_.]+[lL]?/)

            encoder.text_token match, :decorator

          elsif match = scan(/0[xX][0-9A-Fa-f]+[lL]?/)

            encoder.text_token match, :hex

          elsif match = scan(/0[bB][01]+[lL]?/)

            encoder.text_token match, :binary

          elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)

            if scan(/[jJ]/)

              match << matched

              encoder.text_token match, :imaginary

            else

              encoder.text_token match, :float

            end

          elsif match = scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)

            encoder.text_token match, :octal

          elsif match = scan(/\d+([lL])?/)

            if self[1] == nil && scan(/[jJ]/)

              match << matched

              encoder.text_token match, :imaginary

            else

              encoder.text_token match, :integer

            end

          else

            encoder.text_token getch, :error

          end

        elsif state == :def_expected

          state = :initial

          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            encoder.text_token match, :method

          else

            next

          end

        elsif state == :class_expected

          state = :initial

          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)

            encoder.text_token match, :class

          else

            next

          end

        elsif state == :include_expected

          if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)

            if match == 'as'

              encoder.text_token match, :keyword

              from_import_state << :as

            elsif from_import_state.first == :from && match == 'import'

              encoder.text_token match, :keyword

              from_import_state << :import

            elsif from_import_state.last == :as

              # encoder.text_token match, match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method

              encoder.text_token match, :ident

              from_import_state.pop

            elsif IDENT_KIND[match] == :keyword

              unscan

              match = nil

              state = :initial

              next

            else

              encoder.text_token match, :include

            end

          elsif match = scan(/,/)

            from_import_state.pop if from_import_state.last == :as

            encoder.text_token match, :operator

          else

            from_import_state = []

            state = :initial

            next

          end

        else

          raise_inspect 'Unknown state', encoder, state

        end

        last_token_dot = match == '.'

      end

      if state == :string

        encoder.end_group string_type

      end

      encoder

    end

  end

end

end