Chris@0: module CodeRay Chris@0: module Scanners Chris@0: Chris@0: class JavaScript < Scanner Chris@0: Chris@0: include Streamable Chris@0: Chris@0: register_for :java_script Chris@0: file_extension 'js' Chris@0: Chris@0: # The actual JavaScript keywords. Chris@0: KEYWORDS = %w[ Chris@0: break case catch continue default delete do else Chris@0: finally for function if in instanceof new Chris@0: return switch throw try typeof var void while with Chris@0: ] Chris@0: PREDEFINED_CONSTANTS = %w[ Chris@0: false null true undefined Chris@0: ] Chris@0: Chris@0: MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4 Chris@0: Chris@0: KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[ Chris@0: case delete in instanceof new return throw typeof with Chris@0: ] Chris@0: Chris@0: # Reserved for future use. Chris@0: RESERVED_WORDS = %w[ Chris@0: abstract boolean byte char class debugger double enum export extends Chris@0: final float goto implements import int interface long native package Chris@0: private protected public short static super synchronized throws transient Chris@0: volatile Chris@0: ] Chris@0: Chris@0: IDENT_KIND = WordList.new(:ident). Chris@0: add(RESERVED_WORDS, :reserved). Chris@0: add(PREDEFINED_CONSTANTS, :pre_constant). Chris@0: add(MAGIC_VARIABLES, :local_variable). Chris@0: add(KEYWORDS, :keyword) Chris@0: Chris@0: ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x Chris@0: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x Chris@0: REGEXP_ESCAPE = / [bBdDsSwW] /x Chris@0: STRING_CONTENT_PATTERN = { Chris@0: "'" => /[^\\']+/, Chris@0: '"' => /[^\\"]+/, Chris@0: '/' => /[^\\\/]+/, Chris@0: } Chris@0: KEY_CHECK_PATTERN = { Chris@0: "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x, Chris@0: '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x, Chris@0: } Chris@0: Chris@0: def scan_tokens tokens, options Chris@0: Chris@0: state = :initial Chris@0: string_delimiter = nil Chris@0: value_expected = true Chris@0: key_expected = false Chris@0: function_expected = false Chris@0: Chris@0: until eos? Chris@0: Chris@0: kind = nil Chris@0: match = nil Chris@0: Chris@0: case state Chris@0: Chris@0: when :initial Chris@0: Chris@0: if match = scan(/ \s+ | \\\n /x) Chris@0: value_expected = true if !value_expected && match.index(?\n) Chris@0: tokens << [match, :space] Chris@0: next Chris@0: Chris@0: elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) Chris@0: value_expected = true Chris@0: kind = :comment Chris@0: Chris@0: elsif check(/\.?\d/) Chris@0: key_expected = value_expected = false Chris@0: if scan(/0[xX][0-9A-Fa-f]+/) Chris@0: kind = :hex Chris@0: elsif scan(/(?>0[0-7]+)(?![89.eEfF])/) Chris@0: kind = :oct Chris@0: elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) Chris@0: kind = :float Chris@0: elsif scan(/\d+/) Chris@0: kind = :integer Chris@0: end Chris@0: Chris@0: elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim) Chris@0: # FIXME: scan over nested tags Chris@0: xml_scanner.tokenize match Chris@0: value_expected = false Chris@0: next Chris@0: Chris@0: elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x) Chris@0: value_expected = true Chris@0: last_operator = match[-1] Chris@0: key_expected = (last_operator == ?{) || (last_operator == ?,) Chris@0: function_expected = false Chris@0: kind = :operator Chris@0: Chris@0: elsif scan(/ [)\]}]+ /x) Chris@0: function_expected = key_expected = value_expected = false Chris@0: kind = :operator Chris@0: Chris@0: elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x) Chris@0: kind = IDENT_KIND[match] Chris@0: value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match] Chris@0: # TODO: labels Chris@0: if kind == :ident Chris@0: if match.index(?$) # $ allowed inside an identifier Chris@0: kind = :predefined Chris@0: elsif function_expected Chris@0: kind = :function Chris@0: elsif check(/\s*[=:]\s*function\b/) Chris@0: kind = :function Chris@0: elsif key_expected && check(/\s*:/) Chris@0: kind = :key Chris@0: end Chris@0: end Chris@0: function_expected = (kind == :keyword) && (match == 'function') Chris@0: key_expected = false Chris@0: Chris@0: elsif match = scan(/["']/) Chris@0: if key_expected && check(KEY_CHECK_PATTERN[match]) Chris@0: state = :key Chris@0: else Chris@0: state = :string Chris@0: end Chris@0: tokens << [:open, state] Chris@0: string_delimiter = match Chris@0: kind = :delimiter Chris@0: Chris@0: elsif value_expected && (match = scan(/\/(?=\S)/)) Chris@0: tokens << [:open, :regexp] Chris@0: state = :regexp Chris@0: string_delimiter = '/' Chris@0: kind = :delimiter Chris@0: Chris@0: elsif scan(/ \/ /x) Chris@0: value_expected = true Chris@0: key_expected = false Chris@0: kind = :operator Chris@0: Chris@0: else Chris@0: getch Chris@0: kind = :error Chris@0: Chris@0: end Chris@0: Chris@0: when :string, :regexp, :key Chris@0: if scan(STRING_CONTENT_PATTERN[string_delimiter]) Chris@0: kind = :content Chris@0: elsif match = scan(/["'\/]/) Chris@0: tokens << [match, :delimiter] Chris@0: if state == :regexp Chris@0: modifiers = scan(/[gim]+/) Chris@0: tokens << [modifiers, :modifier] if modifiers && !modifiers.empty? Chris@0: end Chris@0: tokens << [:close, state] Chris@0: string_delimiter = nil Chris@0: key_expected = value_expected = false Chris@0: state = :initial Chris@0: next Chris@0: elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)) Chris@0: if string_delimiter == "'" && !(match == "\\\\" || match == "\\'") Chris@0: kind = :content Chris@0: else Chris@0: kind = :char Chris@0: end Chris@0: elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox) Chris@0: kind = :char Chris@0: elsif scan(/\\./m) Chris@0: kind = :content Chris@0: elsif scan(/ \\ | $ /x) Chris@0: tokens << [:close, state] Chris@0: kind = :error Chris@0: key_expected = value_expected = false Chris@0: state = :initial Chris@0: else Chris@0: raise_inspect "else case \" reached; %p not handled." % peek(1), tokens Chris@0: end Chris@0: Chris@0: else Chris@0: raise_inspect 'Unknown state', tokens Chris@0: Chris@0: end Chris@0: Chris@0: match ||= matched Chris@0: if $CODERAY_DEBUG and not kind Chris@0: raise_inspect 'Error token %p in line %d' % Chris@0: [[match, kind], line], tokens Chris@0: end Chris@0: raise_inspect 'Empty token', tokens unless match Chris@0: Chris@0: tokens << [match, kind] Chris@0: Chris@0: end Chris@0: Chris@0: if [:string, :regexp].include? state Chris@0: tokens << [:close, state] Chris@0: end Chris@0: Chris@0: tokens Chris@0: end Chris@0: Chris@0: protected Chris@0: Chris@0: def reset_instance Chris@0: super Chris@0: @xml_scanner.reset if defined? @xml_scanner Chris@0: end Chris@0: Chris@0: def xml_scanner Chris@0: @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true Chris@0: end Chris@0: Chris@0: end Chris@0: Chris@0: end Chris@0: end