SoundSoftware Code Site

module CodeRay

  # GZip library for writing and reading token dumps.

  autoload :GZip, 'coderay/helpers/gzip'

  # = Tokens  TODO: Rewrite!

  #

  # The Tokens class represents a list of tokens returnd from

  # a Scanner.

  #

  # A token is not a special object, just a two-element Array

  # consisting of

  # * the _token_ _text_ (the original source of the token in a String) or

  #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)

  # * the _token_ _kind_ (a Symbol representing the type of the token)

  #

  # A token looks like this:

  #

  #   ['# It looks like this', :comment]

  #   ['3.1415926', :float]

  #   ['$^', :error]

  #

  # Some scanners also yield sub-tokens, represented by special

  # token actions, namely begin_group and end_group.

  #

  # The Ruby scanner, for example, splits "a string" into:

  #

  #  [

  #   [:begin_group, :string],

  #   ['"', :delimiter],

  #   ['a string', :content],

  #   ['"', :delimiter],

  #   [:end_group, :string]

  #  ]

  #

  # Tokens is the interface between Scanners and Encoders:

  # The input is split and saved into a Tokens object. The Encoder

  # then builds the output from this object.

  #

  # Thus, the syntax below becomes clear:

  #

  #   CodeRay.scan('price = 2.59', :ruby).html

  #   # the Tokens object is here -------^

  #

  # See how small it is? ;)

  #

  # Tokens gives you the power to handle pre-scanned code very easily:

  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string

  # that you put in your DB.

  # 

  # It also allows you to generate tokens directly (without using a scanner),

  # to load them from a file, and still use any Encoder that CodeRay provides.

  class Tokens < Array

    # The Scanner instance that created the tokens.

    attr_accessor :scanner

    # Encode the tokens using encoder.

    #

    # encoder can be

    # * a symbol like :html oder :statistic

    # * an Encoder class

    # * an Encoder object

    #

    # options are passed to the encoder.

    def encode encoder, options = {}

      encoder = Encoders[encoder].new options if encoder.respond_to? :to_sym

      encoder.encode_tokens self, options

    end

    # Turn tokens into a string by concatenating them.

    def to_s

      encode CodeRay::Encoders::Encoder.new

    end

    # Redirects unknown methods to encoder calls.

    #

    # For example, if you call +tokens.html+, the HTML encoder

    # is used to highlight the tokens.

    def method_missing meth, options = {}

      encode meth, options

    rescue PluginHost::PluginNotFound

      super

    end

    # Split the tokens into parts of the given +sizes+.

    # 

    # The result will be an Array of Tokens objects. The parts have

    # the text size specified by the parameter. In addition, each

    # part closes all opened tokens. This is useful to insert tokens

    # betweem them.

    # 

    # This method is used by @Scanner#tokenize@ when called with an Array

    # of source strings. The Diff encoder uses it for inline highlighting.

    def split_into_parts *sizes

      parts = []

      opened = []

      content = nil

      part = Tokens.new

      part_size = 0

      size = sizes.first

      i = 0

      for item in self

        case content

        when nil

          content = item

        when String

          if size && part_size + content.size > size  # token must be cut

            if part_size < size  # some part of the token goes into this part

              content = content.dup  # content may no be safe to change

              part << content.slice!(0, size - part_size) << item

            end

            # close all open groups and lines...

            closing = opened.reverse.flatten.map do |content_or_kind|

              case content_or_kind

              when :begin_group

                :end_group

              when :begin_line

                :end_line

              else

                content_or_kind

              end

            end

            part.concat closing

            begin

              parts << part

              part = Tokens.new

              size = sizes[i += 1]

            end until size.nil? || size > 0

            # ...and open them again.

            part.concat opened.flatten

            part_size = 0

            redo unless content.empty?

          else

            part << content << item

            part_size += content.size

          end

          content = nil

        when Symbol

          case content

          when :begin_group, :begin_line

            opened << [content, item]

          when :end_group, :end_line

            opened.pop

          else

            raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item]

          end

          part << content << item

          content = nil

        else

          raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item]

        end

      end

      parts << part

      parts << Tokens.new while parts.size < sizes.size

      parts

    end

    # Dumps the object into a String that can be saved

    # in files or databases.

    #

    # The dump is created with Marshal.dump;

    # In addition, it is gzipped using GZip.gzip.

    #

    # The returned String object includes Undumping

    # so it has an #undump method. See Tokens.load.

    #

    # You can configure the level of compression,

    # but the default value 7 should be what you want

    # in most cases as it is a good compromise between

    # speed and compression rate.

    #

    # See GZip module.

    def dump gzip_level = 7

      dump = Marshal.dump self

      dump = GZip.gzip dump, gzip_level

      dump.extend Undumping

    end

    # Return the actual number of tokens.

    def count

      size / 2

    end

    # Include this module to give an object an #undump

    # method.

    #

    # The string returned by Tokens.dump includes Undumping.

    module Undumping

      # Calls Tokens.load with itself.

      def undump

        Tokens.load self

      end

    end

    # Undump the object using Marshal.load, then

    # unzip it using GZip.gunzip.

    #

    # The result is commonly a Tokens object, but

    # this is not guaranteed.

    def Tokens.load dump

      dump = GZip.gunzip dump

      @dump = Marshal.load dump

    end

    alias text_token push

    def begin_group kind; push :begin_group, kind end

    def end_group kind; push :end_group, kind end

    def begin_line kind; push :begin_line, kind end

    def end_line kind; push :end_line, kind end

    alias tokens concat

  end

end