| Current File : //opt/alt/ruby18/lib64/ruby/1.8/rdoc/parsers/parse_rb.rb |
#!/usr/local/bin/ruby
# Parse a Ruby source file, building a set of objects
# representing the modules, classes, methods,
# requires, and includes we find (these classes
# are defined in code_objects.rb).
# This file contains stuff stolen outright from:
#
# rtags.rb -
# ruby-lex.rb - ruby lexcal analizer
# ruby-token.rb - ruby tokens
# by Keiju ISHITSUKA (Nippon Rational Inc.)
#
require "e2mmap"
require "irb/slex"
require "rdoc/code_objects"
require "rdoc/tokenstream"
require "rdoc/markup/simple_markup/preprocess"
require "rdoc/parsers/parserfactory"
$TOKEN_DEBUG = $DEBUG
# Definitions of all tokens involved in the lexical analysis
module RubyToken
EXPR_BEG = :EXPR_BEG
EXPR_MID = :EXPR_MID
EXPR_END = :EXPR_END
EXPR_ARG = :EXPR_ARG
EXPR_FNAME = :EXPR_FNAME
EXPR_DOT = :EXPR_DOT
EXPR_CLASS = :EXPR_CLASS
class Token
NO_TEXT = "??".freeze
attr :text
def initialize(line_no, char_no)
@line_no = line_no
@char_no = char_no
@text = NO_TEXT
end
# Because we're used in contexts that expect to return a token,
# we set the text string and then return ourselves
def set_text(text)
@text = text
self
end
attr_reader :line_no, :char_no, :text
end
class TkNode < Token
attr :node
end
class TkId < Token
def initialize(line_no, char_no, name)
super(line_no, char_no)
@name = name
end
attr :name
end
class TkKW < TkId
end
class TkVal < Token
def initialize(line_no, char_no, value = nil)
super(line_no, char_no)
set_text(value)
end
end
class TkOp < Token
def name
self.class.op_name
end
end
class TkOPASGN < TkOp
def initialize(line_no, char_no, op)
super(line_no, char_no)
op = TkReading2Token[op] unless op.kind_of?(Symbol)
@op = op
end
attr :op
end
class TkUnknownChar < Token
def initialize(line_no, char_no, id)
super(line_no, char_no)
@name = char_no.chr
end
attr :name
end
class TkError < Token
end
def set_token_position(line, char)
@prev_line_no = line
@prev_char_no = char
end
def Token(token, value = nil)
tk = nil
case token
when String, Symbol
source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
if (tk = source[token]).nil?
IRB.fail TkReading2TokenNoKey, token
end
tk = Token(tk[0], value)
else
tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
token.new(@prev_line_no, @prev_char_no)
else
token.new(@prev_line_no, @prev_char_no, value)
end
end
tk
end
TokenDefinitions = [
[:TkCLASS, TkKW, "class", EXPR_CLASS],
[:TkMODULE, TkKW, "module", EXPR_BEG],
[:TkDEF, TkKW, "def", EXPR_FNAME],
[:TkUNDEF, TkKW, "undef", EXPR_FNAME],
[:TkBEGIN, TkKW, "begin", EXPR_BEG],
[:TkRESCUE, TkKW, "rescue", EXPR_MID],
[:TkENSURE, TkKW, "ensure", EXPR_BEG],
[:TkEND, TkKW, "end", EXPR_END],
[:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
[:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
[:TkTHEN, TkKW, "then", EXPR_BEG],
[:TkELSIF, TkKW, "elsif", EXPR_BEG],
[:TkELSE, TkKW, "else", EXPR_BEG],
[:TkCASE, TkKW, "case", EXPR_BEG],
[:TkWHEN, TkKW, "when", EXPR_BEG],
[:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
[:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
[:TkFOR, TkKW, "for", EXPR_BEG],
[:TkBREAK, TkKW, "break", EXPR_END],
[:TkNEXT, TkKW, "next", EXPR_END],
[:TkREDO, TkKW, "redo", EXPR_END],
[:TkRETRY, TkKW, "retry", EXPR_END],
[:TkIN, TkKW, "in", EXPR_BEG],
[:TkDO, TkKW, "do", EXPR_BEG],
[:TkRETURN, TkKW, "return", EXPR_MID],
[:TkYIELD, TkKW, "yield", EXPR_END],
[:TkSUPER, TkKW, "super", EXPR_END],
[:TkSELF, TkKW, "self", EXPR_END],
[:TkNIL, TkKW, "nil", EXPR_END],
[:TkTRUE, TkKW, "true", EXPR_END],
[:TkFALSE, TkKW, "false", EXPR_END],
[:TkAND, TkKW, "and", EXPR_BEG],
[:TkOR, TkKW, "or", EXPR_BEG],
[:TkNOT, TkKW, "not", EXPR_BEG],
[:TkIF_MOD, TkKW],
[:TkUNLESS_MOD, TkKW],
[:TkWHILE_MOD, TkKW],
[:TkUNTIL_MOD, TkKW],
[:TkALIAS, TkKW, "alias", EXPR_FNAME],
[:TkDEFINED, TkKW, "defined?", EXPR_END],
[:TklBEGIN, TkKW, "BEGIN", EXPR_END],
[:TklEND, TkKW, "END", EXPR_END],
[:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
[:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
[:TkIDENTIFIER, TkId],
[:TkFID, TkId],
[:TkGVAR, TkId],
[:TkIVAR, TkId],
[:TkCONSTANT, TkId],
[:TkINTEGER, TkVal],
[:TkFLOAT, TkVal],
[:TkSTRING, TkVal],
[:TkXSTRING, TkVal],
[:TkREGEXP, TkVal],
[:TkCOMMENT, TkVal],
[:TkDSTRING, TkNode],
[:TkDXSTRING, TkNode],
[:TkDREGEXP, TkNode],
[:TkNTH_REF, TkId],
[:TkBACK_REF, TkId],
[:TkUPLUS, TkOp, "+@"],
[:TkUMINUS, TkOp, "-@"],
[:TkPOW, TkOp, "**"],
[:TkCMP, TkOp, "<=>"],
[:TkEQ, TkOp, "=="],
[:TkEQQ, TkOp, "==="],
[:TkNEQ, TkOp, "!="],
[:TkGEQ, TkOp, ">="],
[:TkLEQ, TkOp, "<="],
[:TkANDOP, TkOp, "&&"],
[:TkOROP, TkOp, "||"],
[:TkMATCH, TkOp, "=~"],
[:TkNMATCH, TkOp, "!~"],
[:TkDOT2, TkOp, ".."],
[:TkDOT3, TkOp, "..."],
[:TkAREF, TkOp, "[]"],
[:TkASET, TkOp, "[]="],
[:TkLSHFT, TkOp, "<<"],
[:TkRSHFT, TkOp, ">>"],
[:TkCOLON2, TkOp],
[:TkCOLON3, TkOp],
# [:OPASGN, TkOp], # +=, -= etc. #
[:TkASSOC, TkOp, "=>"],
[:TkQUESTION, TkOp, "?"], #?
[:TkCOLON, TkOp, ":"], #:
[:TkfLPAREN], # func( #
[:TkfLBRACK], # func[ #
[:TkfLBRACE], # func{ #
[:TkSTAR], # *arg
[:TkAMPER], # &arg #
[:TkSYMBOL, TkId], # :SYMBOL
[:TkSYMBEG, TkId],
[:TkGT, TkOp, ">"],
[:TkLT, TkOp, "<"],
[:TkPLUS, TkOp, "+"],
[:TkMINUS, TkOp, "-"],
[:TkMULT, TkOp, "*"],
[:TkDIV, TkOp, "/"],
[:TkMOD, TkOp, "%"],
[:TkBITOR, TkOp, "|"],
[:TkBITXOR, TkOp, "^"],
[:TkBITAND, TkOp, "&"],
[:TkBITNOT, TkOp, "~"],
[:TkNOTOP, TkOp, "!"],
[:TkBACKQUOTE, TkOp, "`"],
[:TkASSIGN, Token, "="],
[:TkDOT, Token, "."],
[:TkLPAREN, Token, "("], #(exp)
[:TkLBRACK, Token, "["], #[arry]
[:TkLBRACE, Token, "{"], #{hash}
[:TkRPAREN, Token, ")"],
[:TkRBRACK, Token, "]"],
[:TkRBRACE, Token, "}"],
[:TkCOMMA, Token, ","],
[:TkSEMICOLON, Token, ";"],
[:TkRD_COMMENT],
[:TkSPACE],
[:TkNL],
[:TkEND_OF_SCRIPT],
[:TkBACKSLASH, TkUnknownChar, "\\"],
[:TkAT, TkUnknownChar, "@"],
[:TkDOLLAR, TkUnknownChar, "\$"], #"
]
# {reading => token_class}
# {reading => [token_class, *opt]}
TkReading2Token = {}
TkSymbol2Token = {}
def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
token_n = token_n.id2name unless token_n.kind_of?(String)
if RubyToken.const_defined?(token_n)
IRB.fail AlreadyDefinedToken, token_n
end
token_c = Class.new super_token
RubyToken.const_set token_n, token_c
# token_c.inspect
if reading
if TkReading2Token[reading]
IRB.fail TkReading2TokenDuplicateError, token_n, reading
end
if opts.empty?
TkReading2Token[reading] = [token_c]
else
TkReading2Token[reading] = [token_c].concat(opts)
end
end
TkSymbol2Token[token_n.intern] = token_c
if token_c <= TkOp
token_c.class_eval %{
def self.op_name; "#{reading}"; end
}
end
end
for defs in TokenDefinitions
def_token(*defs)
end
NEWLINE_TOKEN = TkNL.new(0,0)
NEWLINE_TOKEN.set_text("\n")
end
# Lexical analyzer for Ruby source
class RubyLex
######################################################################
#
# Read an input stream character by character. We allow for unlimited
# ungetting of characters just read.
#
# We simplify the implementation greatly by reading the entire input
# into a buffer initially, and then simply traversing it using
# pointers.
#
# We also have to allow for the <i>here document diversion</i>. This
# little gem comes about when the lexer encounters a here
# document. At this point we effectively need to split the input
# stream into two parts: one to read the body of the here document,
# the other to read the rest of the input line where the here
# document was initially encountered. For example, we might have
#
# do_something(<<-A, <<-B)
# stuff
# for
# A
# stuff
# for
# B
#
# When the lexer encounters the <<A, it reads until the end of the
# line, and keeps it around for later. It then reads the body of the
# here document. Once complete, it needs to read the rest of the
# original line, but then skip the here document body.
#
class BufferedReader
attr_reader :line_num
def initialize(content)
if /\t/ =~ content
tab_width = Options.instance.tab_width
content = content.split(/\n/).map do |line|
1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
line
end .join("\n")
end
@content = content
@content << "\n" unless @content[-1,1] == "\n"
@size = @content.size
@offset = 0
@hwm = 0
@line_num = 1
@read_back_offset = 0
@last_newline = 0
@newline_pending = false
end
def column
@offset - @last_newline
end
def getc
return nil if @offset >= @size
ch = @content[@offset, 1]
@offset += 1
@hwm = @offset if @hwm < @offset
if @newline_pending
@line_num += 1
@last_newline = @offset - 1
@newline_pending = false
end
if ch == "\n"
@newline_pending = true
end
ch
end
def getc_already_read
getc
end
def ungetc(ch)
raise "unget past beginning of file" if @offset <= 0
@offset -= 1
if @content[@offset] == ?\n
@newline_pending = false
end
end
def get_read
res = @content[@read_back_offset...@offset]
@read_back_offset = @offset
res
end
def peek(at)
pos = @offset + at
if pos >= @size
nil
else
@content[pos, 1]
end
end
def peek_equal(str)
@content[@offset, str.length] == str
end
def divert_read_from(reserve)
@content[@offset, 0] = reserve
@size = @content.size
end
end
# end of nested class BufferedReader
extend Exception2MessageMapper
def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
def_exception(:TkReading2TokenDuplicateError,
"key duplicate(token_n='%s', key='%s')")
def_exception(:SyntaxError, "%s")
include RubyToken
include IRB
attr_reader :continue
attr_reader :lex_state
def RubyLex.debug?
false
end
def initialize(content)
lex_init
@reader = BufferedReader.new(content)
@exp_line_no = @line_no = 1
@base_char_no = 0
@indent = 0
@ltype = nil
@quoted = nil
@lex_state = EXPR_BEG
@space_seen = false
@continue = false
@line = ""
@skip_space = false
@read_auto_clean_up = false
@exception_on_syntax_error = true
end
attr :skip_space, true
attr :read_auto_clean_up, true
attr :exception_on_syntax_error, true
attr :indent
# io functions
def line_no
@reader.line_num
end
def char_no
@reader.column
end
def get_read
@reader.get_read
end
def getc
@reader.getc
end
def getc_of_rests
@reader.getc_already_read
end
def gets
c = getc or return
l = ""
begin
l.concat c unless c == "\r"
break if c == "\n"
end while c = getc
l
end
def ungetc(c = nil)
@reader.ungetc(c)
end
def peek_equal?(str)
@reader.peek_equal(str)
end
def peek(i = 0)
@reader.peek(i)
end
def lex
until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
!@continue or
tk.nil?)
end
line = get_read
if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
nil
else
line
end
end
def token
set_token_position(line_no, char_no)
begin
begin
tk = @OP.match(self)
@space_seen = tk.kind_of?(TkSPACE)
rescue SyntaxError
abort if @exception_on_syntax_error
tk = TkError.new(line_no, char_no)
end
end while @skip_space and tk.kind_of?(TkSPACE)
if @read_auto_clean_up
get_read
end
# throw :eof unless tk
p tk if $DEBUG
tk
end
ENINDENT_CLAUSE = [
"case", "class", "def", "do", "for", "if",
"module", "unless", "until", "while", "begin" #, "when"
]
DEINDENT_CLAUSE = ["end" #, "when"
]
PERCENT_LTYPE = {
"q" => "\'",
"Q" => "\"",
"x" => "\`",
"r" => "/",
"w" => "]"
}
PERCENT_PAREN = {
"{" => "}",
"[" => "]",
"<" => ">",
"(" => ")"
}
Ltype2Token = {
"\'" => TkSTRING,
"\"" => TkSTRING,
"\`" => TkXSTRING,
"/" => TkREGEXP,
"]" => TkDSTRING
}
Ltype2Token.default = TkSTRING
DLtype2Token = {
"\"" => TkDSTRING,
"\`" => TkDXSTRING,
"/" => TkDREGEXP,
}
def lex_init()
@OP = SLex.new
@OP.def_rules("\0", "\004", "\032") do |chars, io|
Token(TkEND_OF_SCRIPT).set_text(chars)
end
@OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
@space_seen = TRUE
while (ch = getc) =~ /[ \t\f\r\13]/
chars << ch
end
ungetc
Token(TkSPACE).set_text(chars)
end
@OP.def_rule("#") do
|op, io|
identify_comment
end
@OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
|op, io|
str = op
@ltype = "="
begin
line = ""
begin
ch = getc
line << ch
end until ch == "\n"
str << line
end until line =~ /^=end/
ungetc
@ltype = nil
if str =~ /\A=begin\s+rdoc/i
str.sub!(/\A=begin.*\n/, '')
str.sub!(/^=end.*/m, '')
Token(TkCOMMENT).set_text(str)
else
Token(TkRD_COMMENT)#.set_text(str)
end
end
@OP.def_rule("\n") do
print "\\n\n" if RubyLex.debug?
case @lex_state
when EXPR_BEG, EXPR_FNAME, EXPR_DOT
@continue = TRUE
else
@continue = FALSE
@lex_state = EXPR_BEG
end
Token(TkNL).set_text("\n")
end
@OP.def_rules("*", "**",
"!", "!=", "!~",
"=", "==", "===",
"=~", "<=>",
"<", "<=",
">", ">=", ">>") do
|op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
@OP.def_rules("<<") do
|op, io|
tk = nil
if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
(@lex_state != EXPR_ARG || @space_seen)
c = peek(0)
if /[-\w_\"\'\`]/ =~ c
tk = identify_here_document
end
end
if !tk
@lex_state = EXPR_BEG
tk = Token(op).set_text(op)
end
tk
end
@OP.def_rules("'", '"') do
|op, io|
identify_string(op)
end
@OP.def_rules("`") do
|op, io|
if @lex_state == EXPR_FNAME
Token(op).set_text(op)
else
identify_string(op)
end
end
@OP.def_rules('?') do
|op, io|
if @lex_state == EXPR_END
@lex_state = EXPR_BEG
Token(TkQUESTION).set_text(op)
else
ch = getc
if @lex_state == EXPR_ARG && ch !~ /\s/
ungetc
@lex_state = EXPR_BEG;
Token(TkQUESTION).set_text(op)
else
str = op
str << ch
if (ch == '\\') #'
str << read_escape
end
@lex_state = EXPR_END
Token(TkINTEGER).set_text(str)
end
end
end
@OP.def_rules("&", "&&", "|", "||") do
|op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
@OP.def_rules("+=", "-=", "*=", "**=",
"&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
|op, io|
@lex_state = EXPR_BEG
op =~ /^(.*)=$/
Token(TkOPASGN, $1).set_text(op)
end
@OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
Token(TkUPLUS).set_text(op)
end
@OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
Token(TkUMINUS).set_text(op)
end
@OP.def_rules("+", "-") do
|op, io|
catch(:RET) do
if @lex_state == EXPR_ARG
if @space_seen and peek(0) =~ /[0-9]/
throw :RET, identify_number(op)
else
@lex_state = EXPR_BEG
end
elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
throw :RET, identify_number(op)
else
@lex_state = EXPR_BEG
end
Token(op).set_text(op)
end
end
@OP.def_rule(".") do
@lex_state = EXPR_BEG
if peek(0) =~ /[0-9]/
ungetc
identify_number("")
else
# for obj.if
@lex_state = EXPR_DOT
Token(TkDOT).set_text(".")
end
end
@OP.def_rules("..", "...") do
|op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
lex_int2
end
def lex_int2
@OP.def_rules("]", "}", ")") do
|op, io|
@lex_state = EXPR_END
@indent -= 1
Token(op).set_text(op)
end
@OP.def_rule(":") do
if @lex_state == EXPR_END || peek(0) =~ /\s/
@lex_state = EXPR_BEG
tk = Token(TkCOLON)
else
@lex_state = EXPR_FNAME;
tk = Token(TkSYMBEG)
end
tk.set_text(":")
end
@OP.def_rule("::") do
# p @lex_state.id2name, @space_seen
if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
@lex_state = EXPR_BEG
tk = Token(TkCOLON3)
else
@lex_state = EXPR_DOT
tk = Token(TkCOLON2)
end
tk.set_text("::")
end
@OP.def_rule("/") do
|op, io|
if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
identify_string(op)
elsif peek(0) == '='
getc
@lex_state = EXPR_BEG
Token(TkOPASGN, :/).set_text("/=") #")
elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
identify_string(op)
else
@lex_state = EXPR_BEG
Token("/").set_text(op)
end
end
@OP.def_rules("^") do
@lex_state = EXPR_BEG
Token("^").set_text("^")
end
# @OP.def_rules("^=") do
# @lex_state = EXPR_BEG
# Token(TkOPASGN, :^)
# end
@OP.def_rules(",", ";") do
|op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
@OP.def_rule("~") do
@lex_state = EXPR_BEG
Token("~").set_text("~")
end
@OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
@lex_state = EXPR_BEG
Token("~").set_text("~@")
end
@OP.def_rule("(") do
@indent += 1
if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
@lex_state = EXPR_BEG
tk = Token(TkfLPAREN)
else
@lex_state = EXPR_BEG
tk = Token(TkLPAREN)
end
tk.set_text("(")
end
@OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
Token("[]").set_text("[]")
end
@OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
Token("[]=").set_text("[]=")
end
@OP.def_rule("[") do
@indent += 1
if @lex_state == EXPR_FNAME
t = Token(TkfLBRACK)
else
if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
t = Token(TkLBRACK)
elsif @lex_state == EXPR_ARG && @space_seen
t = Token(TkLBRACK)
else
t = Token(TkfLBRACK)
end
@lex_state = EXPR_BEG
end
t.set_text("[")
end
@OP.def_rule("{") do
@indent += 1
if @lex_state != EXPR_END && @lex_state != EXPR_ARG
t = Token(TkLBRACE)
else
t = Token(TkfLBRACE)
end
@lex_state = EXPR_BEG
t.set_text("{")
end
@OP.def_rule('\\') do #'
if getc == "\n"
@space_seen = true
@continue = true
Token(TkSPACE).set_text("\\\n")
else
ungetc
Token("\\").set_text("\\") #"
end
end
@OP.def_rule('%') do
|op, io|
if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
identify_quotation('%')
elsif peek(0) == '='
getc
Token(TkOPASGN, "%").set_text("%=")
elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
identify_quotation('%')
else
@lex_state = EXPR_BEG
Token("%").set_text("%")
end
end
@OP.def_rule('$') do #'
identify_gvar
end
@OP.def_rule('@') do
if peek(0) =~ /[@\w_]/
ungetc
identify_identifier
else
Token("@").set_text("@")
end
end
# @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
# |op, io|
# @indent += 1
# @lex_state = EXPR_FNAME
# # @lex_state = EXPR_END
# # until @rests[0] == "\n" or @rests[0] == ";"
# # rests.shift
# # end
# end
@OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
throw :eof
end
@OP.def_rule("") do
|op, io|
printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
if peek(0) =~ /[0-9]/
t = identify_number("")
elsif peek(0) =~ /[\w_]/
t = identify_identifier
end
printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
t
end
p @OP if RubyLex.debug?
end
def identify_gvar
@lex_state = EXPR_END
str = "$"
tk = case ch = getc
when /[~_*$?!@\/\\;,=:<>".]/ #"
str << ch
Token(TkGVAR, str)
when "-"
str << "-" << getc
Token(TkGVAR, str)
when "&", "`", "'", "+"
str << ch
Token(TkBACK_REF, str)
when /[1-9]/
str << ch
while (ch = getc) =~ /[0-9]/
str << ch
end
ungetc
Token(TkNTH_REF)
when /\w/
ungetc
ungetc
return identify_identifier
else
ungetc
Token("$")
end
tk.set_text(str)
end
def identify_identifier
token = ""
token.concat getc if peek(0) =~ /[$@]/
token.concat getc if peek(0) == "@"
while (ch = getc) =~ /\w|_/
print ":", ch, ":" if RubyLex.debug?
token.concat ch
end
ungetc
if ch == "!" or ch == "?"
token.concat getc
end
# fix token
# $stderr.puts "identifier - #{token}, state = #@lex_state"
case token
when /^\$/
return Token(TkGVAR, token).set_text(token)
when /^\@/
@lex_state = EXPR_END
return Token(TkIVAR, token).set_text(token)
end
if @lex_state != EXPR_DOT
print token, "\n" if RubyLex.debug?
token_c, *trans = TkReading2Token[token]
if token_c
# reserved word?
if (@lex_state != EXPR_BEG &&
@lex_state != EXPR_FNAME &&
trans[1])
# modifiers
token_c = TkSymbol2Token[trans[1]]
@lex_state = trans[0]
else
if @lex_state != EXPR_FNAME
if ENINDENT_CLAUSE.include?(token)
@indent += 1
elsif DEINDENT_CLAUSE.include?(token)
@indent -= 1
end
@lex_state = trans[0]
else
@lex_state = EXPR_END
end
end
return Token(token_c, token).set_text(token)
end
end
if @lex_state == EXPR_FNAME
@lex_state = EXPR_END
if peek(0) == '='
token.concat getc
end
elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
@lex_state = EXPR_ARG
else
@lex_state = EXPR_END
end
if token[0, 1] =~ /[A-Z]/
return Token(TkCONSTANT, token).set_text(token)
elsif token[token.size - 1, 1] =~ /[!?]/
return Token(TkFID, token).set_text(token)
else
return Token(TkIDENTIFIER, token).set_text(token)
end
end
def identify_here_document
ch = getc
if ch == "-"
ch = getc
indent = true
end
if /['"`]/ =~ ch # '
lt = ch
quoted = ""
while (c = getc) && c != lt
quoted.concat c
end
else
lt = '"'
quoted = ch.dup
while (c = getc) && c =~ /\w/
quoted.concat c
end
ungetc
end
ltback, @ltype = @ltype, lt
reserve = ""
while ch = getc
reserve << ch
if ch == "\\" #"
ch = getc
reserve << ch
elsif ch == "\n"
break
end
end
str = ""
while (l = gets)
l.chomp!
l.strip! if indent
break if l == quoted
str << l.chomp << "\n"
end
@reader.divert_read_from(reserve)
@ltype = ltback
@lex_state = EXPR_END
Token(Ltype2Token[lt], str).set_text(str.dump)
end
def identify_quotation(initial_char)
ch = getc
if lt = PERCENT_LTYPE[ch]
initial_char += ch
ch = getc
elsif ch =~ /\W/
lt = "\""
else
RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
end
# if ch !~ /\W/
# ungetc
# next
# end
#@ltype = lt
@quoted = ch unless @quoted = PERCENT_PAREN[ch]
identify_string(lt, @quoted, ch, initial_char)
end
def identify_number(start)
str = start.dup
if start == "+" or start == "-" or start == ""
start = getc
str << start
end
@lex_state = EXPR_END
if start == "0"
if peek(0) == "x"
ch = getc
str << ch
match = /[0-9a-f_]/
else
match = /[0-7_]/
end
while ch = getc
if ch !~ match
ungetc
break
else
str << ch
end
end
return Token(TkINTEGER).set_text(str)
end
type = TkINTEGER
allow_point = TRUE
allow_e = TRUE
while ch = getc
case ch
when /[0-9_]/
str << ch
when allow_point && "."
type = TkFLOAT
if peek(0) !~ /[0-9]/
ungetc
break
end
str << ch
allow_point = false
when allow_e && "e", allow_e && "E"
str << ch
type = TkFLOAT
if peek(0) =~ /[+-]/
str << getc
end
allow_e = false
allow_point = false
else
ungetc
break
end
end
Token(type).set_text(str)
end
def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
@ltype = ltype
@quoted = quoted
subtype = nil
str = ""
str << initial_char if initial_char
str << (opener||quoted)
nest = 0
begin
while ch = getc
str << ch
if @quoted == ch
if nest == 0
break
else
nest -= 1
end
elsif opener == ch
nest += 1
elsif @ltype != "'" && @ltype != "]" and ch == "#"
ch = getc
if ch == "{"
subtype = true
str << ch << skip_inner_expression
else
ungetc(ch)
end
elsif ch == '\\' #'
str << read_escape
end
end
if @ltype == "/"
if peek(0) =~ /i|o|n|e|s/
str << getc
end
end
if subtype
Token(DLtype2Token[ltype], str)
else
Token(Ltype2Token[ltype], str)
end.set_text(str)
ensure
@ltype = nil
@quoted = nil
@lex_state = EXPR_END
end
end
def skip_inner_expression
res = ""
nest = 0
while (ch = getc)
res << ch
if ch == '}'
break if nest.zero?
nest -= 1
elsif ch == '{'
nest += 1
end
end
res
end
def identify_comment
@ltype = "#"
comment = "#"
while ch = getc
if ch == "\\"
ch = getc
if ch == "\n"
ch = " "
else
comment << "\\"
end
else
if ch == "\n"
@ltype = nil
ungetc
break
end
end
comment << ch
end
return Token(TkCOMMENT).set_text(comment)
end
def read_escape
res = ""
case ch = getc
when /[0-7]/
ungetc ch
3.times do
case ch = getc
when /[0-7]/
when nil
break
else
ungetc
break
end
res << ch
end
when "x"
res << ch
2.times do
case ch = getc
when /[0-9a-fA-F]/
when nil
break
else
ungetc
break
end
res << ch
end
when "M"
res << ch
if (ch = getc) != '-'
ungetc
else
res << ch
if (ch = getc) == "\\" #"
res << ch
res << read_escape
else
res << ch
end
end
when "C", "c" #, "^"
res << ch
if ch == "C" and (ch = getc) != "-"
ungetc
else
res << ch
if (ch = getc) == "\\" #"
res << ch
res << read_escape
else
res << ch
end
end
else
res << ch
end
res
end
end
# Extract code elements from a source file, returning a TopLevel
# object containing the constituent file elements.
#
# This file is based on rtags
module RDoc
GENERAL_MODIFIERS = [ 'nodoc' ].freeze
CLASS_MODIFIERS = GENERAL_MODIFIERS
ATTR_MODIFIERS = GENERAL_MODIFIERS
CONSTANT_MODIFIERS = GENERAL_MODIFIERS
METHOD_MODIFIERS = GENERAL_MODIFIERS +
[ 'arg', 'args', 'yield', 'yields', 'notnew', 'not-new', 'not_new', 'doc' ]
class RubyParser
include RubyToken
include TokenStream
extend ParserFactory
parse_files_matching(/\.rbw?$/)
def initialize(top_level, file_name, content, options, stats)
@options = options
@stats = stats
@size = 0
@token_listeners = nil
@input_file_name = file_name
@scanner = RubyLex.new(content)
@scanner.exception_on_syntax_error = false
@top_level = top_level
@progress = $stderr unless options.quiet
end
def scan
@tokens = []
@unget_read = []
@read = []
catch(:eof) do
catch(:enddoc) do
begin
parse_toplevel_statements(@top_level)
rescue Exception => e
$stderr.puts "\n\n"
$stderr.puts "RDoc failure in #@input_file_name at or around " +
"line #{@scanner.line_no} column #{@scanner.char_no}"
$stderr.puts
$stderr.puts "Before reporting this, could you check that the file"
$stderr.puts "you're documenting compiles cleanly--RDoc is not a"
$stderr.puts "full Ruby parser, and gets confused easily if fed"
$stderr.puts "invalid programs."
$stderr.puts
$stderr.puts "The internal error was:\n\n"
e.set_backtrace(e.backtrace[0,4])
raise
end
end
end
@top_level
end
private
def make_message(msg)
prefix = "\n" + @input_file_name + ":"
if @scanner
prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
end
return prefix + msg
end
def warn(msg)
return if @options.quiet
msg = make_message msg
$stderr.puts msg
end
def error(msg)
msg = make_message msg
$stderr.puts msg
exit(1)
end
def progress(char)
unless @options.quiet
@progress.print(char)
@progress.flush
end
end
def add_token_listener(obj)
@token_listeners ||= []
@token_listeners << obj
end
def remove_token_listener(obj)
@token_listeners.delete(obj)
end
def get_tk
tk = nil
if @tokens.empty?
tk = @scanner.token
@read.push @scanner.get_read
puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
else
@read.push @unget_read.shift
tk = @tokens.shift
puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
end
if tk.kind_of?(TkSYMBEG)
set_token_position(tk.line_no, tk.char_no)
tk1 = get_tk
if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
tk = Token(TkSYMBOL).set_text(":" + tk1.name)
# remove the identifier we just read (we're about to
# replace it with a symbol)
@token_listeners.each do |obj|
obj.pop_token
end if @token_listeners
else
warn("':' not followed by identifier or operator")
tk = tk1
end
end
# inform any listeners of our shiny new token
@token_listeners.each do |obj|
obj.add_token(tk)
end if @token_listeners
tk
end
def peek_tk
unget_tk(tk = get_tk)
tk
end
def unget_tk(tk)
@tokens.unshift tk
@unget_read.unshift @read.pop
# Remove this token from any listeners
@token_listeners.each do |obj|
obj.pop_token
end if @token_listeners
end
def skip_tkspace(skip_nl = true)
tokens = []
while ((tk = get_tk).kind_of?(TkSPACE) ||
(skip_nl && tk.kind_of?(TkNL)))
tokens.push tk
end
unget_tk(tk)
tokens
end
def get_tkread
read = @read.join("")
@read = []
read
end
def peek_read
@read.join('')
end
NORMAL = "::"
SINGLE = "<<"
# Look for the first comment in a file that isn't
# a shebang line.
def collect_first_comment
skip_tkspace
res = ''
first_line = true
tk = get_tk
while tk.kind_of?(TkCOMMENT)
if first_line && /\A#!/ =~ tk.text
skip_tkspace
tk = get_tk
elsif first_line && /\A#\s*-\*-/ =~ tk.text
first_line = false
skip_tkspace
tk = get_tk
else
first_line = false
res << tk.text << "\n"
tk = get_tk
if tk.kind_of? TkNL
skip_tkspace(false)
tk = get_tk
end
end
end
unget_tk(tk)
res
end
def parse_toplevel_statements(container)
comment = collect_first_comment
look_for_directives_in(container, comment)
container.comment = comment unless comment.empty?
parse_statements(container, NORMAL, nil, comment)
end
def parse_statements(container, single=NORMAL, current_method=nil, comment='')
nest = 1
save_visibility = container.visibility
# if container.kind_of?(TopLevel)
# else
# comment = ''
# end
non_comment_seen = true
while tk = get_tk
keep_comment = false
non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
case tk
when TkNL
skip_tkspace(true) # Skip blanks and newlines
tk = get_tk
if tk.kind_of?(TkCOMMENT)
if non_comment_seen
comment = ''
non_comment_seen = false
end
while tk.kind_of?(TkCOMMENT)
comment << tk.text << "\n"
tk = get_tk # this is the newline
skip_tkspace(false) # leading spaces
tk = get_tk
end
unless comment.empty?
look_for_directives_in(container, comment)
if container.done_documenting
container.ongoing_visibility = save_visibility
# return
end
end
keep_comment = true
else
non_comment_seen = true
end
unget_tk(tk)
keep_comment = true
when TkCLASS
if container.document_children
parse_class(container, single, tk, comment)
else
nest += 1
end
when TkMODULE
if container.document_children
parse_module(container, single, tk, comment)
else
nest += 1
end
when TkDEF
if container.document_self
parse_method(container, single, tk, comment)
else
nest += 1
end
when TkCONSTANT
if container.document_self
parse_constant(container, single, tk, comment)
end
when TkALIAS
if container.document_self
parse_alias(container, single, tk, comment)
end
when TkYIELD
if current_method.nil?
warn("Warning: yield outside of method") if container.document_self
else
parse_yield(container, single, tk, current_method)
end
# Until and While can have a 'do', which shouldn't increas
# the nesting. We can't solve the general case, but we can
# handle most occurrences by ignoring a do at the end of a line
when TkUNTIL, TkWHILE
nest += 1
puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
"line #{tk.line_no}" if $DEBUG
skip_optional_do_after_expression
# 'for' is trickier
when TkFOR
nest += 1
puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
"line #{tk.line_no}" if $DEBUG
skip_for_variable
skip_optional_do_after_expression
when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
nest += 1
puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
"line #{tk.line_no}" if $DEBUG
when TkIDENTIFIER
if nest == 1 and current_method.nil?
case tk.name
when "private", "protected", "public",
"private_class_method", "public_class_method"
parse_visibility(container, single, tk)
keep_comment = true
when "attr"
parse_attr(container, single, tk, comment)
when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
parse_attr_accessor(container, single, tk, comment)
when "alias_method"
if container.document_self
parse_alias(container, single, tk, comment)
end
end
end
case tk.name
when "require"
parse_require(container, comment)
when "include"
parse_include(container, comment)
end
when TkEND
nest -= 1
puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
puts "Method = #{current_method.name}" if $DEBUG and current_method
if nest == 0
read_documentation_modifiers(container, CLASS_MODIFIERS)
container.ongoing_visibility = save_visibility
return
end
end
comment = '' unless keep_comment
begin
get_tkread
skip_tkspace(false)
end while peek_tk == TkNL
end
end
def parse_class(container, single, tk, comment, &block)
progress("c")
@stats.num_classes += 1
container, name_t = get_class_or_module(container)
case name_t
when TkCONSTANT
name = name_t.name
superclass = "Object"
if peek_tk.kind_of?(TkLT)
get_tk
skip_tkspace(true)
superclass = get_class_specification
superclass = "<unknown>" if superclass.empty?
end
if single == SINGLE
cls_type = SingleClass
else
cls_type = NormalClass
end
cls = container.add_class(cls_type, name, superclass)
read_documentation_modifiers(cls, CLASS_MODIFIERS)
cls.record_location(@top_level)
parse_statements(cls)
cls.comment = comment
when TkLSHFT
case name = get_class_specification
when "self", container.name
parse_statements(container, SINGLE, &block)
else
other = TopLevel.find_class_named(name)
unless other
# other = @top_level.add_class(NormalClass, name, nil)
# other.record_location(@top_level)
# other.comment = comment
other = NormalClass.new("Dummy", nil)
end
read_documentation_modifiers(other, CLASS_MODIFIERS)
parse_statements(other, SINGLE, &block)
end
else
warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
end
end
def parse_module(container, single, tk, comment)
progress("m")
@stats.num_modules += 1
container, name_t = get_class_or_module(container)
# skip_tkspace
name = name_t.name
mod = container.add_module(NormalModule, name)
mod.record_location(@top_level)
read_documentation_modifiers(mod, CLASS_MODIFIERS)
parse_statements(mod)
mod.comment = comment
end
# Look for the name of a class of module (optionally with a leading :: or
# with :: separated named) and return the ultimate name and container
def get_class_or_module(container)
skip_tkspace
name_t = get_tk
# class ::A -> A is in the top level
if name_t.kind_of?(TkCOLON2)
name_t = get_tk
container = @top_level
end
skip_tkspace(false)
while peek_tk.kind_of?(TkCOLON2)
prev_container = container
container = container.find_module_named(name_t.name)
if !container
# warn("Couldn't find module #{name_t.name}")
container = prev_container.add_module(NormalModule, name_t.name)
end
get_tk
name_t = get_tk
end
skip_tkspace(false)
return [container, name_t]
end
def parse_constant(container, single, tk, comment)
name = tk.name
skip_tkspace(false)
eq_tk = get_tk
unless eq_tk.kind_of?(TkASSIGN)
unget_tk(eq_tk)
return
end
nest = 0
get_tkread
tk = get_tk
if tk.kind_of? TkGT
unget_tk(tk)
unget_tk(eq_tk)
return
end
loop do
puts("Param: #{tk}, #{@scanner.continue} " +
"#{@scanner.lex_state} #{nest}") if $DEBUG
case tk
when TkSEMICOLON
break
when TkLPAREN, TkfLPAREN
nest += 1
when TkRPAREN
nest -= 1
when TkCOMMENT
if nest <= 0 && @scanner.lex_state == EXPR_END
unget_tk(tk)
break
end
when TkNL
if (@scanner.lex_state == EXPR_END and nest <= 0) || !@scanner.continue
unget_tk(tk)
break
end
end
tk = get_tk
end
res = get_tkread.tr("\n", " ").strip
res = "" if res == ";"
con = Constant.new(name, res, comment)
read_documentation_modifiers(con, CONSTANT_MODIFIERS)
if con.document_self
container.add_constant(con)
end
end
def parse_method(container, single, tk, comment)
progress(".")
@stats.num_methods += 1
line_no = tk.line_no
column = tk.char_no
start_collecting_tokens
add_token(tk)
add_token_listener(self)
@scanner.instance_eval{@lex_state = EXPR_FNAME}
skip_tkspace(false)
name_t = get_tk
back_tk = skip_tkspace
meth = nil
added_container = false
dot = get_tk
if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
@scanner.instance_eval{@lex_state = EXPR_FNAME}
skip_tkspace
name_t2 = get_tk
case name_t
when TkSELF
name = name_t2.name
when TkCONSTANT
name = name_t2.name
prev_container = container
container = container.find_module_named(name_t.name)
if !container
added_container = true
obj = name_t.name.split("::").inject(Object) do |state, item|
state.const_get(item)
end rescue nil
type = obj.class == Class ? NormalClass : NormalModule
if not [Class, Module].include?(obj.class)
warn("Couldn't find #{name_t.name}. Assuming it's a module")
end
if type == NormalClass then
container = prev_container.add_class(type, name_t.name, obj.superclass.name)
else
container = prev_container.add_module(type, name_t.name)
end
end
else
# warn("Unexpected token '#{name_t2.inspect}'")
# break
skip_method(container)
return
end
meth = AnyMethod.new(get_tkread, name)
meth.singleton = true
else
unget_tk dot
back_tk.reverse_each do
|tk|
unget_tk tk
end
name = name_t.name
meth = AnyMethod.new(get_tkread, name)
meth.singleton = (single == SINGLE)
end
remove_token_listener(self)
meth.start_collecting_tokens
indent = TkSPACE.new(1,1)
indent.set_text(" " * column)
meth.add_tokens([TkCOMMENT.new(line_no,
1,
"# File #{@top_level.file_absolute_name}, line #{line_no}"),
NEWLINE_TOKEN,
indent])
meth.add_tokens(@token_stream)
add_token_listener(meth)
@scanner.instance_eval{@continue = false}
parse_method_parameters(meth)
if meth.document_self
container.add_method(meth)
elsif added_container
container.document_self = false
end
# Having now read the method parameters and documentation modifiers, we
# now know whether we have to rename #initialize to ::new
if name == "initialize" && !meth.singleton
if meth.dont_rename_initialize
meth.visibility = :protected
else
meth.singleton = true
meth.name = "new"
meth.visibility = :public
end
end
parse_statements(container, single, meth)
remove_token_listener(meth)
# Look for a 'call-seq' in the comment, and override the
# normal parameter stuff
if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
seq = $1
seq.gsub!(/^\s*\#\s*/, '')
meth.call_seq = seq
end
meth.comment = comment
end
def skip_method(container)
meth = AnyMethod.new("", "anon")
parse_method_parameters(meth)
parse_statements(container, false, meth)
end
# Capture the method's parameters. Along the way,
# look for a comment containing
#
# # yields: ....
#
# and add this as the block_params for the method
def parse_method_parameters(method)
res = parse_method_or_yield_parameters(method)
res = "(" + res + ")" unless res[0] == ?(
method.params = res unless method.params
if method.block_params.nil?
skip_tkspace(false)
read_documentation_modifiers(method, METHOD_MODIFIERS)
end
end
def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
skip_tkspace(false)
tk = get_tk
# Little hack going on here. In the statement
# f = 2*(1+yield)
# We see the RPAREN as the next token, so we need
# to exit early. This still won't catch all cases
# (such as "a = yield + 1"
end_token = case tk
when TkLPAREN, TkfLPAREN
TkRPAREN
when TkRPAREN
return ""
else
TkNL
end
nest = 0
loop do
puts("Param: #{tk.inspect}, #{@scanner.continue} " +
"#{@scanner.lex_state} #{nest}") if $DEBUG
case tk
when TkSEMICOLON
break
when TkLBRACE
nest += 1
when TkRBRACE
# we might have a.each {|i| yield i }
unget_tk(tk) if nest.zero?
nest -= 1
break if nest <= 0
when TkLPAREN, TkfLPAREN
nest += 1
when end_token
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest <= 0
else
break unless @scanner.continue
end
when method && method.block_params.nil? && TkCOMMENT
unget_tk(tk)
read_documentation_modifiers(method, modifiers)
end
tk = get_tk
end
res = get_tkread.tr("\n", " ").strip
res = "" if res == ";"
res
end
# skip the var [in] part of a 'for' statement
def skip_for_variable
skip_tkspace(false)
tk = get_tk
skip_tkspace(false)
tk = get_tk
unget_tk(tk) unless tk.kind_of?(TkIN)
end
# while, until, and for have an optional
def skip_optional_do_after_expression
skip_tkspace(false)
tk = get_tk
case tk
when TkLPAREN, TkfLPAREN
end_token = TkRPAREN
else
end_token = TkNL
end
nest = 0
@scanner.instance_eval{@continue = false}
loop do
puts("\nWhile: #{tk}, #{@scanner.continue} " +
"#{@scanner.lex_state} #{nest}") if $DEBUG
case tk
when TkSEMICOLON
break
when TkLPAREN, TkfLPAREN
nest += 1
when TkDO
break if nest.zero?
when end_token
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest.zero?
else
break unless @scanner.continue
end
end
tk = get_tk
end
skip_tkspace(false)
if peek_tk.kind_of? TkDO
get_tk
end
end
# Return a superclass, which can be either a constant
# of an expression
def get_class_specification
tk = get_tk
return "self" if tk.kind_of?(TkSELF)
res = ""
while tk.kind_of?(TkCOLON2) ||
tk.kind_of?(TkCOLON3) ||
tk.kind_of?(TkCONSTANT)
res += tk.text
tk = get_tk
end
unget_tk(tk)
skip_tkspace(false)
get_tkread # empty out read buffer
tk = get_tk
case tk
when TkNL, TkCOMMENT, TkSEMICOLON
unget_tk(tk)
return res
end
res += parse_call_parameters(tk)
res
end
def parse_call_parameters(tk)
end_token = case tk
when TkLPAREN, TkfLPAREN
TkRPAREN
when TkRPAREN
return ""
else
TkNL
end
nest = 0
loop do
puts("Call param: #{tk}, #{@scanner.continue} " +
"#{@scanner.lex_state} #{nest}") if $DEBUG
case tk
when TkSEMICOLON
break
when TkLPAREN, TkfLPAREN
nest += 1
when end_token
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest <= 0
else
break unless @scanner.continue
end
when TkCOMMENT
unget_tk(tk)
break
end
tk = get_tk
end
res = get_tkread.tr("\n", " ").strip
res = "" if res == ";"
res
end
# Parse a constant, which might be qualified by
# one or more class or module names
def get_constant
res = ""
skip_tkspace(false)
tk = get_tk
while tk.kind_of?(TkCOLON2) ||
tk.kind_of?(TkCOLON3) ||
tk.kind_of?(TkCONSTANT)
res += tk.text
tk = get_tk
end
# if res.empty?
# warn("Unexpected token #{tk} in constant")
# end
unget_tk(tk)
res
end
# Get a constant that may be surrounded by parens
def get_constant_with_optional_parens
skip_tkspace(false)
nest = 0
while (tk = peek_tk).kind_of?(TkLPAREN) || tk.kind_of?(TkfLPAREN)
get_tk
skip_tkspace(true)
nest += 1
end
name = get_constant
while nest > 0
skip_tkspace(true)
tk = get_tk
nest -= 1 if tk.kind_of?(TkRPAREN)
end
name
end
# Directives are modifier comments that can appear after class, module,
# or method names. For example
#
# def fred # :yields: a, b
#
# or
#
# class SM # :nodoc:
#
# we return the directive name and any parameters as a two element array
def read_directive(allowed)
tk = get_tk
puts "directive: #{tk.inspect}" if $DEBUG
result = nil
if tk.kind_of?(TkCOMMENT)
if tk.text =~ /\s*:?(\w+):\s*(.*)/
directive = $1.downcase
if allowed.include?(directive)
result = [directive, $2]
end
end
else
unget_tk(tk)
end
result
end
def read_documentation_modifiers(context, allow)
dir = read_directive(allow)
case dir[0]
when "notnew", "not_new", "not-new"
context.dont_rename_initialize = true
when "nodoc"
context.document_self = false
if dir[1].downcase == "all"
context.document_children = false
end
when "doc"
context.document_self = true
context.force_documentation = true
when "yield", "yields"
unless context.params.nil?
context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
end
context.block_params = dir[1]
when "arg", "args"
context.params = dir[1]
end if dir
end
# Look for directives in a normal comment block:
#
# #-- - don't display comment from this point forward
#
#
# This routine modifies it's parameter
def look_for_directives_in(context, comment)
preprocess = SM::PreProcess.new(@input_file_name,
@options.rdoc_include)
preprocess.handle(comment) do |directive, param|
case directive
when "stopdoc"
context.stop_doc
""
when "startdoc"
context.start_doc
context.force_documentation = true
""
when "enddoc"
#context.done_documenting = true
#""
throw :enddoc
when "main"
options = Options.instance
options.main_page = param
""
when "title"
options = Options.instance
options.title = param
""
when "section"
context.set_current_section(param, comment)
comment.replace("") # 1.8 doesn't support #clear
break
else
warn "Unrecognized directive '#{directive}'"
break
end
end
remove_private_comments(comment)
end
def remove_private_comments(comment)
comment.gsub!(/^#--.*?^#\+\+/m, '')
comment.sub!(/^#--.*/m, '')
end
def get_symbol_or_name
tk = get_tk
case tk
when TkSYMBOL
tk.text.sub(/^:/, '')
when TkId, TkOp
tk.name
when TkSTRING
tk.text
else
raise "Name or symbol expected (got #{tk})"
end
end
def parse_alias(context, single, tk, comment)
skip_tkspace
if (peek_tk.kind_of? TkLPAREN)
get_tk
skip_tkspace
end
new_name = get_symbol_or_name
@scanner.instance_eval{@lex_state = EXPR_FNAME}
skip_tkspace
if (peek_tk.kind_of? TkCOMMA)
get_tk
skip_tkspace
end
old_name = get_symbol_or_name
al = Alias.new(get_tkread, old_name, new_name, comment)
read_documentation_modifiers(al, ATTR_MODIFIERS)
if al.document_self
context.add_alias(al)
end
end
def parse_yield_parameters
parse_method_or_yield_parameters
end
def parse_yield(context, single, tk, method)
if method.block_params.nil?
get_tkread
@scanner.instance_eval{@continue = false}
method.block_params = parse_yield_parameters
end
end
def parse_require(context, comment)
skip_tkspace_comment
tk = get_tk
if tk.kind_of? TkLPAREN
skip_tkspace_comment
tk = get_tk
end
name = nil
case tk
when TkSTRING
name = tk.text
# when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
# name = tk.name
when TkDSTRING
warn "Skipping require of dynamic string: #{tk.text}"
# else
# warn "'require' used as variable"
end
if name
context.add_require(Require.new(name, comment))
else
unget_tk(tk)
end
end
def parse_include(context, comment)
loop do
skip_tkspace_comment
name = get_constant_with_optional_parens
unless name.empty?
context.add_include(Include.new(name, comment))
end
return unless peek_tk.kind_of?(TkCOMMA)
get_tk
end
end
def get_bool
skip_tkspace
tk = get_tk
case tk
when TkTRUE
true
when TkFALSE, TkNIL
false
else
unget_tk tk
true
end
end
def parse_attr(context, single, tk, comment)
args = parse_symbol_arg(1)
if args.size > 0
name = args[0]
rw = "R"
skip_tkspace(false)
tk = get_tk
if tk.kind_of? TkCOMMA
rw = "RW" if get_bool
else
unget_tk tk
end
att = Attr.new(get_tkread, name, rw, comment)
read_documentation_modifiers(att, ATTR_MODIFIERS)
if att.document_self
context.add_attribute(att)
end
else
warn("'attr' ignored - looks like a variable")
end
end
def parse_visibility(container, single, tk)
singleton = (single == SINGLE)
vis = case tk.name
when "private" then :private
when "protected" then :protected
when "public" then :public
when "private_class_method"
singleton = true
:private
when "public_class_method"
singleton = true
:public
else raise "Invalid visibility: #{tk.name}"
end
skip_tkspace_comment(false)
case peek_tk
# Ryan Davis suggested the extension to ignore modifiers, because he
# often writes
#
# protected unless $TESTING
#
when TkNL, TkUNLESS_MOD, TkIF_MOD
# error("Missing argument") if singleton
container.ongoing_visibility = vis
else
args = parse_symbol_arg
container.set_visibility_for(args, vis, singleton)
end
end
def parse_attr_accessor(context, single, tk, comment)
args = parse_symbol_arg
read = get_tkread
rw = "?"
# If nodoc is given, don't document any of them
tmp = CodeObject.new
read_documentation_modifiers(tmp, ATTR_MODIFIERS)
return unless tmp.document_self
case tk.name
when "attr_reader" then rw = "R"
when "attr_writer" then rw = "W"
when "attr_accessor" then rw = "RW"
else
rw = @options.extra_accessor_flags[tk.name]
end
for name in args
att = Attr.new(get_tkread, name, rw, comment)
context.add_attribute(att)
end
end
def skip_tkspace_comment(skip_nl = true)
loop do
skip_tkspace(skip_nl)
return unless peek_tk.kind_of? TkCOMMENT
get_tk
end
end
def parse_symbol_arg(no = nil)
args = []
skip_tkspace_comment
case tk = get_tk
when TkLPAREN
loop do
skip_tkspace_comment
if tk1 = parse_symbol_in_arg
args.push tk1
break if no and args.size >= no
end
skip_tkspace_comment
case tk2 = get_tk
when TkRPAREN
break
when TkCOMMA
else
warn("unexpected token: '#{tk2.inspect}'") if $DEBUG
break
end
end
else
unget_tk tk
if tk = parse_symbol_in_arg
args.push tk
return args if no and args.size >= no
end
loop do
# skip_tkspace_comment(false)
skip_tkspace(false)
tk1 = get_tk
unless tk1.kind_of?(TkCOMMA)
unget_tk tk1
break
end
skip_tkspace_comment
if tk = parse_symbol_in_arg
args.push tk
break if no and args.size >= no
end
end
end
args
end
def parse_symbol_in_arg
case tk = get_tk
when TkSYMBOL
tk.text.sub(/^:/, '')
when TkSTRING
eval @read[-1]
else
warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
nil
end
end
end
end