X-Git-Url: http://git.roojs.org/?a=blobdiff_plain;f=JSDOC%2FTokenReader.vala;h=2244139f1f5be168d1146216654910386b706cd0;hb=e75f0105cbc65fd1c41fb8fc8b7699d8703680dc;hp=11a853a21b23a415c576e489d759de4473b75166;hpb=519a39ee5b7e241632bf5d8d98eaadecef6bf884;p=gnome.introspection-doc-generator diff --git a/JSDOC/TokenReader.vala b/JSDOC/TokenReader.vala index 11a853a..2244139 100644 --- a/JSDOC/TokenReader.vala +++ b/JSDOC/TokenReader.vala @@ -16,7 +16,7 @@ namespace JSDOC { public Gee.ArrayList tokens; public int length { - get { return this.tokens.size } + get { return this.tokens.size; } } public TokenArray() @@ -41,6 +41,14 @@ namespace JSDOC { public void push (Token t) { this.tokens.add(t); } + public Token? pop () + { + if (this.size > 0) { + return this.tokens.remove_at(this.size-1); + } + return null; + } + public Token get(int i) { return this.tokens.get(i); } @@ -288,14 +296,18 @@ namespace JSDOC { if (found === "") { return false; } - //print("WHITE = " + JSON.stringify(found)); + //print("WHITE = " + JSON.stringify(found)); + + if (this.collapseWhite) { - found = " "; // this might work better if it was a ';' ??? + found = " "; // this might work better if it was a '\n' ??? + } + if (this.keepWhite) { + tokens.push(new Token(found, "WHIT", "SPACE", this.line)); } - if (this.keepWhite) tokens.push(new Token(found, "WHIT", "SPACE", this.line)); return true; - }, + } /** @returns {Boolean} Was the token found? @@ -303,7 +315,7 @@ namespace JSDOC { public bool read_newline (TokenStream stream, TokenArray tokens) var found = ""; var line = this.line; - while (!stream.look().eof && Lang.isNewline(stream.look())) { + while (!stream.lookEOF() && Lang.isNewline(stream.look())) { this.line++; found += stream.next(); } @@ -311,16 +323,22 @@ namespace JSDOC { if (found === "") { return false; } + + // if we found a new line, then we could check if previous character was a ';' - if so we can drop it. + // otherwise generally keep it.. in which case it should reduce our issue with stripping new lines.. + + //this.line++; if (this.collapseWhite) { - found = "\n"; + found = "\n"; // reduces multiple line breaks into a single one... } - if (this.keepWhite) { - var last = tokens ? tokens.pop() : false; - if (last && last.name != "WHIT") { + + if (this.keepWhite) { + var last = tokens.pop(); + if (last != null && last.name != "WHIT") { tokens.push(last); } - + // replaces last new line... tokens.push(new Token(found, "WHIT", "NEWLINE", line)); } return true; @@ -329,42 +347,54 @@ namespace JSDOC { /** @returns {Boolean} Was the token found? */ - read_mlcomment : function(/**JSDOC.TokenStream*/stream, tokens) { - if (stream.look() == "/" && stream.look(1) == "*") { - var found = stream.next(2); - var c = ''; - var line = this.line; - while (!stream.look().eof && !(stream.look(-1) == "/" && stream.look(-2) == "*")) { - c = stream.next(); - if (c == "\n") this.line++; - found += c; + public bool read_mlcomment (TokenStream stream, TokenArray tokens) + { + if (stream.look() != "/") { + return false; + } + if (stream.look(1) != "*") { + return false; + } + var found = stream.next(2); + var c = ''; + var line = this.line; + while (!stream.lookEOF() && !(stream.look(-1) == "/" && stream.look(-2) == "*")) { + c = stream.next(); + if (c == "\n") { + this.line++; } - - // to start doclet we allow /** or /*** but not /**/ or /**** - if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new Token(found, "COMM", "JSDOC", this.line)); - else if (this.keepComments) tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", line)); - return true; + found += c; } - return false; - }, + + // to start doclet we allow /** or /*** but not /**/ or /**** + //if (found.length /^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) { + if ((this.keepDocs && found.length > 4 && found.index_of("/**") == 0 && found[3] != "/") { + tokens.push(new Token(found, "COMM", "JSDOC", this.line)); + } else if (this.keepComments) { + tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", line)); + } + return true; + + } /** @returns {Boolean} Was the token found? */ - read_slcomment : function(/**JSDOC.TokenStream*/stream, tokens) { - var found; + public bool read_slcomment (TokenStream stream, TokenArray tokens) + { + var found = ""; if ( (stream.look() == "/" && stream.look(1) == "/" && (found=stream.next(2))) || (stream.look() == "<" && stream.look(1) == "!" && stream.look(2) == "-" && stream.look(3) == "-" && (found=stream.next(4))) ) { var line = this.line; - while (!stream.look().eof && !Lang.isNewline(stream.look())) { + while (!stream.lookEOF() && !Lang.isNewline(stream.look())) { found += stream.next(); } - if (!stream.look().eof) { + //if (!stream.lookEOF()) { // what? << eat the EOL? found += stream.next(); - } + //} if (this.keepComments) { tokens.push(new Token(found, "COMM", "SINGLE_LINE_COMM", line)); } @@ -372,128 +402,114 @@ namespace JSDOC { return true; } return false; - }, + } /** @returns {Boolean} Was the token found? */ - read_dbquote : function(/**JSDOC.TokenStream*/stream, tokens) { - if (stream.look() == "\"") { + public bool read_dbquote (TokenStream stream, TokenArray tokens) + { + if (stream.look() != "\"") { + return false; + } // find terminator - var string = stream.next(); - - while (!stream.look().eof) { - if (stream.look() == "\\") { - if (Lang.isNewline(stream.look(1))) { - do { - stream.next(); - } while (!stream.look().eof && Lang.isNewline(stream.look())); - string += "\\\n"; - } - else { - string += stream.next(2); - } - } - else if (stream.look() == "\"") { - string += stream.next(); - tokens.push(new Token(string, "STRN", "DOUBLE_QUOTE", this.line)); - return true; + var str = stream.next(); + + while (!stream.lookEOF()) { + if (stream.look() == "\\") { + if (Lang.isNewline(stream.look(1))) { + do { + stream.next(); + } while (!stream.lookEOF() && Lang.isNewline(stream.look())); + str += "\\\n"; } else { - string += stream.next(); + str += stream.next(2); } + continue; + } + if (stream.look() == "\"") { + str += stream.next(); + tokens.push(new Token(str, "STRN", "DOUBLE_QUOTE", this.line)); + return true; } + + str += stream.next(); + } - return false; // error! unterminated string + return false; }, /** @returns {Boolean} Was the token found? */ - read_snquote : function(/**JSDOC.TokenStream*/stream, tokens) { - if (stream.look() == "'") { - // find terminator - var string = stream.next(); - - while (!stream.look().eof) { - if (stream.look() == "\\") { // escape sequence - string += stream.next(2); - } - else if (stream.look() == "'") { - string += stream.next(); - tokens.push(new Token(string, "STRN", "SINGLE_QUOTE", this.line)); - return true; - } - else { - string += stream.next(); - } + public bool read_snquote (TokenStream stream, TokenArray tokens) + { + if (stream.look() != "'") { + return false; + } + // find terminator + var str = stream.next(); + + while (!stream.look().eof) { + if (stream.look() == "\\") { // escape sequence + str += stream.next(2); + continue; + } + if (stream.look() == "'") { + str += stream.next(); + tokens.push(new Token(str, "STRN", "SINGLE_QUOTE", this.line)); + return true; } + str += stream.next(); + } - return false; // error! unterminated string - }, + return false; + } + /** @returns {Boolean} Was the token found? */ - read_numb : function(/**JSDOC.TokenStream*/stream, tokens) { + public bool read_numb (TokenStream stream, TokenArray tokens) + { if (stream.look() === "0" && stream.look(1) == "x") { return this.read_hex(stream, tokens); } var found = ""; - while (!stream.look().eof && Lang.isNumber(found+stream.look())){ + while (!stream.lookEOF() && Lang.isNumber(found+stream.look())){ found += stream.next(); } if (found === "") { return false; } - else { - if (/^0[0-7]/.test(found)) tokens.push(new Token(found, "NUMB", "OCTAL", this.line)); - else tokens.push(new Token(found, "NUMB", "DECIMAL", this.line)); + if (GLib.Regex.match_simple("^0[0-7]", found)) { + tokens.push(new Token(found, "NUMB", "OCTAL", this.line)); return true; } - }, - /*t: - requires("../lib/JSDOC/TextStream.js"); - requires("../lib/JSDOC/Token.js"); - requires("../lib/JSDOC/Lang.js"); - - plan(3, "testing read_numb"); - - //// setup - var src = "function foo(num){while (num+8.0 >= 0x20 && num < 0777){}}"; - var tr = new TokenReader(); - var tokens = tr.tokenize(new TextStream(src)); - - var hexToken, octToken, decToken; - for (var i = 0; i < tokens.length; i++) { - if (tokens[i].name == "HEX_DEC") hexToken = tokens[i]; - if (tokens[i].name == "OCTAL") octToken = tokens[i]; - if (tokens[i].name == "DECIMAL") decToken = tokens[i]; - } - //// - - is(decToken.data, "8.0", "decimal number is found in source."); - is(hexToken.data, "0x20", "hexdec number is found in source (issue #99)."); - is(octToken.data, "0777", "octal number is found in source."); - */ - + tokens.push(new Token(found, "NUMB", "DECIMAL", this.line)); + return true; + + } + /** @returns {Boolean} Was the token found? */ - read_hex : function(/**JSDOC.TokenStream*/stream, tokens) { + public bool read_hex (TokenStream stream, TokenArray tokens) + { var found = stream.next(2); - while (!stream.look().eof) { + while (!stream.lookEOF()) { if (Lang.isHexDec(found) && !Lang.isHexDec(found+stream.look())) { // done tokens.push(new Token(found, "NUMB", "HEX_DEC", this.line)); return true; } - else { - found += stream.next(); - } + + found += stream.next(); + } return false; }, @@ -501,47 +517,47 @@ namespace JSDOC { /** @returns {Boolean} Was the token found? */ - read_regx : function(/**JSDOC.TokenStream*/stream, tokens) { - var last; + public bool read_regx (TokenStream stream, TokenArray tokens) + { + Token last; + if (stream.look() != "/") { + return false; + } + var last = tokens.lastSym(); if ( - stream.look() == "/" - && + (last == null) + || ( - - ( - !(last = tokens.lastSym()) // there is no last, the regex is the first symbol - || - ( - !last.is("NUMB") - && !last.is("NAME") - && !last.is("RIGHT_PAREN") - && !last.is("RIGHT_BRACKET") - ) - ) + !last.is("NUMB") // stuff that can not appear before a regex.. + && !last.is("NAME") + && !last.is("RIGHT_PAREN") + && !last.is("RIGHT_BRACKET") ) - ) { + ) { var regex = stream.next(); - while (!stream.look().eof) { + while (!stream.lookEOF()) { if (stream.look() == "\\") { // escape sequence regex += stream.next(2); + continue; } - else if (stream.look() == "/") { + if (stream.look() == "/") { regex += stream.next(); - while (/[gmi]/.test(stream.look())) { + while (GLib.Regex.match_simple("[gmi]", stream.look()) { regex += stream.next(); } tokens.push(new Token(regex, "REGX", "REGX", this.line)); return true; } - else { - regex += stream.next(); - } + + regex += stream.next(); + } // error: unterminated regex } return false; } -}); \ No newline at end of file + } +} \ No newline at end of file