X-Git-Url: http://git.roojs.org/?a=blobdiff_plain;f=JSDOC%2FTokenReader.js;h=f9aa83109eaf78eef1506976044c429d9a596a54;hb=05141d13558c377a970de1e5771865a567ac2526;hp=ad4b3712347c521e57a4b341b00fde09cce5b362;hpb=54b836da976a02d69134a31acd75345eb84ad05a;p=gnome.introspection-doc-generator diff --git a/JSDOC/TokenReader.js b/JSDOC/TokenReader.js index ad4b371..f9aa831 100644 --- a/JSDOC/TokenReader.js +++ b/JSDOC/TokenReader.js @@ -14,17 +14,24 @@ Lang = imports.Lang.Lang; TokenReader = XObject.define( function(o) { - this.keepDocs = true; - this.keepWhite = false; - this.keepComments = false; - this.sepIdents = false; // seperate '.' in identifiers.. XObject.extend(this, o || {}); }, Object, { - - + /** @cfg {Boolean} collapseWhite merge multiple whitespace/comments into a single token **/ + collapseWhite : false, // only reduces white space... + /** @cfg {Boolean} keepDocs keep JSDOC comments **/ + keepDocs : true, + /** @cfg {Boolean} keepWhite keep White space **/ + keepWhite : false, + /** @cfg {Boolean} keepComments keep all comments **/ + keepComments : false, + /** @cfg {Boolean} sepIdents seperate identifiers (eg. a.b.c into ['a', '.', 'b', '.', 'c'] ) **/ + sepIdents : false, + /** @cfg {String} filename name of file being parsed. **/ + filename : '', + /** * tokenize a stream * @return {Array} of tokens @@ -40,8 +47,10 @@ TokenReader = XObject.define( tokenize : function(/**JSDOC.TextStream*/stream) { this.line =1; var tokens = []; - /**@ignore*/ tokens.last = function() { return tokens[tokens.length-1]; } - /**@ignore*/ tokens.lastSym = function() { + /**@ignore*/ + tokens.last = function() { return tokens[tokens.length-1]; } + /**@ignore*/ + tokens.lastSym = function() { for (var i = tokens.length-1; i >= 0; i--) { if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i]; } @@ -80,28 +89,36 @@ TokenReader = XObject.define( if (found === "") { return false; } - else { - var name; - if ((name = Lang.keyword(found))) { - tokens.push(new Token(found, "KEYW", name, this.line)); - return true; - } - if (!this.sepIdents || found.indexOf('.') < 0 ) { - tokens.push(new Token(found, "NAME", "NAME", this.line)); - return true; - } - var n = found.split('.'); - var p = false; - n.forEach(function(nm) { - if (p) { - tokens.push(new Token('.', "PUNC", "DOT", this.line)); - } - p=true; - tokens.push(new Token(nm, "NAME", "NAME", this.line)); - }); - return true; + + var name; + if ((name = Lang.keyword(found))) { + if (found == 'return' && tokens.lastSym().data == ')') { + throw { + name : "ArgumentError", + message: "\n" + this.filename + ':' + this.line + " Error - return found after )" + } + + tokens.push(new Token(found, "KEYW", name, this.line)); + return true; } + if (!this.sepIdents || found.indexOf('.') < 0 ) { + tokens.push(new Token(found, "NAME", "NAME", this.line)); + return true; + } + var n = found.split('.'); + var p = false; + var _this = this; + n.forEach(function(nm) { + if (p) { + tokens.push(new Token('.', "PUNC", "DOT", _this.line)); + } + p=true; + tokens.push(new Token(nm, "NAME", "NAME", _this.line)); + }); + return true; + + }, /** @@ -114,13 +131,23 @@ TokenReader = XObject.define( found += stream.next(); } + if (found === "") { return false; } - else { - tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line)); - return true; + + if ((found == '}' || found == ']') && tokens.lastSym().data == ',') { + //print("Error - comma found before " + found); + //print(JSON.stringify(tokens.lastSym(), null,4)); + throw { + name : "ArgumentError", + message: "\n" + this.filename + ':' + this.line + " Error - comma found before " + found + } } + + tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line)); + return true; + }, /** @@ -129,18 +156,18 @@ TokenReader = XObject.define( read_space : function(/**JSDOC.TokenStream*/stream, tokens) { var found = ""; - while (!stream.look().eof && Lang.isSpace(stream.look())) { + while (!stream.look().eof && Lang.isSpace(stream.look()) && !Lang.isNewline(stream.look())) { found += stream.next(); } if (found === "") { return false; } - else { - if (this.collapseWhite) found = " "; - if (this.keepWhite) tokens.push(new Token(found, "WHIT", "SPACE", this.line)); - return true; - } + //print("WHITE = " + JSON.stringify(found)); + if (this.collapseWhite) found = " "; + if (this.keepWhite) tokens.push(new Token(found, "WHIT", "SPACE", this.line)); + return true; + }, /** @@ -148,7 +175,7 @@ TokenReader = XObject.define( */ read_newline : function(/**JSDOC.TokenStream*/stream, tokens) { var found = ""; - + var line = this.line; while (!stream.look().eof && Lang.isNewline(stream.look())) { this.line++; found += stream.next(); @@ -157,13 +184,19 @@ TokenReader = XObject.define( if (found === "") { return false; } - else { - if (this.collapseWhite) found = "\n"; - if (this.keepWhite) { - tokens.push(new Token(found, "WHIT", "NEWLINE", this.line)); + //this.line++; + if (this.collapseWhite) { + found = "\n"; + } + if (this.keepWhite) { + var last = tokens.pop(); + if (last && last.name != "WHIT") { + tokens.push(last); } - return true; + + tokens.push(new Token(found, "WHIT", "NEWLINE", line)); } + return true; }, /** @@ -173,6 +206,7 @@ TokenReader = XObject.define( if (stream.look() == "/" && stream.look(1) == "*") { var found = stream.next(2); var c = ''; + var line = this.line; while (!stream.look().eof && !(stream.look(-1) == "/" && stream.look(-2) == "*")) { c = stream.next(); if (c == "\n") this.line++; @@ -181,7 +215,7 @@ TokenReader = XObject.define( // to start doclet we allow /** or /*** but not /**/ or /**** if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new Token(found, "COMM", "JSDOC", this.line)); - else if (this.keepComments) tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", this.line)); + else if (this.keepComments) tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", line)); return true; } return false; @@ -197,13 +231,15 @@ TokenReader = XObject.define( || (stream.look() == "<" && stream.look(1) == "!" && stream.look(2) == "-" && stream.look(3) == "-" && (found=stream.next(4))) ) { - + var line = this.line; while (!stream.look().eof && !Lang.isNewline(stream.look())) { found += stream.next(); } - + if (!stream.look().eof) { + found += stream.next(); + } if (this.keepComments) { - tokens.push(new Token(found, "COMM", "SINGLE_LINE_COMM", this.line)); + tokens.push(new Token(found, "COMM", "SINGLE_LINE_COMM", line)); } this.line++; return true;