X-Git-Url: http://git.roojs.org/?a=blobdiff_plain;f=JSDOC%2FTokenReader.js;h=c38e13b2b70c1260568277e6ca7fa65ae54bbd4c;hb=fe8b15c8053ee83103669ffcc032474ea41e6b38;hp=6a71e1925083fbc25e858a282f6dcc623d157b0e;hpb=f3b4a6ed633b24553d11767e6d514df748ca6b27;p=gnome.introspection-doc-generator diff --git a/JSDOC/TokenReader.js b/JSDOC/TokenReader.js index 6a71e19..c38e13b 100644 --- a/JSDOC/TokenReader.js +++ b/JSDOC/TokenReader.js @@ -14,17 +14,25 @@ Lang = imports.Lang.Lang; TokenReader = XObject.define( function(o) { - this.keepDocs = true; - this.keepWhite = false; - this.keepComments = false; - this.sepIdents = false; // seperate '.' in identifiers.. XObject.extend(this, o || {}); }, Object, { + /** @cfg {Boolean} collapseWhite merge multiple whitespace/comments into a single token **/ collapseWhite : false, // only reduces white space... - + /** @cfg {Boolean} keepDocs keep JSDOC comments **/ + keepDocs : true, + /** @cfg {Boolean} keepWhite keep White space **/ + keepWhite : false, + /** @cfg {Boolean} keepComments keep all comments **/ + keepComments : false, + /** @cfg {Boolean} sepIdents seperate identifiers (eg. a.b.c into ['a', '.', 'b', '.', 'c'] ) **/ + sepIdents : false, + /** @cfg {String} filename name of file being parsed. **/ + filename : '', + /** @config {Boolean} ignoreBadGrammer do not throw errors if we find stuff that might break compression **/ + ignoreBadGrammer : false, /** * tokenize a stream * @return {Array} of tokens @@ -34,14 +42,13 @@ TokenReader = XObject.define( * tr.tokenize(ts) * */ - - - tokenize : function(/**JSDOC.TextStream*/stream) { this.line =1; var tokens = []; - /**@ignore*/ tokens.last = function() { return tokens[tokens.length-1]; } - /**@ignore*/ tokens.lastSym = function() { + /**@ignore*/ + tokens.last = function() { return tokens[tokens.length-1]; } + /**@ignore*/ + tokens.lastSym = function() { for (var i = tokens.length-1; i >= 0; i--) { if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i]; } @@ -68,6 +75,57 @@ TokenReader = XObject.define( return tokens; }, + /** + * findPuncToken - find the id of a token (previous to current) + * need to back check syntax.. + * + * @arg {Array} tokens the array of tokens. + * @arg {String} token data (eg. '(') + * @arg {Number} offset where to start reading from + * @return {Number} position of token + */ + findPuncToken : function(tokens, data, n) { + n = n || tokens.length -1; + var stack = 0; + while (n > -1) { + + if (!stack && tokens[n].data == data) { + return n; + } + + if (tokens[n].data == ')' || tokens[n].data == '}') { + stack++; + n--; + continue; + } + if (stack && (tokens[n].data == '{' || tokens[n].data == '(')) { + stack--; + n--; + continue; + } + + + n--; + } + return -1; + }, + /** + * lastSym - find the last token symbol + * need to back check syntax.. + * + * @arg {Array} tokens the array of tokens. + * @arg {Number} offset where to start.. + * @return {Token} the token + */ + lastSym : function(tokens, n) { + for (var i = n-1; i >= 0; i--) { + if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i]; + } + return null; + }, + + + /** @returns {Boolean} Was the token found? */ @@ -80,29 +138,51 @@ TokenReader = XObject.define( if (found === "") { return false; } - else { - var name; - if ((name = Lang.keyword(found))) { - tokens.push(new Token(found, "KEYW", name, this.line)); - return true; - } - if (!this.sepIdents || found.indexOf('.') < 0 ) { - tokens.push(new Token(found, "NAME", "NAME", this.line)); - return true; - } - var n = found.split('.'); - var p = false; - var _this = this; - n.forEach(function(nm) { - if (p) { - tokens.push(new Token('.', "PUNC", "DOT", _this.line)); + + var name; + if ((name = Lang.keyword(found))) { + if (found == 'return' && tokens.lastSym().data == ')') { + //Seed.print('@' + tokens.length); + var n = this.findPuncToken(tokens, ')'); + //Seed.print(')@' + n); + n = this.findPuncToken(tokens, '(', n-1); + //Seed.print('(@' + n); + + var lt = this.lastSym(tokens, n); + Seed.print(JSON.stringify(lt)); + if (lt.type != 'KEYW' || ['IF', 'WHILE'].indexOf(lt.name) < -1) { + if (!this.ignoreBadGrammer) { + throw { + name : "ArgumentError", + message: "\n" + this.filename + ':' + this.line + " Error - return found after )" + } + } } - p=true; - tokens.push(new Token(nm, "NAME", "NAME", _this.line)); - }); - return true; + + + + } + tokens.push(new Token(found, "KEYW", name, this.line)); + return true; } + if (!this.sepIdents || found.indexOf('.') < 0 ) { + tokens.push(new Token(found, "NAME", "NAME", this.line)); + return true; + } + var n = found.split('.'); + var p = false; + var _this = this; + n.forEach(function(nm) { + if (p) { + tokens.push(new Token('.', "PUNC", "DOT", _this.line)); + } + p=true; + tokens.push(new Token(nm, "NAME", "NAME", _this.line)); + }); + return true; + + }, /** @@ -115,13 +195,28 @@ TokenReader = XObject.define( found += stream.next(); } + if (found === "") { return false; } - else { - tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line)); - return true; + + if ((found == '}' || found == ']') && tokens.lastSym().data == ',') { + //print("Error - comma found before " + found); + //print(JSON.stringify(tokens.lastSym(), null,4)); + if (this.ignoreBadGrammer) { + print("\n" + this.filename + ':' + this.line + " Error - comma found before " + found); + } else { + + throw { + name : "ArgumentError", + message: "\n" + this.filename + ':' + this.line + " Error - comma found before " + found + } + } } + + tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line)); + return true; + }, /** @@ -149,7 +244,7 @@ TokenReader = XObject.define( */ read_newline : function(/**JSDOC.TokenStream*/stream, tokens) { var found = ""; - + var line = this.line; while (!stream.look().eof && Lang.isNewline(stream.look())) { this.line++; found += stream.next(); @@ -164,11 +259,11 @@ TokenReader = XObject.define( } if (this.keepWhite) { var last = tokens.pop(); - if (last.name != "WHIT") { + if (last && last.name != "WHIT") { tokens.push(last); } - tokens.push(new Token(found, "WHIT", "NEWLINE", this.line)); + tokens.push(new Token(found, "WHIT", "NEWLINE", line)); } return true; },