JSDOC/TokenReader.js

[gnome.introspection-doc-generator] / JSDOC / TokenReader.js

diff --git a/JSDOC/TokenReader.js b/JSDOC/TokenReader.js
index 21db9f0..5e5cec8 100644 (file)
--- a/JSDOC/TokenReader.js
+++ b/JSDOC/TokenReader.js
@@ -1,17 +1,17 @@
 //<script type="text/javascript">
 
  
-XObject = imports.XObject.XObject;
-console = imports.console.console;
+const XObject = imports.XObject.XObject;
+const console = imports.console.console;
 
 
-Token   = imports.Token.Token;
-Lang    = imports.Lang.Lang;
+const Token   = imports.Token.Token;
+const Lang    = imports.Lang.Lang;
 
 /**
        @class Search a {@link JSDOC.TextStream} for language tokens.
 */
-TokenReader = XObject.define(
+const TokenReader = XObject.define(
     function(o) {
         
         XObject.extend(this, o || {});
@@ -31,7 +31,8 @@ TokenReader = XObject.define(
         sepIdents : false,
         /** @cfg {String} filename name of file being parsed. **/
         filename : '',
-        
+        /** @config {Boolean} ignoreBadGrammer do not throw errors if we find stuff that might break compression **/
+        ignoreBadGrammer : false,
         /**
          * tokenize a stream
          * @return {Array} of tokens
@@ -41,9 +42,6 @@ TokenReader = XObject.define(
          * tr.tokenize(ts)
          * 
          */
-            
-
-
         tokenize : function(/**JSDOC.TextStream*/stream) {
             this.line =1;
             var tokens = [];
@@ -54,6 +52,7 @@ TokenReader = XObject.define(
                 for (var i = tokens.length-1; i >= 0; i--) {
                     if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i];
                 }
+                return true;
             }
 
             while (!stream.look().eof) {
@@ -77,6 +76,57 @@ TokenReader = XObject.define(
             return tokens;
         },
 
+        /**
+         * findPuncToken - find the id of a token (previous to current)
+         * need to back check syntax..
+         * 
+         * @arg {Array} tokens the array of tokens.
+         * @arg {String} token data (eg. '(')
+         * @arg {Number} offset where to start reading from
+         * @return {Number} position of token
+         */
+        findPuncToken : function(tokens, data, n) {
+            n = n || tokens.length -1;
+            var stack = 0;
+            while (n > -1) {
+                
+                if (!stack && tokens[n].data == data) {
+                    return n;
+                }
+                
+                if (tokens[n].data  == ')' || tokens[n].data  == '}') {
+                    stack++;
+                    n--;
+                    continue;
+                }
+                if (stack && (tokens[n].data  == '{' || tokens[n].data  == '(')) {
+                    stack--;
+                    n--;
+                    continue;
+                }
+                
+                
+                n--;
+            }
+            return -1;
+        },
+        /**
+         * lastSym - find the last token symbol
+         * need to back check syntax..
+         * 
+         * @arg {Array} tokens the array of tokens.
+         * @arg {Number} offset where to start..
+         * @return {Token} the token
+         */
+        lastSym : function(tokens, n) {
+            for (var i = n-1; i >= 0; i--) {
+                if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i];
+            }
+            return null;
+        },
+        
+         
+        
         /**
             @returns {Boolean} Was the token found?
          */
@@ -93,10 +143,25 @@ TokenReader = XObject.define(
             var name;
             if ((name = Lang.keyword(found))) {
                 if (found == 'return' && tokens.lastSym().data == ')') {
-                    throw {
-                        name : "ArgumentError", 
-                        message: "\n" + this.filename + ':' + this.line + " Error - return found after )"
-                    }   
+                    //Seed.print('@' + tokens.length);
+                    var n = this.findPuncToken(tokens, ')');
+                    //Seed.print(')@' + n);
+                    n = this.findPuncToken(tokens, '(', n-1);
+                    //Seed.print('(@' + n);
+                    
+                    var lt = this.lastSym(tokens, n);
+                    print(JSON.stringify(lt));
+                    if (lt.type != 'KEYW' || ['IF', 'WHILE'].indexOf(lt.name) < -1) {
+                        if (!this.ignoreBadGrammer) {
+                            throw {
+                                name : "ArgumentError", 
+                                message: "\n" + this.filename + ':' + this.line + " Error - return found after )"
+                            }
+                        }
+                    }
+                    
+                    
+                    
                 }
                 
                 tokens.push(new Token(found, "KEYW", name, this.line));
@@ -139,10 +204,15 @@ TokenReader = XObject.define(
             if ((found == '}' || found == ']') && tokens.lastSym().data == ',') {
                 //print("Error - comma found before " + found);
                 //print(JSON.stringify(tokens.lastSym(), null,4));
-                throw {
-                    name : "ArgumentError", 
-                    message: "\n" + this.filename + ':' + this.line + " Error - comma found before " + found
-                }   
+                if (this.ignoreBadGrammer) {
+                    print("\n" + this.filename + ':' + this.line + " Error - comma found before " + found);
+                } else {
+                    
+                    throw {
+                        name : "ArgumentError", 
+                        message: "\n" + this.filename + ':' + this.line + " Error - comma found before " + found
+                    }
+                }
             }
             
             tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line));
@@ -188,8 +258,8 @@ TokenReader = XObject.define(
             if (this.collapseWhite) {
                 found = "\n";
             }
-            if (this.keepWhite) {
-                var last = tokens.pop();
+             if (this.keepWhite) {
+                var last = tokens ? tokens.pop() : false;
                 if (last && last.name != "WHIT") {
                     tokens.push(last);
                 }
@@ -394,12 +464,25 @@ TokenReader = XObject.define(
                 )
             ) {
                 var regex = stream.next();
-                
+                var unbrace = false;
                 while (!stream.look().eof) {
+                    
+                    if (stream.look() == "[") { // escape sequence
+                        in_brace = true;
+                        continue;
+                    }
+                    
+                    if (in_brace && stream.look() == "[") { // escape sequence
+                        in_brace = true;
+                        continue;
+                    }
+                    
                     if (stream.look() == "\\") { // escape sequence
                         regex += stream.next(2);
+                        continue;
                     }
-                    else if (stream.look() == "/") {
+                    
+                    if (!in_brace && stream.look() == "/") {
                         regex += stream.next();
                         
                         while (/[gmi]/.test(stream.look())) {
@@ -409,9 +492,9 @@ TokenReader = XObject.define(
                         tokens.push(new Token(regex, "REGX", "REGX", this.line));
                         return true;
                     }
-                    else {
-                        regex += stream.next();
-                    }
+                    
+                    regex += stream.next();
+                    
                 }
                 // error: unterminated regex
             }