JSDOC/TokenReader.vala

[gnome.introspection-doc-generator] / JSDOC / TokenReader.vala

diff --git a/JSDOC/TokenReader.vala b/JSDOC/TokenReader.vala
index b4127d7..29deac8 100644 (file)
--- a/JSDOC/TokenReader.vala
+++ b/JSDOC/TokenReader.vala
@@ -41,6 +41,14 @@ namespace JSDOC {
         public void push (Token t) {
             this.tokens.add(t);
         }
+        public Token? pop ()
+        {
+            if (this.size > 0) {
+                return this.tokens.remove_at(this.size-1);
+            }
+            return null;
+        }
+        
         public Token get(int i) {
             return this.tokens.get(i);
         }
@@ -288,14 +296,18 @@ namespace JSDOC {
             if (found === "") {
                 return false;
             }
-            //print("WHITE = " + JSON.stringify(found)); 
+            //print("WHITE = " + JSON.stringify(found));
+            
+             
             if (this.collapseWhite) {
-                found = " ";
+                found = " "; // this might work better if it was a '\n' ???
+            }
+            if (this.keepWhite) {
+                tokens.push(new Token(found, "WHIT", "SPACE", this.line));
             }
-            if (this.keepWhite) tokens.push(new Token(found, "WHIT", "SPACE", this.line));
             return true;
         
-        },
+        }
 
         /**
             @returns {Boolean} Was the token found?
@@ -303,7 +315,7 @@ namespace JSDOC {
         public bool read_newline  (TokenStream stream, TokenArray tokens)
             var found = "";
             var line = this.line;
-            while (!stream.look().eof && Lang.isNewline(stream.look())) {
+            while (!stream.lookEOF() && Lang.isNewline(stream.look())) {
                 this.line++;
                 found += stream.next();
             }
@@ -311,16 +323,22 @@ namespace JSDOC {
             if (found === "") {
                 return false;
             }
+            
+            // if we found a new line, then we could check if previous character was a ';' - if so we can drop it.
+            // otherwise generally keep it.. in which case it should reduce our issue with stripping new lines..
+           
+            
             //this.line++;
             if (this.collapseWhite) {
-                found = "\n";
+                found = "\n"; // reduces multiple line breaks into a single one...
             }
-             if (this.keepWhite) {
-                var last = tokens ? tokens.pop() : false;
-                if (last && last.name != "WHIT") {
+            
+            if (this.keepWhite) {
+                var last = tokens.pop();
+                if (last != null && last.name != "WHIT") {
                     tokens.push(last);
                 }
-                
+                // replaces last new line... 
                 tokens.push(new Token(found, "WHIT", "NEWLINE", line));
             }
             return true;
@@ -329,23 +347,34 @@ namespace JSDOC {
         /**
             @returns {Boolean} Was the token found?
          */
-        read_mlcomment : function(/**JSDOC.TokenStream*/stream, tokens) {
-            if (stream.look() == "/" && stream.look(1) == "*") {
-                var found = stream.next(2);
-                var c = '';
-                var line = this.line;
-                while (!stream.look().eof && !(stream.look(-1) == "/" && stream.look(-2) == "*")) {
-                    c = stream.next();
-                    if (c == "\n") this.line++;
-                    found += c;
+        public bool read_mlcomment  (TokenStream stream, TokenArray tokens)
+        {
+            if (stream.look() != "/") {
+                return false;
+            }
+            if (stream.look(1) != "*") {
+                return false;
+            }
+            var found = stream.next(2);
+            var c = '';
+            var line = this.line;
+            while (!stream.lookEOF() && !(stream.look(-1) == "/" && stream.look(-2) == "*")) {
+                c = stream.next();
+                if (c == "\n") {
+                    this.line++;
                 }
-                
-                // to start doclet we allow /** or /*** but not /**/ or /****
-                if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new Token(found, "COMM", "JSDOC", this.line));
-                else if (this.keepComments) tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", line));
-                return true;
+                found += c;
             }
-            return false;
+            
+            // to start doclet we allow /** or /*** but not /**/ or /****
+            //if (found.length /^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) {
+            if ((this.keepDocs && found.length > 4 && found.index_of("/**") == 0 && found[3] != "/") {
+                tokens.push(new Token(found, "COMM", "JSDOC", this.line));
+            } else if (this.keepComments) {
+                tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", line));
+            }
+            return true;
+        
         },
 
         /**