X-Git-Url: http://git.roojs.org/?p=gnome.introspection-doc-generator;a=blobdiff_plain;f=JSDOC%2FTokenStream.vala;h=559141b51151534cb95362adc91f5f513b28ffba;hp=05d5a8fad9afc54277348c109a9e352578ad29e7;hb=HEAD;hpb=b0af888f8a133dc45213d3545d60c620d70fe025 diff --git a/JSDOC/TokenStream.vala b/JSDOC/TokenStream.vala index 05d5a8f..559141b 100644 --- a/JSDOC/TokenStream.vala +++ b/JSDOC/TokenStream.vala @@ -12,14 +12,14 @@ namespace JSDOC { - errordomain TokenStreamError { + public errordomain TokenStreamError { ArgumentError } public class TokenStream : Object { - Gee.ArrayList tokens; - int cursor; // where are we in the stream. + protected Gee.ArrayList tokens; + public int cursor; // where are we in the stream. public TokenStream(Gee.ArrayList tokens) { @@ -28,7 +28,10 @@ namespace JSDOC { this.rewind(); } - + public Gee.ArrayList toArray() + { + return this.tokens; + } public void rewind() { @@ -38,7 +41,7 @@ namespace JSDOC { /** @type JSDOC.Token */ - public Token? look (int n, bool considerWhitespace) + public Token? look (int n, bool considerWhitespace) // depricated... causes all sorts of problems... { @@ -58,7 +61,7 @@ namespace JSDOC { if (i < 0) { return new Token("", "VOID", "START_OF_STREAM"); } - if (i > this.tokens.size) { + if (i >= this.tokens.size) { return new Token("", "VOID", "END_OF_STREAM"); } @@ -74,9 +77,24 @@ namespace JSDOC { i += (n < 0) ? -1 : 1; } - return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object + // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object } + // look through token stream, including white space... + public Token lookAny (int n) + { + + + if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) { + return new Token("", "VOID", "START_OF_STREAM"); + } + return this.tokens.get(this.cursor+n); + + + + } + + public int lookFor (string data) { @@ -94,7 +112,7 @@ namespace JSDOC { } // should not get here! - return -1; + // return -1; } @@ -121,7 +139,7 @@ namespace JSDOC { } return new Token("", "VOID", "END_OF_STREAM"); } - if (i > this.tokens.size) { + if (i >= this.tokens.size) { return new Token("", "VOID", "END_OF_STREAM"); } @@ -137,7 +155,7 @@ namespace JSDOC { i += (n < 0) ? -1 : 1; } // should never get here.. - return new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object; + // return new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object; } @@ -161,11 +179,11 @@ namespace JSDOC { } - public Gee.ArrayList nextM(uint howMany) { + public Gee.ArrayList? nextM(int howMany) throws TokenStreamError { //if (typeof howMany == "undefined") howMany = 1; if (howMany < 2) { - throw new TokenStreamError.ArgumentError("nextM called with wrong number : %d", (int)howMany); + throw new TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany); } var got = new Gee.ArrayList(); @@ -193,7 +211,7 @@ namespace JSDOC { { while (true) { - tok = this.next(); + var tok = this.next(); if (tok == null) { return null; } @@ -210,26 +228,30 @@ namespace JSDOC { * @param start {String} token name or data (eg. '{' * @param stop {String} (Optional) token name or data (eg. '}' */ - public Gee.ArrayList balance (string start, string stop = "") { + public Gee.ArrayList balance (string start, string in_stop = "") throws TokenStreamError + { // accepts names or "{" etc.. - + var stop = in_stop; start = Lang.punc(start) == null ? start : Lang.punc(start); if (stop=="") { - stop = Lang.matching(start); + var newstop = Lang.matching(start); + stop = newstop; } if (stop == null) { - throw new JSDOC.TokenStreamError("balance called with invalid start/stop : %s",start); + throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start); } - + debug("START=%s, STOP=%s \n", start,stop); var depth = 0; var got = new Gee.ArrayList(); var started = false; //Seed.print("START:" + start); //Seed.print("STOP:" + stop); + Token token; - while ((token = this.look())) { + while (null != (token = this.lookAny(1))) { + debug("BALANCE: %d %s " , this.cursor, token.asString()); if (token.is(start)) { // Seed.print("balance: START : " + depth + " " + token.data); depth++; @@ -237,50 +259,64 @@ namespace JSDOC { } if (started) { - got.push(token); + got.add(token); } if (token.is(stop)) { depth--; - // Seed.print("balance: STOP: " + depth + " " + token.data); + + //print("balance (%d): STOP: %s\n" , depth , token.data); if (depth < 1) { + this.next(); // shift cursor to eat closer... + //print("returning got %d\n", got.size); return got; } + } - if (!this.next()) { + if (null == this.next()) { break; } } return new Gee.ArrayList(); } - public Token getMatchingToken(string start, string stop) { + public Token? getMatchingToken(string start, string stop) + { var depth = 0; var cursor = this.cursor; - if (!start) { - start = Lang.matching(stop); + if (start.length < 1) { + var ns = Lang.matching(stop); + start = ns; depth = 1; } - if (!stop) stop = Lang.matching(start); + if (stop.length < 1) { + var ns = Lang.matching(start); + stop = ns; + } + Token token; - while ((token = this.tokens[cursor])) { + while (null != (token = this.tokens[cursor])) { if (token.is(start)) { depth++; } - if (token.is(stop) && cursor) { + if (token.is(stop) && cursor != 0) { depth--; - if (depth == 0) return this.tokens[cursor]; + if (depth == 0) { + return this.tokens[cursor]; + } } cursor++; } - return false; + return null; } - - public Gee.ArrayList insertAhead(Token token) { + /* + public Gee.ArrayList insertAhead(Token token) + { this.tokens.splice(this.cursor+1, 0, token); // fixme... } + */ public Gee.ArrayList remaining() { var ret = new Gee.ArrayList(); @@ -289,10 +325,21 @@ namespace JSDOC { if (tok.is("VOID")) { return ret; } - ret.push(this.next(1)); + var nt = this.next(); + if (nt != null) { + ret.add(nt); + } } } + + public void printRange(int start, int end) { + + for(var i = start; i < end +1; i++) { + print(this.tokens.get(i).asString()); + } + } + /* arrayToString : function(ar) { console.log(typeof(ar)); @@ -302,18 +349,36 @@ namespace JSDOC { }) return ret.join(''); }, - dump: function(start, end) + */ + public void dump(int start, int end) { - start = Math.max(start || 0, 0); - end = Math.min(end || this.tokens.length, this.tokens.length); - var out=''; + start = int.max(start , 0); + end = int.min(end, this.tokens.size); + var outs = "";; for (var i =start;i < end; i++) { - out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData; - }; - print(out); + outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData; + } + print(outs); } - */ + + public void dumpAll(string indent) + { + for (var i = 0;i < this.tokens.size; i++) { + + this.tokens[i].dump(""); + } + + } + public void dumpAllFlat() + { + for (var i = 0;i < this.tokens.size; i++) { + + print("%d: %s\n", i, this.tokens[i].asString()); + } + + } + } }