15 public errordomain TokenStreamError {
18 public class TokenStream : Object
21 protected Gee.ArrayList<Token> tokens;
22 public int cursor; // where are we in the stream.
25 public TokenStream(Gee.ArrayList<Token> tokens) {
31 public Gee.ArrayList<Token> toArray()
37 public void rewind() {
44 public Token? look (int n, bool considerWhitespace) // depricated... causes all sorts of problems...
48 if (considerWhitespace == true) {
50 if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
51 return new Token("", "VOID", "START_OF_STREAM");
53 return this.tokens.get(this.cursor+n);
62 return new Token("", "VOID", "START_OF_STREAM");
64 if (i >= this.tokens.size) {
65 return new Token("", "VOID", "END_OF_STREAM");
68 if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
69 i += (n < 0) ? -1 : 1;
74 return this.tokens.get(i);
77 i += (n < 0) ? -1 : 1;
80 // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
83 // look through token stream, including white space...
84 public Token lookAny (int n)
88 if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
89 return new Token("", "VOID", "START_OF_STREAM");
91 return this.tokens.get(this.cursor+n);
99 public int lookFor (string data)
101 // non tree version..
102 var i = this.cursor < 0 ? 0 : this.cursor ;
105 if (i >= this.tokens.size) {
108 if (this.tokens.get(i).data == data) {
114 // should not get here!
121 * look ahead (or back) x number of tokens (which are not comment or whitespace)
124 public Token lookTok (int n) {
140 return new Token("", "VOID", "END_OF_STREAM");
142 if (i >= this.tokens.size) {
143 return new Token("", "VOID", "END_OF_STREAM");
146 if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
147 i += (n < 0) ? -1 : 1;
152 return this.tokens.get(i);
155 i += (n < 0) ? -1 : 1;
157 // should never get here..
158 // return new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
163 * @return {Token|null}
164 * next token (with white space)
168 public Token? next() {
171 //if (typeof howMany == "undefined") howMany = 1;
172 // if (howMany < 1) { return null; }
174 if (this.cursor+1 >= this.tokens.size) {
178 return this.tokens.get(this.cursor);
182 public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
184 //if (typeof howMany == "undefined") howMany = 1;
186 throw new TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
188 var got = new Gee.ArrayList<Token>();
190 for (var i = 1; i <= howMany; i++) {
191 if (this.cursor+i >= this.tokens.size) {
194 got.add(this.tokens.get(this.cursor+i));
196 this.cursor += howMany;
204 // what about comments after 'function'...
206 public Token? nextTok() {
207 return this.nextNonSpace();
210 public Token? nextNonSpace ()
214 var tok = this.next();
218 if (tok.is("WHIT") || tok.is("COMM")) {
227 * -- returns all the tokens betweeen and including stop token eg.. from {... to }
228 * @param start {String} token name or data (eg. '{'
229 * @param stop {String} (Optional) token name or data (eg. '}'
231 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
234 // accepts names or "{" etc..
236 start = Lang.punc(start) == null ? start : Lang.punc(start);
239 var newstop = Lang.matching(start);
243 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
245 debug("START=%s, STOP=%s \n", start,stop);
247 var got = new Gee.ArrayList<Token>();
249 //Seed.print("START:" + start);
250 //Seed.print("STOP:" + stop);
253 while (null != (token = this.lookAny(1))) {
254 debug("BALANCE: %d %s " , this.cursor, token.asString());
255 if (token.is(start)) {
256 // Seed.print("balance: START : " + depth + " " + token.data);
265 if (token.is(stop)) {
268 print("balance (%d): STOP: %s\n" , depth , token.data);
270 this.next(); // shift cursor to eat closer...
271 print("returning got %d\n", got.size);
276 if (null == this.next()) {
280 return new Gee.ArrayList<Token>();
283 public Token? getMatchingToken(string start, string stop)
286 var cursor = this.cursor;
288 if (start.length < 1) {
289 var ns = Lang.matching(stop);
293 if (stop.length < 1) {
294 var ns = Lang.matching(start);
299 while (null != (token = this.tokens[cursor])) {
300 if (token.is(start)) {
304 if (token.is(stop) && cursor != 0) {
307 return this.tokens[cursor];
315 public Gee.ArrayList<Token> insertAhead(Token token)
317 this.tokens.splice(this.cursor+1, 0, token); // fixme...
321 public Gee.ArrayList<Token> remaining() {
322 var ret = new Gee.ArrayList<Token>();
324 var tok = this.look(1,true);
325 if (tok.is("VOID")) {
328 var nt = this.next();
336 public void printRange(int start, int end) {
338 for(var i = start; i < end +1; i++) {
339 print(this.tokens.get(i).asString());
344 arrayToString : function(ar) {
345 console.log(typeof(ar));
347 ar.forEach(function(e) {
353 public void dump(int start, int end)
355 start = int.max(start , 0);
356 end = int.min(end, this.tokens.size);
358 for (var i =start;i < end; i++) {
360 outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
365 public void dumpAll(string indent)
367 for (var i = 0;i < this.tokens.size; i++) {
369 this.tokens[i].dump("");
373 public void dumpAllFlat()
375 for (var i = 0;i < this.tokens.size; i++) {
377 print("%d: %s\n", i, this.tokens[i].asString());