15 public errordomain TokenStreamError {
18 public class TokenStream : Object
21 protected Gee.ArrayList<Token> tokens;
22 public int cursor; // where are we in the stream.
25 public TokenStream(Gee.ArrayList<Token> tokens) {
31 public Gee.ArrayList<Token> toArray()
37 public void rewind() {
44 public Token? look (int n, bool considerWhitespace) // depricated... causes all sorts of problems...
48 if (considerWhitespace == true) {
50 if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
51 return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
53 return this.tokens.get(this.cursor+n);
62 return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
64 if (i >= this.tokens.size) {
65 return new Token("", TokenType.VOID, TokenName.END_OF_STREAM);
68 if (i != this.cursor && this.tokens.get(i).isType(TokenType.WHIT)) {
69 i += (n < 0) ? -1 : 1;
74 return this.tokens.get(i);
77 i += (n < 0) ? -1 : 1;
80 // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
83 // look through token stream, including white space...
84 public Token lookAny (int n)
88 if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
89 return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
91 return this.tokens.get(this.cursor+n);
99 public int lookFor (string data)
101 // non tree version..
102 var i = this.cursor < 0 ? 0 : this.cursor ;
105 if (i >= this.tokens.size) {
108 if (this.tokens.get(i).data == data) {
114 // should not get here!
121 * look ahead (or back) x number of tokens (which are not comment or whitespace)
122 * ?? used by scope parser & compress white to look back?
124 public Token lookTok (int n) {
127 var step = (n < 0) ? -1 : 1;
133 // print("lookTok:i=%d n= %d count=%d\n" , i, n, count);
135 if (i < 0 && n > -1) {
142 // beyond beginnnig..
143 if (i < 0 && n < 0) {
144 return new Token("BEG", TokenType.VOID, TokenName.END_OF_STREAM);
149 if (i >= this.tokens.size) {
150 return new Token("END", TokenType.VOID, TokenName.END_OF_STREAM);
152 // print("lookTok:i= %d n= %d : %s\n" , i, n, this.tokens.get(i).asString());
153 var tok = this.tokens.get(i);
155 if (i != this.cursor && (
156 tok.isType(TokenType.WHIT) || tok.isType(TokenType.COMM)
163 return this.tokens.get(i);
168 // should never get here..
169 // return new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
174 * @return {Token|null}
175 * next token (with white space)
179 public Token? next() {
182 //if (typeof howMany == "undefined") howMany = 1;
183 // if (howMany < 1) { return null; }
185 if (this.cursor+1 >= this.tokens.size) {
189 return this.tokens.get(this.cursor);
193 public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
195 //if (typeof howMany == "undefined") howMany = 1;
197 throw new TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
199 var got = new Gee.ArrayList<Token>();
201 for (var i = 1; i <= howMany; i++) {
202 if (this.cursor+i >= this.tokens.size) {
205 got.add(this.tokens.get(this.cursor+i));
207 this.cursor += howMany;
215 // what about comments after 'function'...
217 public Token? nextTok() {
218 return this.nextNonSpace();
221 public Token? nextNonSpace ()
225 var tok = this.next();
229 if (tok.isType(TokenType.WHIT) || tok.isType(TokenType.COMM)) {
238 * -- returns all the tokens betweeen and including stop token eg.. from {... to }
239 * @param start {String} token name or data (eg. '{'
240 * @param stop {String} (Optional) token name or data (eg. '}'
244 //public Gee.ArrayList<Token> balanceStr (string start) throws TokenStreamError
246 // return this.balance( Lang.punc(start));
250 public Gee.ArrayList<Token> balance (TokenName in_start) throws TokenStreamError
253 // fixme -- validate start...
255 // accepts names or "{" etc..
257 var start = in_start;
258 var stop = Lang.matching(start); /// validates start..
260 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start.to_string());
263 //print("START=%s, STOP=%s \n", start.to_string(),stop.to_string());
265 var got = new Gee.ArrayList<Token>();
267 //Seed.print("START:" + start);
268 //Seed.print("STOP:" + stop);
271 while (null != (token = this.lookAny(1))) {
272 //print("BALANCE: %d d=%d, %s \n" , this.cursor, depth, token.asString());
273 if (token.isName(start)) {
274 // Seed.print("balance: START : " + depth + " " + token.data);
283 if (token.isName(stop)) {
286 //debug("balance (%d): STOP: %s" , depth , token.data);
288 this.next(); // shift cursor to eat closer...
289 //debug("returning got %d", got.size);
294 if (null == this.next()) {
298 return new Gee.ArrayList<Token>();
300 // designed to get either end or start..
303 public Token? getMatchingTokenEnd(TokenName end)
305 return this.getMatchingToken(Lang.matching(end), 1);
308 public Token? getMatchingToken(TokenName start, int depth = 0)
311 var cursor = this.cursor;
314 var stop= Lang.matching(start);
317 while (null != (token = this.tokens[cursor])) {
318 if (token.isName(start)) {
322 if (token.isName(stop) && cursor != 0) {
325 return this.tokens[cursor];
333 public Gee.ArrayList<Token> insertAhead(Token token)
335 this.tokens.splice(this.cursor+1, 0, token); // fixme...
339 public Gee.ArrayList<Token> remaining() {
340 var ret = new Gee.ArrayList<Token>();
342 var tok = this.look(1,true);
343 if (tok.isType(TokenType.VOID)) {
346 var nt = this.next();
354 public void printRange(int start, int end) {
356 for(var i = start; i < end +1; i++) {
357 print(this.tokens.get(i).asString());
362 arrayToString : function(ar) {
363 console.log(typeof(ar));
365 ar.forEach(function(e) {
371 public void dump(int start, int end)
373 start = int.max(start , 0);
374 end = int.min(end, this.tokens.size);
376 for (var i =start;i < end; i++) {
378 outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
383 public void dumpAll(string indent)
385 for (var i = 0;i < this.tokens.size; i++) {
387 this.tokens[i].dump("");
391 public void dumpAllFlat()
393 for (var i = 0;i < this.tokens.size; i++) {
395 print("%d: %s\n", i, this.tokens[i].asString());