15 public errordomain TokenStreamError {
18 public class TokenStream : Object
21 protected Gee.ArrayList<Token> tokens;
22 public int cursor; // where are we in the stream.
25 public TokenStream(Gee.ArrayList<Token> tokens) {
31 public Gee.ArrayList<Token> toArray()
37 public void rewind() {
44 public Token? look (int n, bool considerWhitespace)
48 if (considerWhitespace == true) {
50 if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
51 return new Token("", "VOID", "START_OF_STREAM");
53 return this.tokens.get(this.cursor+n);
62 return new Token("", "VOID", "START_OF_STREAM");
64 if (i >= this.tokens.size) {
65 return new Token("", "VOID", "END_OF_STREAM");
68 if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
69 i += (n < 0) ? -1 : 1;
74 return this.tokens.get(i);
77 i += (n < 0) ? -1 : 1;
80 // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
84 public int lookFor (string data)
87 var i = this.cursor < 0 ? 0 : this.cursor ;
90 if (i >= this.tokens.size) {
93 if (this.tokens.get(i).data == data) {
99 // should not get here!
106 * look ahead (or back) x number of tokens (which are not comment or whitespace)
109 public Token lookTok (int n) {
125 return new Token("", "VOID", "END_OF_STREAM");
127 if (i >= this.tokens.size) {
128 return new Token("", "VOID", "END_OF_STREAM");
131 if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
132 i += (n < 0) ? -1 : 1;
137 return this.tokens.get(i);
140 i += (n < 0) ? -1 : 1;
142 // should never get here..
143 // return new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
148 * @return {Token|null}
149 * next token (with white space)
153 public Token? next() {
156 //if (typeof howMany == "undefined") howMany = 1;
157 // if (howMany < 1) { return null; }
159 if (this.cursor+1 >= this.tokens.size) {
163 return this.tokens.get(this.cursor);
167 public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
169 //if (typeof howMany == "undefined") howMany = 1;
171 throw new TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
173 var got = new Gee.ArrayList<Token>();
175 for (var i = 1; i <= howMany; i++) {
176 if (this.cursor+i >= this.tokens.size) {
179 got.add(this.tokens.get(this.cursor+i));
181 this.cursor += howMany;
189 // what about comments after 'function'...
191 public Token? nextTok() {
192 return this.nextNonSpace();
195 public Token? nextNonSpace ()
199 var tok = this.next();
203 if (tok.is("WHIT") || tok.is("COMM")) {
212 * -- returns all the tokens betweeen and including stop token eg.. from {... to }
213 * @param start {String} token name or data (eg. '{'
214 * @param stop {String} (Optional) token name or data (eg. '}'
216 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
219 // accepts names or "{" etc..
221 start = Lang.punc(start) == null ? start : Lang.punc(start);
224 var newstop = Lang.matching(start);
228 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
230 debug("START=%s, STOP=%s \n", start,stop);
232 var got = new Gee.ArrayList<Token>();
234 //Seed.print("START:" + start);
235 //Seed.print("STOP:" + stop);
238 while (null != (token = this.look(1,false))) {
239 if (token.is(start)) {
240 // Seed.print("balance: START : " + depth + " " + token.data);
249 if (token.is(stop)) {
251 // Seed.print("balance: STOP: " + depth + " " + token.data);
256 if (null == this.next()) {
260 return new Gee.ArrayList<Token>();
263 public Token? getMatchingToken(string start, string stop)
266 var cursor = this.cursor;
268 if (start.length < 1) {
269 var ns = Lang.matching(stop);
273 if (stop.length < 1) {
274 var ns = Lang.matching(start);
279 while (null != (token = this.tokens[cursor])) {
280 if (token.is(start)) {
284 if (token.is(stop) && cursor != 0) {
287 return this.tokens[cursor];
295 public Gee.ArrayList<Token> insertAhead(Token token)
297 this.tokens.splice(this.cursor+1, 0, token); // fixme...
301 public Gee.ArrayList<Token> remaining() {
302 var ret = new Gee.ArrayList<Token>();
304 var tok = this.look(1,true);
305 if (tok.is("VOID")) {
308 var nt = this.next();
316 public void printRange(int start, int end) {
318 for(var i = start; i < end +1; i++) {
319 print(this.tokens.get(i).asString());
324 arrayToString : function(ar) {
325 console.log(typeof(ar));
327 ar.forEach(function(e) {
333 public void dump(int start, int end)
335 start = int.max(start , 0);
336 end = int.min(end, this.tokens.size);
338 for (var i =start;i < end; i++) {
340 outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
345 public void dumpAll(string indent)
347 for (var i = 0;i < this.tokens.size; i++) {
349 this.tokens[i].dump("");