git.roojs.org Git - roojspacker/blob - roojspacker/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31                 public  Gee.ArrayList<Token> toArray()
  32                 {
  33                         return this.tokens;
  34                 }
  35
  36
  37                 public void rewind() {
  38                     this.cursor = -1;
  39                 }
  40
  41                 /**
  42                     @type JSDOC.Token
  43                 */
  44                 public Token? look (int n, bool considerWhitespace)  // depricated... causes all sorts of problems...
  45                 {
  46
  47
  48                     if (considerWhitespace == true) {
  49
  50                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  51                             return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
  52                         }
  53                         return this.tokens.get(this.cursor+n);
  54                     }
  55
  56
  57                 var count = 0;
  58                 var i = this.cursor;
  59
  60                 while (true) {
  61                     if (i < 0) {
  62                                 return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
  63                         }
  64                     if (i >= this.tokens.size) {
  65                                 return new Token("", TokenType.VOID, TokenName.END_OF_STREAM);
  66                         }
  67
  68                     if (i != this.cursor && this.tokens.get(i).isType(TokenType.WHIT)) {
  69                                 i += (n < 0) ? -1 : 1;
  70                         continue;
  71                     }
  72
  73                     if (count == n) {
  74                         return this.tokens.get(i);
  75                     }
  76                     count++;
  77                     i += (n < 0) ? -1 : 1;
  78                 }
  79
  80                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  81
  82                 }
  83                 // look through token stream, including white space...
  84                 public Token  lookAny (int n)
  85                 {
  86
  87
  88                  if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  89                     return new Token("", TokenType.VOID, TokenName.START_OF_STREAM);
  90                 }
  91                 return this.tokens.get(this.cursor+n);
  92
  93
  94
  95                 }
  96
  97
  98
  99                 public int lookFor  (string data)
 100                 {
 101                     // non tree version..
 102                     var i = this.cursor < 0 ? 0 : this.cursor ;
 103
 104                     while (true) {
 105                         if (i >= this.tokens.size) {
 106                                 return -1;
 107                         }
 108                         if (this.tokens.get(i).data == data) {
 109                             return i;
 110                         }
 111                         i++;
 112
 113                     }
 114                     // should not get here!
 115                    // return -1;
 116
 117                 }
 118
 119
 120                 /**
 121                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 122                  * ?? used by scope parser & compress white to look back?
 123                  */
 124                  public Token lookTok (int n) {
 125
 126
 127                     var step =  (n < 0) ? -1 : 1;
 128                     var count = 0;
 129
 130                     var i = this.cursor;
 131
 132                     while (true) {
 133                         // print("lookTok:i=%d n= %d count=%d\n" , i, n, count);
 134
 135                         if (i < 0 &&  n > -1) {
 136                                 i = 0;
 137                                 count += step;
 138                                 continue;
 139                     }
 140
 141
 142                     // beyond beginnnig..
 143                     if (i < 0 &&  n < 0) {
 144                             return  new Token("BEG", TokenType.VOID, TokenName.END_OF_STREAM);
 145                         }
 146
 147
 148                         // beyond end..
 149                         if (i >= this.tokens.size) {
 150                                 return  new Token("END", TokenType.VOID, TokenName.END_OF_STREAM);
 151                         }
 152                                 // print("lookTok:i= %d n= %d : %s\n" , i, n, this.tokens.get(i).asString());
 153                                 var tok = this.tokens.get(i);
 154
 155                         if (i != this.cursor && (
 156                                                 tok.isType(TokenType.WHIT) || tok.isType(TokenType.COMM)
 157                                 )) {
 158                             i += step;
 159                             continue;
 160                         }
 161
 162                         if (count == n) {
 163                             return this.tokens.get(i);
 164                         }
 165                         count+=step;
 166                         i += step;
 167                     }
 168                 // should never get here..
 169                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 170
 171                 }
 172
 173                 /**
 174                  *  @return {Token|null}
 175                  * next token (with white space)
 176                  */
 177
 178
 179                 public Token? next() {
 180
 181
 182                     //if (typeof howMany == "undefined") howMany = 1;
 183                     // if (howMany < 1) { return  null;                 }
 184
 185                     if (this.cursor+1 >= this.tokens.size) {
 186                         return null;
 187                 }
 188                     this.cursor++;
 189                     return this.tokens.get(this.cursor);
 190
 191                 }
 192
 193             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 194
 195                     //if (typeof howMany == "undefined") howMany = 1;
 196                     if (howMany < 2) {
 197                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 198                     }
 199                     var got = new Gee.ArrayList<Token>();
 200
 201                     for (var i = 1; i <= howMany; i++) {
 202                         if (this.cursor+i >= this.tokens.size) {
 203                             return null;
 204                         }
 205                         got.add(this.tokens.get(this.cursor+i));
 206                     }
 207                     this.cursor += howMany;
 208
 209                         return got;
 210                 }
 211
 212
 213
 214
 215                 // what about comments after 'function'...
 216                 // is this used ???
 217                 public Token? nextTok() {
 218                     return this.nextNonSpace();
 219                 }
 220
 221                 public Token? nextNonSpace ()
 222                 {
 223
 224                     while (true) {
 225                         var tok = this.next();
 226                         if (tok == null) {
 227                             return null;
 228                         }
 229                         if (tok.isType(TokenType.WHIT) ||  tok.isType(TokenType.COMM)) {
 230                             continue;
 231                         }
 232                         return tok;
 233                     }
 234                 }
 235
 236                 /**
 237                  *  balance
 238                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 239                  * @param start {String}  token name or data (eg. '{'
 240                  * @param stop {String} (Optional) token name or data (eg. '}'
 241                  */
 242
 243
 244                 //public Gee.ArrayList<Token> balanceStr (string start) throws TokenStreamError
 245                 //{
 246                 //      return this.balance( Lang.punc(start));
 247                 //}
 248
 249
 250                 public Gee.ArrayList<Token> balance (TokenName in_start) throws TokenStreamError
 251                 {
 252
 253                     // fixme -- validate start...
 254
 255                     // accepts names or "{" etc..
 256
 257                     var start = in_start;
 258                     var stop =  Lang.matching(start); /// validates start..
 259                     if (stop == null) {
 260                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start.to_string());
 261                         }
 262
 263                     //print("START=%s, STOP=%s \n", start.to_string(),stop.to_string());
 264                     var depth = 0;
 265                     var got = new Gee.ArrayList<Token>();
 266                     var started = false;
 267                     //Seed.print("START:" + start);
 268                     //Seed.print("STOP:" + stop);
 269                     Token token;
 270
 271                     while (null != (token = this.lookAny(1))) {
 272                                 //print("BALANCE: %d d=%d, %s  \n" , this.cursor,  depth, token.asString());
 273                         if (token.isName(start)) {
 274                       //      Seed.print("balance: START : " + depth + " " + token.data);
 275                             depth++;
 276                             started = true;
 277                         }
 278
 279                         if (started) {
 280                             got.add(token);
 281                         }
 282
 283                         if (token.isName(stop)) {
 284                             depth--;
 285
 286                                 //debug("balance (%d): STOP: %s" ,  depth ,  token.data);
 287                             if (depth < 1) {
 288                                     this.next(); // shift cursor to eat closer...
 289                                         //debug("returning got %d", got.size);
 290                                         return got;
 291                                 }
 292
 293                         }
 294                         if (null == this.next()) {
 295                                 break;
 296                         }
 297                     }
 298                     return new Gee.ArrayList<Token>();
 299                 }
 300                 // designed to get either end or start..
 301
 302
 303                 public Token? getMatchingTokenEnd(TokenName end)
 304                 {
 305                         return this.getMatchingToken(Lang.matching(end), 1);
 306                 }
 307
 308                 public Token? getMatchingToken(TokenName start, int depth = 0)
 309                 {
 310
 311                     var cursor = this.cursor;
 312
 313
 314                                 var stop= Lang.matching(start);
 315                         Token token;
 316
 317                     while (null != (token = this.tokens[cursor])) {
 318                         if (token.isName(start)) {
 319                             depth++;
 320                         }
 321
 322                         if (token.isName(stop) && cursor != 0) {
 323                             depth--;
 324                             if (depth == 0) {
 325                                         return this.tokens[cursor];
 326                                 }
 327                         }
 328                         cursor++;
 329                     }
 330                     return null;
 331                 }
 332                 /*
 333                 public Gee.ArrayList<Token> insertAhead(Token token)
 334                 {
 335                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 336                 }
 337                 */
 338
 339                 public Gee.ArrayList<Token> remaining() {
 340                     var ret = new Gee.ArrayList<Token>();
 341                     while (true) {
 342                         var tok = this.look(1,true);
 343                         if (tok.isType(TokenType.VOID)) {
 344                             return ret;
 345                         }
 346                         var nt = this.next();
 347                         if (nt != null) {
 348                                 ret.add(nt);
 349                         }
 350                     }
 351                 }
 352
 353
 354                 public void printRange(int start,  int end) {
 355
 356                         for(var i = start; i < end +1; i++) {
 357                     print(this.tokens.get(i).asString());
 358                         }
 359                 }
 360
 361                 /*
 362                 arrayToString : function(ar) {
 363                     console.log(typeof(ar));
 364                     var ret = [];
 365                     ar.forEach(function(e) {
 366                         ret.push(e.data);
 367                     })
 368                     return ret.join('');
 369                 },
 370                 */
 371                 public void dump(int start, int end)
 372                 {
 373                     start = int.max(start , 0);
 374                     end = int.min(end, this.tokens.size);
 375                     var  outs = "";;
 376                     for (var i =start;i < end; i++) {
 377
 378                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 379                     }
 380                     print(outs);
 381                 }
 382
 383                 public void dumpAll(string indent)
 384                 {
 385                     for (var i = 0;i < this.tokens.size; i++) {
 386
 387                          this.tokens[i].dump("");
 388                     }
 389
 390                 }
 391                 public void dumpAllFlat()
 392                 {
 393                     for (var i = 0;i < this.tokens.size; i++) {
 394
 395                          print("%d: %s\n", i, this.tokens[i].asString());
 396                     }
 397
 398                 }
 399
 400         }
 401 }
 402