git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31                 public  Gee.ArrayList<Token> toArray()
  32                 {
  33                         return this.tokens;
  34                 }
  35
  36
  37                 public void rewind() {
  38                     this.cursor = -1;
  39                 }
  40
  41                 /**
  42                     @type JSDOC.Token
  43                 */
  44                 public Token? look (int n, bool considerWhitespace)  // depricated... causes all sorts of problems...
  45                 {
  46
  47
  48                     if (considerWhitespace == true) {
  49
  50                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  51                             return new Token("", "VOID", "START_OF_STREAM");
  52                         }
  53                         return this.tokens.get(this.cursor+n);
  54                     }
  55
  56
  57                 var count = 0;
  58                 var i = this.cursor;
  59
  60                 while (true) {
  61                     if (i < 0) {
  62                                 return new Token("", "VOID", "START_OF_STREAM");
  63                         }
  64                     if (i >= this.tokens.size) {
  65                                 return new Token("", "VOID", "END_OF_STREAM");
  66                         }
  67
  68                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  69                                 i += (n < 0) ? -1 : 1;
  70                         continue;
  71                     }
  72
  73                     if (count == n) {
  74                         return this.tokens.get(i);
  75                     }
  76                     count++;
  77                     i += (n < 0) ? -1 : 1;
  78                 }
  79
  80                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  81
  82                 }
  83                 // look through token stream, including white space...
  84                 public Token? lookAny (int n)
  85                 {
  86
  87
  88                     if (considerWhitespace == true) {
  89
  90                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  91                             return new Token("", "VOID", "START_OF_STREAM");
  92                         }
  93                         return this.tokens.get(this.cursor+n);
  94                     }
  95
  96
  97                 var count = 0;
  98                 var i = this.cursor;
  99
 100                 while (true) {
 101                     if (i < 0) {
 102                                 return new Token("", "VOID", "START_OF_STREAM");
 103                         }
 104                     if (i >= this.tokens.size) {
 105                                 return new Token("", "VOID", "END_OF_STREAM");
 106                         }
 107
 108                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
 109                                 i += (n < 0) ? -1 : 1;
 110                         continue;
 111                     }
 112
 113                     if (count == n) {
 114                         return this.tokens.get(i);
 115                     }
 116                     count++;
 117                     i += (n < 0) ? -1 : 1;
 118                 }
 119
 120                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
 121
 122                 }
 123
 124
 125
 126                 public int lookFor  (string data)
 127                 {
 128                     // non tree version..
 129                     var i = this.cursor < 0 ? 0 : this.cursor ;
 130
 131                     while (true) {
 132                         if (i >= this.tokens.size) {
 133                                 return -1;
 134                         }
 135                         if (this.tokens.get(i).data == data) {
 136                             return i;
 137                         }
 138                         i++;
 139
 140                     }
 141                     // should not get here!
 142                    // return -1;
 143
 144                 }
 145
 146
 147                 /**
 148                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 149                  * ?? used any more?
 150                  */
 151                 public Token lookTok (int n) {
 152
 153
 154
 155                     var count = 0;
 156                     var i = this.cursor;
 157
 158                     while (true) {
 159                        // print(i);
 160                         if (i < 0) {
 161                             if (n > -1) {
 162                                 i = 0;
 163                                 count++;
 164                                 continue;
 165
 166                             }
 167                             return  new Token("", "VOID", "END_OF_STREAM");
 168                         }
 169                         if (i >= this.tokens.size) {
 170                                 return  new Token("", "VOID", "END_OF_STREAM");
 171                         }
 172
 173                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 174                             i += (n < 0) ? -1 : 1;
 175                             continue;
 176                         }
 177
 178                         if (count == n) {
 179                             return this.tokens.get(i);
 180                         }
 181                         count++;
 182                         i += (n < 0) ? -1 : 1;
 183                     }
 184                 // should never get here..
 185                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 186
 187                 }
 188
 189                 /**
 190                  *  @return {Token|null}
 191                  * next token (with white space)
 192                  */
 193
 194
 195                 public Token? next() {
 196
 197
 198                     //if (typeof howMany == "undefined") howMany = 1;
 199                     // if (howMany < 1) { return  null;                 }
 200
 201                     if (this.cursor+1 >= this.tokens.size) {
 202                         return null;
 203                 }
 204                     this.cursor++;
 205                     return this.tokens.get(this.cursor);
 206
 207                 }
 208
 209             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 210
 211                     //if (typeof howMany == "undefined") howMany = 1;
 212                     if (howMany < 2) {
 213                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 214                     }
 215                     var got = new Gee.ArrayList<Token>();
 216
 217                     for (var i = 1; i <= howMany; i++) {
 218                         if (this.cursor+i >= this.tokens.size) {
 219                             return null;
 220                         }
 221                         got.add(this.tokens.get(this.cursor+i));
 222                     }
 223                     this.cursor += howMany;
 224
 225                         return got;
 226                 }
 227
 228
 229
 230
 231                 // what about comments after 'function'...
 232                 // is this used ???
 233                 public Token? nextTok() {
 234                     return this.nextNonSpace();
 235                 }
 236
 237                 public Token? nextNonSpace ()
 238                 {
 239
 240                     while (true) {
 241                         var tok = this.next();
 242                         if (tok == null) {
 243                             return null;
 244                         }
 245                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 246                             continue;
 247                         }
 248                         return tok;
 249                     }
 250                 }
 251
 252                 /**
 253                  *  balance
 254                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 255                  * @param start {String}  token name or data (eg. '{'
 256                  * @param stop {String} (Optional) token name or data (eg. '}'
 257                  */
 258                 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
 259                 {
 260
 261                     // accepts names or "{" etc..
 262                     var stop = in_stop;
 263                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 264
 265                     if (stop=="") {
 266                                 var newstop = Lang.matching(start);
 267                                 stop = newstop;
 268                         }
 269                         if (stop == null) {
 270                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 271                         }
 272                     debug("START=%s, STOP=%s \n", start,stop);
 273                     var depth = 0;
 274                     var got = new Gee.ArrayList<Token>();
 275                     var started = false;
 276                     //Seed.print("START:" + start);
 277                     //Seed.print("STOP:" + stop);
 278                     Token token;
 279
 280                     while (null != (token = this.look(1,false))) {
 281                                 debug("BALANCE: %d %s " , this.cursor,  token.asString());
 282                         if (token.is(start)) {
 283                       //      Seed.print("balance: START : " + depth + " " + token.data);
 284                             depth++;
 285                             started = true;
 286                         }
 287
 288                         if (started) {
 289                             got.add(token);
 290                         }
 291
 292                         if (token.is(stop)) {
 293                             depth--;
 294
 295                                 print("balance (%d): STOP: %s\n" ,  depth ,  token.data);
 296                             if (depth < 1) {
 297                                     this.next(); // shift cursor to eat closer...
 298                                         print("returning got %d\n", got.size);
 299                                         return got;
 300                                 }
 301
 302                         }
 303                         if (null == this.next()) {
 304                                 break;
 305                         }
 306                     }
 307                     return new Gee.ArrayList<Token>();
 308                 }
 309
 310                 public Token? getMatchingToken(string start, string stop)
 311                 {
 312                     var depth = 0;
 313                     var cursor = this.cursor;
 314
 315                     if (start.length < 1) {
 316                             var ns = Lang.matching(stop);
 317                         start = ns;
 318                         depth = 1;
 319                     }
 320                     if (stop.length < 1) {
 321                                 var ns = Lang.matching(start);
 322                                 stop = ns;
 323                         }
 324                         Token token;
 325
 326                     while (null != (token = this.tokens[cursor])) {
 327                         if (token.is(start)) {
 328                             depth++;
 329                         }
 330
 331                         if (token.is(stop) && cursor != 0) {
 332                             depth--;
 333                             if (depth == 0) {
 334                                         return this.tokens[cursor];
 335                                 }
 336                         }
 337                         cursor++;
 338                     }
 339                     return null;
 340                 }
 341                 /*
 342                 public Gee.ArrayList<Token> insertAhead(Token token)
 343                 {
 344                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 345                 }
 346                 */
 347
 348                 public Gee.ArrayList<Token> remaining() {
 349                     var ret = new Gee.ArrayList<Token>();
 350                     while (true) {
 351                         var tok = this.look(1,true);
 352                         if (tok.is("VOID")) {
 353                             return ret;
 354                         }
 355                         var nt = this.next();
 356                         if (nt != null) {
 357                                 ret.add(nt);
 358                         }
 359                     }
 360                 }
 361
 362
 363                 public void printRange(int start,  int end) {
 364
 365                         for(var i = start; i < end +1; i++) {
 366                     print(this.tokens.get(i).asString());
 367                         }
 368                 }
 369
 370                 /*
 371                 arrayToString : function(ar) {
 372                     console.log(typeof(ar));
 373                     var ret = [];
 374                     ar.forEach(function(e) {
 375                         ret.push(e.data);
 376                     })
 377                     return ret.join('');
 378                 },
 379                 */
 380                 public void dump(int start, int end)
 381                 {
 382                     start = int.max(start , 0);
 383                     end = int.min(end, this.tokens.size);
 384                     var  outs = "";;
 385                     for (var i =start;i < end; i++) {
 386
 387                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 388                     }
 389                     print(outs);
 390                 }
 391
 392                 public void dumpAll(string indent)
 393                 {
 394                     for (var i = 0;i < this.tokens.size; i++) {
 395
 396                          this.tokens[i].dump("");
 397                     }
 398
 399                 }
 400                 public void dumpAllFlat()
 401                 {
 402                     for (var i = 0;i < this.tokens.size; i++) {
 403
 404                          print("%d: %s\n", i, this.tokens[i].asString());
 405                     }
 406
 407                 }
 408
 409         }
 410 }
 411