git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31                 public  Gee.ArrayList<Token> toArray()
  32                 {
  33                         return this.tokens;
  34                 }
  35
  36
  37                 public void rewind() {
  38                     this.cursor = -1;
  39                 }
  40
  41                 /**
  42                     @type JSDOC.Token
  43                 */
  44                 public Token? look (int n, bool considerWhitespace)  // depricated... causes all sorts of problems...
  45                 {
  46
  47
  48                     if (considerWhitespace == true) {
  49
  50                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  51                             return new Token("", "VOID", "START_OF_STREAM");
  52                         }
  53                         return this.tokens.get(this.cursor+n);
  54                     }
  55
  56
  57                 var count = 0;
  58                 var i = this.cursor;
  59
  60                 while (true) {
  61                     if (i < 0) {
  62                                 return new Token("", "VOID", "START_OF_STREAM");
  63                         }
  64                     if (i >= this.tokens.size) {
  65                                 return new Token("", "VOID", "END_OF_STREAM");
  66                         }
  67
  68                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  69                                 i += (n < 0) ? -1 : 1;
  70                         continue;
  71                     }
  72
  73                     if (count == n) {
  74                         return this.tokens.get(i);
  75                     }
  76                     count++;
  77                     i += (n < 0) ? -1 : 1;
  78                 }
  79
  80                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  81
  82                 }
  83                 // look through token stream, including white space...
  84                 public Token  lookAny (int n)
  85                 {
  86
  87
  88                  if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  89                     return new Token("", "VOID", "START_OF_STREAM");
  90                 }
  91                 return this.tokens.get(this.cursor+n);
  92
  93
  94
  95                 }
  96
  97
  98
  99                 public int lookFor  (string data)
 100                 {
 101                     // non tree version..
 102                     var i = this.cursor < 0 ? 0 : this.cursor ;
 103
 104                     while (true) {
 105                         if (i >= this.tokens.size) {
 106                                 return -1;
 107                         }
 108                         if (this.tokens.get(i).data == data) {
 109                             return i;
 110                         }
 111                         i++;
 112
 113                     }
 114                     // should not get here!
 115                    // return -1;
 116
 117                 }
 118
 119
 120                 /**
 121                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 122                  * ?? used any more?
 123                  */
 124                 public Token lookTok (int n) {
 125
 126
 127
 128                     var count = 0;
 129                     var i = this.cursor;
 130
 131                     while (true) {
 132                        // print(i);
 133                         if (i < 0) {
 134                             if (n > -1) {
 135                                 i = 0;
 136                                 count++;
 137                                 continue;
 138
 139                             }
 140                             return  new Token("", "VOID", "END_OF_STREAM");
 141                         }
 142                         if (i >= this.tokens.size) {
 143                                 return  new Token("", "VOID", "END_OF_STREAM");
 144                         }
 145
 146                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 147                             i += (n < 0) ? -1 : 1;
 148                             continue;
 149                         }
 150
 151                         if (count == n) {
 152                             return this.tokens.get(i);
 153                         }
 154                         count++;
 155                         i += (n < 0) ? -1 : 1;
 156                     }
 157                 // should never get here..
 158                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 159
 160                 }
 161
 162                 /**
 163                  *  @return {Token|null}
 164                  * next token (with white space)
 165                  */
 166
 167
 168                 public Token? next() {
 169
 170
 171                     //if (typeof howMany == "undefined") howMany = 1;
 172                     // if (howMany < 1) { return  null;                 }
 173
 174                     if (this.cursor+1 >= this.tokens.size) {
 175                         return null;
 176                 }
 177                     this.cursor++;
 178                     return this.tokens.get(this.cursor);
 179
 180                 }
 181
 182             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 183
 184                     //if (typeof howMany == "undefined") howMany = 1;
 185                     if (howMany < 2) {
 186                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 187                     }
 188                     var got = new Gee.ArrayList<Token>();
 189
 190                     for (var i = 1; i <= howMany; i++) {
 191                         if (this.cursor+i >= this.tokens.size) {
 192                             return null;
 193                         }
 194                         got.add(this.tokens.get(this.cursor+i));
 195                     }
 196                     this.cursor += howMany;
 197
 198                         return got;
 199                 }
 200
 201
 202
 203
 204                 // what about comments after 'function'...
 205                 // is this used ???
 206                 public Token? nextTok() {
 207                     return this.nextNonSpace();
 208                 }
 209
 210                 public Token? nextNonSpace ()
 211                 {
 212
 213                     while (true) {
 214                         var tok = this.next();
 215                         if (tok == null) {
 216                             return null;
 217                         }
 218                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 219                             continue;
 220                         }
 221                         return tok;
 222                     }
 223                 }
 224
 225                 /**
 226                  *  balance
 227                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 228                  * @param start {String}  token name or data (eg. '{'
 229                  * @param stop {String} (Optional) token name or data (eg. '}'
 230                  */
 231                 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
 232                 {
 233
 234                     // accepts names or "{" etc..
 235                     var stop = in_stop;
 236                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 237
 238                     if (stop=="") {
 239                                 var newstop = Lang.matching(start);
 240                                 stop = newstop;
 241                         }
 242                         if (stop == null) {
 243                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 244                         }
 245                     debug("START=%s, STOP=%s \n", start,stop);
 246                     var depth = 0;
 247                     var got = new Gee.ArrayList<Token>();
 248                     var started = false;
 249                     //Seed.print("START:" + start);
 250                     //Seed.print("STOP:" + stop);
 251                     Token token;
 252
 253                     while (null != (token = this.lookAny(1))) {
 254                                 debug("BALANCE: %d %s " , this.cursor,  token.asString());
 255                         if (token.is(start)) {
 256                       //      Seed.print("balance: START : " + depth + " " + token.data);
 257                             depth++;
 258                             started = true;
 259                         }
 260
 261                         if (started) {
 262                             got.add(token);
 263                         }
 264
 265                         if (token.is(stop)) {
 266                             depth--;
 267
 268                                 //print("balance (%d): STOP: %s\n" ,  depth ,  token.data);
 269                             if (depth < 1) {
 270                                     this.next(); // shift cursor to eat closer...
 271                                         //print("returning got %d\n", got.size);
 272                                         return got;
 273                                 }
 274
 275                         }
 276                         if (null == this.next()) {
 277                                 break;
 278                         }
 279                     }
 280                     return new Gee.ArrayList<Token>();
 281                 }
 282
 283                 public Token? getMatchingToken(string start, string stop)
 284                 {
 285                     var depth = 0;
 286                     var cursor = this.cursor;
 287
 288                     if (start.length < 1) {
 289                             var ns = Lang.matching(stop);
 290                         start = ns;
 291                         depth = 1;
 292                     }
 293                     if (stop.length < 1) {
 294                                 var ns = Lang.matching(start);
 295                                 stop = ns;
 296                         }
 297                         Token token;
 298
 299                     while (null != (token = this.tokens[cursor])) {
 300                         if (token.is(start)) {
 301                             depth++;
 302                         }
 303
 304                         if (token.is(stop) && cursor != 0) {
 305                             depth--;
 306                             if (depth == 0) {
 307                                         return this.tokens[cursor];
 308                                 }
 309                         }
 310                         cursor++;
 311                     }
 312                     return null;
 313                 }
 314                 /*
 315                 public Gee.ArrayList<Token> insertAhead(Token token)
 316                 {
 317                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 318                 }
 319                 */
 320
 321                 public Gee.ArrayList<Token> remaining() {
 322                     var ret = new Gee.ArrayList<Token>();
 323                     while (true) {
 324                         var tok = this.look(1,true);
 325                         if (tok.is("VOID")) {
 326                             return ret;
 327                         }
 328                         var nt = this.next();
 329                         if (nt != null) {
 330                                 ret.add(nt);
 331                         }
 332                     }
 333                 }
 334
 335
 336                 public void printRange(int start,  int end) {
 337
 338                         for(var i = start; i < end +1; i++) {
 339                     print(this.tokens.get(i).asString());
 340                         }
 341                 }
 342
 343                 /*
 344                 arrayToString : function(ar) {
 345                     console.log(typeof(ar));
 346                     var ret = [];
 347                     ar.forEach(function(e) {
 348                         ret.push(e.data);
 349                     })
 350                     return ret.join('');
 351                 },
 352                 */
 353                 public void dump(int start, int end)
 354                 {
 355                     start = int.max(start , 0);
 356                     end = int.min(end, this.tokens.size);
 357                     var  outs = "";;
 358                     for (var i =start;i < end; i++) {
 359
 360                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 361                     }
 362                     print(outs);
 363                 }
 364
 365                 public void dumpAll(string indent)
 366                 {
 367                     for (var i = 0;i < this.tokens.size; i++) {
 368
 369                          this.tokens[i].dump("");
 370                     }
 371
 372                 }
 373                 public void dumpAllFlat()
 374                 {
 375                     for (var i = 0;i < this.tokens.size; i++) {
 376
 377                          print("%d: %s\n", i, this.tokens[i].asString());
 378                     }
 379
 380                 }
 381
 382         }
 383 }
 384