git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31                 public  Gee.ArrayList<Token> toArray()
  32                 {
  33                         return this.tokens;
  34                 }
  35
  36
  37                 public void rewind() {
  38                     this.cursor = -1;
  39                 }
  40
  41                 /**
  42                     @type JSDOC.Token
  43                 */
  44                 public Token? look (int n, bool considerWhitespace)  // depricated... causes all sorts of problems...
  45                 {
  46
  47
  48                     if (considerWhitespace == true) {
  49
  50                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  51                             return new Token("", "VOID", "START_OF_STREAM");
  52                         }
  53                         return this.tokens.get(this.cursor+n);
  54                     }
  55
  56
  57                 var count = 0;
  58                 var i = this.cursor;
  59
  60                 while (true) {
  61                     if (i < 0) {
  62                                 return new Token("", "VOID", "START_OF_STREAM");
  63                         }
  64                     if (i >= this.tokens.size) {
  65                                 return new Token("", "VOID", "END_OF_STREAM");
  66                         }
  67
  68                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  69                                 i += (n < 0) ? -1 : 1;
  70                         continue;
  71                     }
  72
  73                     if (count == n) {
  74                         return this.tokens.get(i);
  75                     }
  76                     count++;
  77                     i += (n < 0) ? -1 : 1;
  78                 }
  79
  80                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  81
  82                 }
  83
  84
  85
  86
  87
  88                 public int lookFor  (string data)
  89                 {
  90                     // non tree version..
  91                     var i = this.cursor < 0 ? 0 : this.cursor ;
  92
  93                     while (true) {
  94                         if (i >= this.tokens.size) {
  95                                 return -1;
  96                         }
  97                         if (this.tokens.get(i).data == data) {
  98                             return i;
  99                         }
 100                         i++;
 101
 102                     }
 103                     // should not get here!
 104                    // return -1;
 105
 106                 }
 107
 108
 109                 /**
 110                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 111                  * ?? used any more?
 112                  */
 113                 public Token lookTok (int n) {
 114
 115
 116
 117                     var count = 0;
 118                     var i = this.cursor;
 119
 120                     while (true) {
 121                        // print(i);
 122                         if (i < 0) {
 123                             if (n > -1) {
 124                                 i = 0;
 125                                 count++;
 126                                 continue;
 127
 128                             }
 129                             return  new Token("", "VOID", "END_OF_STREAM");
 130                         }
 131                         if (i >= this.tokens.size) {
 132                                 return  new Token("", "VOID", "END_OF_STREAM");
 133                         }
 134
 135                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 136                             i += (n < 0) ? -1 : 1;
 137                             continue;
 138                         }
 139
 140                         if (count == n) {
 141                             return this.tokens.get(i);
 142                         }
 143                         count++;
 144                         i += (n < 0) ? -1 : 1;
 145                     }
 146                 // should never get here..
 147                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 148
 149                 }
 150
 151                 /**
 152                  *  @return {Token|null}
 153                  * next token (with white space)
 154                  */
 155
 156
 157                 public Token? next() {
 158
 159
 160                     //if (typeof howMany == "undefined") howMany = 1;
 161                     // if (howMany < 1) { return  null;                 }
 162
 163                     if (this.cursor+1 >= this.tokens.size) {
 164                         return null;
 165                 }
 166                     this.cursor++;
 167                     return this.tokens.get(this.cursor);
 168
 169                 }
 170
 171             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 172
 173                     //if (typeof howMany == "undefined") howMany = 1;
 174                     if (howMany < 2) {
 175                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 176                     }
 177                     var got = new Gee.ArrayList<Token>();
 178
 179                     for (var i = 1; i <= howMany; i++) {
 180                         if (this.cursor+i >= this.tokens.size) {
 181                             return null;
 182                         }
 183                         got.add(this.tokens.get(this.cursor+i));
 184                     }
 185                     this.cursor += howMany;
 186
 187                         return got;
 188                 }
 189
 190
 191
 192
 193                 // what about comments after 'function'...
 194                 // is this used ???
 195                 public Token? nextTok() {
 196                     return this.nextNonSpace();
 197                 }
 198
 199                 public Token? nextNonSpace ()
 200                 {
 201
 202                     while (true) {
 203                         var tok = this.next();
 204                         if (tok == null) {
 205                             return null;
 206                         }
 207                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 208                             continue;
 209                         }
 210                         return tok;
 211                     }
 212                 }
 213
 214                 /**
 215                  *  balance
 216                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 217                  * @param start {String}  token name or data (eg. '{'
 218                  * @param stop {String} (Optional) token name or data (eg. '}'
 219                  */
 220                 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
 221                 {
 222
 223                     // accepts names or "{" etc..
 224                     var stop = in_stop;
 225                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 226
 227                     if (stop=="") {
 228                                 var newstop = Lang.matching(start);
 229                                 stop = newstop;
 230                         }
 231                         if (stop == null) {
 232                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 233                         }
 234                     debug("START=%s, STOP=%s \n", start,stop);
 235                     var depth = 0;
 236                     var got = new Gee.ArrayList<Token>();
 237                     var started = false;
 238                     //Seed.print("START:" + start);
 239                     //Seed.print("STOP:" + stop);
 240                     Token token;
 241
 242                     while (null != (token = this.look(1,false))) {
 243                                 debug("BALANCE: %d %s " , this.cursor,  token.asString());
 244                         if (token.is(start)) {
 245                       //      Seed.print("balance: START : " + depth + " " + token.data);
 246                             depth++;
 247                             started = true;
 248                         }
 249
 250                         if (started) {
 251                             got.add(token);
 252                         }
 253
 254                         if (token.is(stop)) {
 255                             depth--;
 256
 257                                 print("balance (%d): STOP: %s\n" ,  depth ,  token.data);
 258                             if (depth < 1) {
 259                                     this.next(); // shift cursor to eat closer...
 260                                         print("returning got %d\n", got.size);
 261                                         return got;
 262                                 }
 263
 264                         }
 265                         if (null == this.next()) {
 266                                 break;
 267                         }
 268                     }
 269                     return new Gee.ArrayList<Token>();
 270                 }
 271
 272                 public Token? getMatchingToken(string start, string stop)
 273                 {
 274                     var depth = 0;
 275                     var cursor = this.cursor;
 276
 277                     if (start.length < 1) {
 278                             var ns = Lang.matching(stop);
 279                         start = ns;
 280                         depth = 1;
 281                     }
 282                     if (stop.length < 1) {
 283                                 var ns = Lang.matching(start);
 284                                 stop = ns;
 285                         }
 286                         Token token;
 287
 288                     while (null != (token = this.tokens[cursor])) {
 289                         if (token.is(start)) {
 290                             depth++;
 291                         }
 292
 293                         if (token.is(stop) && cursor != 0) {
 294                             depth--;
 295                             if (depth == 0) {
 296                                         return this.tokens[cursor];
 297                                 }
 298                         }
 299                         cursor++;
 300                     }
 301                     return null;
 302                 }
 303                 /*
 304                 public Gee.ArrayList<Token> insertAhead(Token token)
 305                 {
 306                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 307                 }
 308                 */
 309
 310                 public Gee.ArrayList<Token> remaining() {
 311                     var ret = new Gee.ArrayList<Token>();
 312                     while (true) {
 313                         var tok = this.look(1,true);
 314                         if (tok.is("VOID")) {
 315                             return ret;
 316                         }
 317                         var nt = this.next();
 318                         if (nt != null) {
 319                                 ret.add(nt);
 320                         }
 321                     }
 322                 }
 323
 324
 325                 public void printRange(int start,  int end) {
 326
 327                         for(var i = start; i < end +1; i++) {
 328                     print(this.tokens.get(i).asString());
 329                         }
 330                 }
 331
 332                 /*
 333                 arrayToString : function(ar) {
 334                     console.log(typeof(ar));
 335                     var ret = [];
 336                     ar.forEach(function(e) {
 337                         ret.push(e.data);
 338                     })
 339                     return ret.join('');
 340                 },
 341                 */
 342                 public void dump(int start, int end)
 343                 {
 344                     start = int.max(start , 0);
 345                     end = int.min(end, this.tokens.size);
 346                     var  outs = "";;
 347                     for (var i =start;i < end; i++) {
 348
 349                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 350                     }
 351                     print(outs);
 352                 }
 353
 354                 public void dumpAll(string indent)
 355                 {
 356                     for (var i = 0;i < this.tokens.size; i++) {
 357
 358                          this.tokens[i].dump("");
 359                     }
 360
 361                 }
 362                 public void dumpAllFlat()
 363                 {
 364                     for (var i = 0;i < this.tokens.size; i++) {
 365
 366                          print("%d: %s\n", i, this.tokens[i].asString());
 367                     }
 368
 369                 }
 370
 371         }
 372 }
 373