git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31
  32
  33
  34                 public void rewind() {
  35                     this.cursor = -1;
  36                 }
  37
  38                 /**
  39                     @type JSDOC.Token
  40                 */
  41                 public Token? look (int n, bool considerWhitespace)
  42                 {
  43
  44
  45                     if (considerWhitespace == true) {
  46
  47                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  48                             return new Token("", "VOID", "START_OF_STREAM");
  49                         }
  50                         return this.tokens.get(this.cursor+n);
  51                     }
  52
  53
  54                 var count = 0;
  55                 var i = this.cursor;
  56
  57                 while (true) {
  58                     if (i < 0) {
  59                                 return new Token("", "VOID", "START_OF_STREAM");
  60                         }
  61                     if (i >= this.tokens.size) {
  62                                 return new Token("", "VOID", "END_OF_STREAM");
  63                         }
  64
  65                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  66                                 i += (n < 0) ? -1 : 1;
  67                         continue;
  68                     }
  69
  70                     if (count == n) {
  71                         return this.tokens.get(i);
  72                     }
  73                     count++;
  74                     i += (n < 0) ? -1 : 1;
  75                 }
  76
  77                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  78
  79                 }
  80
  81                 public int lookFor  (string data)
  82                 {
  83                     // non tree version..
  84                     var i = this.cursor < 0 ? 0 : this.cursor ;
  85
  86                     while (true) {
  87                         if (i >= this.tokens.size) {
  88                                 return -1;
  89                         }
  90                         if (this.tokens.get(i).data == data) {
  91                             return i;
  92                         }
  93                         i++;
  94
  95                     }
  96                     // should not get here!
  97                    // return -1;
  98
  99                 }
 100
 101
 102                 /**
 103                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 104                  * ?? used any more?
 105                  */
 106                 public Token lookTok (int n) {
 107
 108
 109
 110                     var count = 0;
 111                     var i = this.cursor;
 112
 113                     while (true) {
 114                        // print(i);
 115                         if (i < 0) {
 116                             if (n > -1) {
 117                                 i = 0;
 118                                 count++;
 119                                 continue;
 120
 121                             }
 122                             return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124                         if (i >= this.tokens.size) {
 125                                 return  new Token("", "VOID", "END_OF_STREAM");
 126                         }
 127
 128                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 129                             i += (n < 0) ? -1 : 1;
 130                             continue;
 131                         }
 132
 133                         if (count == n) {
 134                             return this.tokens.get(i);
 135                         }
 136                         count++;
 137                         i += (n < 0) ? -1 : 1;
 138                     }
 139                 // should never get here..
 140                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 141
 142                 }
 143
 144                 /**
 145                  *  @return {Token|null}
 146                  * next token (with white space)
 147                  */
 148
 149
 150                 public Token? next() {
 151
 152
 153                     //if (typeof howMany == "undefined") howMany = 1;
 154                     // if (howMany < 1) { return  null;                 }
 155
 156                     if (this.cursor+1 >= this.tokens.size) {
 157                         return null;
 158                 }
 159                     this.cursor++;
 160                     return this.tokens.get(this.cursor);
 161
 162                 }
 163
 164             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 165
 166                     //if (typeof howMany == "undefined") howMany = 1;
 167                     if (howMany < 2) {
 168                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 169                     }
 170                     var got = new Gee.ArrayList<Token>();
 171
 172                     for (var i = 1; i <= howMany; i++) {
 173                         if (this.cursor+i >= this.tokens.size) {
 174                             return null;
 175                         }
 176                         got.add(this.tokens.get(this.cursor+i));
 177                     }
 178                     this.cursor += howMany;
 179
 180                         return got;
 181                 }
 182
 183
 184
 185
 186                 // what about comments after 'function'...
 187                 // is this used ???
 188                 public Token? nextTok() {
 189                     return this.nextNonSpace();
 190                 }
 191
 192                 public Token? nextNonSpace ()
 193                 {
 194
 195                     while (true) {
 196                         var tok = this.next();
 197                         if (tok == null) {
 198                             return null;
 199                         }
 200                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 201                             continue;
 202                         }
 203                         return tok;
 204                     }
 205                 }
 206
 207                 /**
 208                  *  balance
 209                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 210                  * @param start {String}  token name or data (eg. '{'
 211                  * @param stop {String} (Optional) token name or data (eg. '}'
 212                  */
 213                 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
 214                 {
 215
 216                     // accepts names or "{" etc..
 217                     var stop = in_stop;
 218                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 219
 220                     if (stop=="") {
 221                                 var newstop = Lang.matching(start);
 222                                 stop = newstop;
 223                         }
 224                         if (stop == null) {
 225                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 226                         }
 227                     debug("START=%s, STOP=%s \n", start,stop);
 228                     var depth = 0;
 229                     var got = new Gee.ArrayList<Token>();
 230                     var started = false;
 231                     //Seed.print("START:" + start);
 232                     //Seed.print("STOP:" + stop);
 233                     Token token;
 234
 235                     while (null != (token = this.look(1,false))) {
 236                         if (token.is(start)) {
 237                       //      Seed.print("balance: START : " + depth + " " + token.data);
 238                             depth++;
 239                             started = true;
 240                         }
 241
 242                         if (started) {
 243                             got.add(token);
 244                         }
 245
 246                         if (token.is(stop)) {
 247                             depth--;
 248                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 249                             if (depth < 1) {
 250                                         return got;
 251                                 }
 252                         }
 253                         if (null == this.next()) {
 254                                 break;
 255                         }
 256                     }
 257                     return new Gee.ArrayList<Token>();
 258                 }
 259
 260                 public Token? getMatchingToken(string start, string stop)
 261                 {
 262                     var depth = 0;
 263                     var cursor = this.cursor;
 264
 265                     if (start.length < 1) {
 266                             var ns = Lang.matching(stop);
 267                         start = ns;
 268                         depth = 1;
 269                     }
 270                     if (stop.length < 1) {
 271                                 var ns = Lang.matching(start);
 272                                 stop = ns;
 273                         }
 274                         Token token;
 275
 276                     while (null != (token = this.tokens[cursor])) {
 277                         if (token.is(start)) {
 278                             depth++;
 279                         }
 280
 281                         if (token.is(stop) && cursor != 0) {
 282                             depth--;
 283                             if (depth == 0) {
 284                                         return this.tokens[cursor];
 285                                 }
 286                         }
 287                         cursor++;
 288                     }
 289                     return null;
 290                 }
 291                 /*
 292                 public Gee.ArrayList<Token> insertAhead(Token token)
 293                 {
 294                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 295                 }
 296                 */
 297
 298                 public Gee.ArrayList<Token> remaining() {
 299                     var ret = new Gee.ArrayList<Token>();
 300                     while (true) {
 301                         var tok = this.look(1,true);
 302                         if (tok.is("VOID")) {
 303                             return ret;
 304                         }
 305                         var nt = this.next();
 306                         if (nt != null) {
 307                                 ret.add(nt);
 308                         }
 309                     }
 310                 }
 311
 312
 313                 public void printRange(int start,  int end) {
 314
 315                         for(var i = start; i < end +1; i++) {
 316                     print(this.tokens.get(i).asString());
 317                         }
 318                 }
 319
 320                 /*
 321                 arrayToString : function(ar) {
 322                     console.log(typeof(ar));
 323                     var ret = [];
 324                     ar.forEach(function(e) {
 325                         ret.push(e.data);
 326                     })
 327                     return ret.join('');
 328                 },
 329                 */
 330                 public void dump(int start, int end)
 331                 {
 332                     start = int.max(start , 0);
 333                     end = int.min(end, this.tokens.size);
 334                     var  outs = "";;
 335                     for (var i =start;i < end; i++) {
 336
 337                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 338                     }
 339                     print(outs);
 340                 }
 341
 342                 public void dumpAll(string indent)
 343                 {
 344                     for (var i = 0;i < this.tokens.size; i++) {
 345
 346                          this.tokens[i].dump("");
 347                     }
 348
 349                 }
 350
 351
 352         }
 353 }
 354