git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 protected Gee.ArrayList<Token> tokens;
  22                 public int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31                 public  Gee.ArrayList<Token> toArray()
  32                 {
  33                         return this.tokens;
  34                 }
  35
  36
  37                 public void rewind() {
  38                     this.cursor = -1;
  39                 }
  40
  41                 /**
  42                     @type JSDOC.Token
  43                 */
  44                 public Token? look (int n, bool considerWhitespace)
  45                 {
  46
  47
  48                     if (considerWhitespace == true) {
  49
  50                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  51                             return new Token("", "VOID", "START_OF_STREAM");
  52                         }
  53                         return this.tokens.get(this.cursor+n);
  54                     }
  55
  56
  57                 var count = 0;
  58                 var i = this.cursor;
  59
  60                 while (true) {
  61                     if (i < 0) {
  62                                 return new Token("", "VOID", "START_OF_STREAM");
  63                         }
  64                     if (i >= this.tokens.size) {
  65                                 return new Token("", "VOID", "END_OF_STREAM");
  66                         }
  67
  68                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  69                                 i += (n < 0) ? -1 : 1;
  70                         continue;
  71                     }
  72
  73                     if (count == n) {
  74                         return this.tokens.get(i);
  75                     }
  76                     count++;
  77                     i += (n < 0) ? -1 : 1;
  78                 }
  79
  80                // return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  81
  82                 }
  83
  84                 public int lookFor  (string data)
  85                 {
  86                     // non tree version..
  87                     var i = this.cursor < 0 ? 0 : this.cursor ;
  88
  89                     while (true) {
  90                         if (i >= this.tokens.size) {
  91                                 return -1;
  92                         }
  93                         if (this.tokens.get(i).data == data) {
  94                             return i;
  95                         }
  96                         i++;
  97
  98                     }
  99                     // should not get here!
 100                    // return -1;
 101
 102                 }
 103
 104
 105                 /**
 106                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 107                  * ?? used any more?
 108                  */
 109                 public Token lookTok (int n) {
 110
 111
 112
 113                     var count = 0;
 114                     var i = this.cursor;
 115
 116                     while (true) {
 117                        // print(i);
 118                         if (i < 0) {
 119                             if (n > -1) {
 120                                 i = 0;
 121                                 count++;
 122                                 continue;
 123
 124                             }
 125                             return  new Token("", "VOID", "END_OF_STREAM");
 126                         }
 127                         if (i >= this.tokens.size) {
 128                                 return  new Token("", "VOID", "END_OF_STREAM");
 129                         }
 130
 131                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 132                             i += (n < 0) ? -1 : 1;
 133                             continue;
 134                         }
 135
 136                         if (count == n) {
 137                             return this.tokens.get(i);
 138                         }
 139                         count++;
 140                         i += (n < 0) ? -1 : 1;
 141                     }
 142                 // should never get here..
 143                 //    return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 144
 145                 }
 146
 147                 /**
 148                  *  @return {Token|null}
 149                  * next token (with white space)
 150                  */
 151
 152
 153                 public Token? next() {
 154
 155
 156                     //if (typeof howMany == "undefined") howMany = 1;
 157                     // if (howMany < 1) { return  null;                 }
 158
 159                     if (this.cursor+1 >= this.tokens.size) {
 160                         return null;
 161                 }
 162                     this.cursor++;
 163                     return this.tokens.get(this.cursor);
 164
 165                 }
 166
 167             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 168
 169                     //if (typeof howMany == "undefined") howMany = 1;
 170                     if (howMany < 2) {
 171                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 172                     }
 173                     var got = new Gee.ArrayList<Token>();
 174
 175                     for (var i = 1; i <= howMany; i++) {
 176                         if (this.cursor+i >= this.tokens.size) {
 177                             return null;
 178                         }
 179                         got.add(this.tokens.get(this.cursor+i));
 180                     }
 181                     this.cursor += howMany;
 182
 183                         return got;
 184                 }
 185
 186
 187
 188
 189                 // what about comments after 'function'...
 190                 // is this used ???
 191                 public Token? nextTok() {
 192                     return this.nextNonSpace();
 193                 }
 194
 195                 public Token? nextNonSpace ()
 196                 {
 197
 198                     while (true) {
 199                         var tok = this.next();
 200                         if (tok == null) {
 201                             return null;
 202                         }
 203                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 204                             continue;
 205                         }
 206                         return tok;
 207                     }
 208                 }
 209
 210                 /**
 211                  *  balance
 212                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 213                  * @param start {String}  token name or data (eg. '{'
 214                  * @param stop {String} (Optional) token name or data (eg. '}'
 215                  */
 216                 public Gee.ArrayList<Token> balance (string start, string in_stop = "") throws TokenStreamError
 217                 {
 218
 219                     // accepts names or "{" etc..
 220                     var stop = in_stop;
 221                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 222
 223                     if (stop=="") {
 224                                 var newstop = Lang.matching(start);
 225                                 stop = newstop;
 226                         }
 227                         if (stop == null) {
 228                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 229                         }
 230                     debug("START=%s, STOP=%s \n", start,stop);
 231                     var depth = 0;
 232                     var got = new Gee.ArrayList<Token>();
 233                     var started = false;
 234                     //Seed.print("START:" + start);
 235                     //Seed.print("STOP:" + stop);
 236                     Token token;
 237
 238                     while (null != (token = this.look(1,false))) {
 239                                 debug("BALANCE: %d %s " , this.cursor,  token.asString());
 240                         if (token.is(start)) {
 241                       //      Seed.print("balance: START : " + depth + " " + token.data);
 242                             depth++;
 243                             started = true;
 244                         }
 245
 246                         if (started) {
 247                             got.add(token);
 248                         }
 249
 250                         if (token.is(stop)) {
 251                             depth--;
 252
 253                                 print("balance (%d): STOP: %s\n" ,  depth ,  token.data);
 254                             if (depth < 1) {
 255                                     this.next(); // shift cursor to eat closer...
 256                                         print("returning got %d\n", got.size);
 257                                         return got;
 258                                 }
 259
 260                         }
 261                         if (null == this.next()) {
 262                                 break;
 263                         }
 264                     }
 265                     return new Gee.ArrayList<Token>();
 266                 }
 267
 268                 public Token? getMatchingToken(string start, string stop)
 269                 {
 270                     var depth = 0;
 271                     var cursor = this.cursor;
 272
 273                     if (start.length < 1) {
 274                             var ns = Lang.matching(stop);
 275                         start = ns;
 276                         depth = 1;
 277                     }
 278                     if (stop.length < 1) {
 279                                 var ns = Lang.matching(start);
 280                                 stop = ns;
 281                         }
 282                         Token token;
 283
 284                     while (null != (token = this.tokens[cursor])) {
 285                         if (token.is(start)) {
 286                             depth++;
 287                         }
 288
 289                         if (token.is(stop) && cursor != 0) {
 290                             depth--;
 291                             if (depth == 0) {
 292                                         return this.tokens[cursor];
 293                                 }
 294                         }
 295                         cursor++;
 296                     }
 297                     return null;
 298                 }
 299                 /*
 300                 public Gee.ArrayList<Token> insertAhead(Token token)
 301                 {
 302                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 303                 }
 304                 */
 305
 306                 public Gee.ArrayList<Token> remaining() {
 307                     var ret = new Gee.ArrayList<Token>();
 308                     while (true) {
 309                         var tok = this.look(1,true);
 310                         if (tok.is("VOID")) {
 311                             return ret;
 312                         }
 313                         var nt = this.next();
 314                         if (nt != null) {
 315                                 ret.add(nt);
 316                         }
 317                     }
 318                 }
 319
 320
 321                 public void printRange(int start,  int end) {
 322
 323                         for(var i = start; i < end +1; i++) {
 324                     print(this.tokens.get(i).asString());
 325                         }
 326                 }
 327
 328                 /*
 329                 arrayToString : function(ar) {
 330                     console.log(typeof(ar));
 331                     var ret = [];
 332                     ar.forEach(function(e) {
 333                         ret.push(e.data);
 334                     })
 335                     return ret.join('');
 336                 },
 337                 */
 338                 public void dump(int start, int end)
 339                 {
 340                     start = int.max(start , 0);
 341                     end = int.min(end, this.tokens.size);
 342                     var  outs = "";;
 343                     for (var i =start;i < end; i++) {
 344
 345                         outs += (this.tokens[i].outData == "") ? this.tokens[i].data : this.tokens[i].outData;
 346                     }
 347                     print(outs);
 348                 }
 349
 350                 public void dumpAll(string indent)
 351                 {
 352                     for (var i = 0;i < this.tokens.size; i++) {
 353
 354                          this.tokens[i].dump("");
 355                     }
 356
 357                 }
 358
 359
 360         }
 361 }
 362