git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 Gee.ArrayList<Token> tokens;
  22                 int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31
  32
  33
  34                 public void rewind() {
  35                     this.cursor = -1;
  36                 }
  37
  38                 /**
  39                     @type JSDOC.Token
  40                 */
  41                 public Token? look (int n, bool considerWhitespace)
  42                 {
  43
  44
  45                     if (considerWhitespace == true) {
  46
  47                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  48                             return new Token("", "VOID", "START_OF_STREAM");
  49                         }
  50                         return this.tokens.get(this.cursor+n);
  51                     }
  52
  53
  54                 var count = 0;
  55                 var i = this.cursor;
  56
  57                 while (true) {
  58                     if (i < 0) {
  59                                 return new Token("", "VOID", "START_OF_STREAM");
  60                         }
  61                     if (i > this.tokens.size) {
  62                                 return new Token("", "VOID", "END_OF_STREAM");
  63                         }
  64
  65                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  66                                 i += (n < 0) ? -1 : 1;
  67                         continue;
  68                     }
  69
  70                     if (count == n) {
  71                         return this.tokens.get(i);
  72                     }
  73                     count++;
  74                     i += (n < 0) ? -1 : 1;
  75                 }
  76
  77                 return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  78
  79                 }
  80
  81                 public int lookFor  (string data)
  82                 {
  83                     // non tree version..
  84                     var i = this.cursor < 0 ? 0 : this.cursor ;
  85
  86                     while (true) {
  87                         if (i >= this.tokens.size) {
  88                                 return -1;
  89                         }
  90                         if (this.tokens.get(i).data == data) {
  91                             return i;
  92                         }
  93                         i++;
  94
  95                     }
  96                     // should not get here!
  97                     return -1;
  98
  99                 }
 100
 101
 102                 /**
 103                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 104                  * ?? used any more?
 105                  */
 106                 public Token lookTok (int n) {
 107
 108
 109
 110                     var count = 0;
 111                     var i = this.cursor;
 112
 113                     while (true) {
 114                        // print(i);
 115                         if (i < 0) {
 116                             if (n > -1) {
 117                                 i = 0;
 118                                 count++;
 119                                 continue;
 120
 121                             }
 122                             return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124                         if (i > this.tokens.size) {
 125                                 return  new Token("", "VOID", "END_OF_STREAM");
 126                         }
 127
 128                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 129                             i += (n < 0) ? -1 : 1;
 130                             continue;
 131                         }
 132
 133                         if (count == n) {
 134                             return this.tokens.get(i);
 135                         }
 136                         count++;
 137                         i += (n < 0) ? -1 : 1;
 138                     }
 139                 // should never get here..
 140                     return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 141
 142                 }
 143
 144                 /**
 145                  *  @return {Token|null}
 146                  * next token (with white space)
 147                  */
 148
 149
 150                 public Token? next() {
 151
 152
 153                     //if (typeof howMany == "undefined") howMany = 1;
 154                     // if (howMany < 1) { return  null;                 }
 155
 156                     if (this.cursor+1 >= this.tokens.size) {
 157                         return null;
 158                 }
 159                     this.cursor++;
 160                     return this.tokens.get(this.cursor);
 161
 162                 }
 163
 164             public Gee.ArrayList<Token>? nextM(int howMany) throws TokenStreamError {
 165
 166                     //if (typeof howMany == "undefined") howMany = 1;
 167                     if (howMany < 2) {
 168                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", howMany);
 169                     }
 170                     var got = new Gee.ArrayList<Token>();
 171
 172                     for (var i = 1; i <= howMany; i++) {
 173                         if (this.cursor+i >= this.tokens.size) {
 174                             return null;
 175                         }
 176                         got.add(this.tokens.get(this.cursor+i));
 177                     }
 178                     this.cursor += howMany;
 179
 180                         return got;
 181                 }
 182
 183
 184
 185
 186                 // what about comments after 'function'...
 187                 // is this used ???
 188                 public Token? nextTok() {
 189                     return this.nextNonSpace();
 190                 }
 191
 192                 public Token? nextNonSpace ()
 193                 {
 194
 195                     while (true) {
 196                         var tok = this.next();
 197                         if (tok == null) {
 198                             return null;
 199                         }
 200                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 201                             continue;
 202                         }
 203                         return tok;
 204                     }
 205                 }
 206
 207                 /**
 208                  *  balance
 209                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 210                  * @param start {String}  token name or data (eg. '{'
 211                  * @param stop {String} (Optional) token name or data (eg. '}'
 212                  */
 213                 public Gee.ArrayList<Token> balance (string start, string stop = "") throws TokenStreamError {
 214
 215                     // accepts names or "{" etc..
 216
 217                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 218
 219                     if (stop=="") {
 220                                 var newstop = Lang.matching(start);
 221                                 stop = newstop;
 222                         }
 223                         if (stop == null) {
 224                                 throw new TokenStreamError.ArgumentError("balance called with invalid start/stop : %s",start);
 225                         }
 226
 227                     var depth = 0;
 228                     var got = new Gee.ArrayList<Token>();
 229                     var started = false;
 230                     //Seed.print("START:" + start);
 231                     //Seed.print("STOP:" + stop);
 232                     Token token;
 233
 234                     while ((token = this.look(1,false))) {
 235                         if (token.is(start)) {
 236                       //      Seed.print("balance: START : " + depth + " " + token.data);
 237                             depth++;
 238                             started = true;
 239                         }
 240
 241                         if (started) {
 242                             got.push(token);
 243                         }
 244
 245                         if (token.is(stop)) {
 246                             depth--;
 247                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 248                             if (depth < 1) {
 249                                         return got;
 250                                 }
 251                         }
 252                         if (!this.next()) {
 253                                 break;
 254                         }
 255                     }
 256                     return new Gee.ArrayList<Token>();
 257                 }
 258
 259                 public Token getMatchingToken(string start, string stop) {
 260                     var depth = 0;
 261                     var cursor = this.cursor;
 262
 263                     if (!start) {
 264                         start = Lang.matching(stop);
 265                         depth = 1;
 266                     }
 267                     if (!stop) stop = Lang.matching(start);
 268
 269                     while ((token = this.tokens[cursor])) {
 270                         if (token.is(start)) {
 271                             depth++;
 272                         }
 273
 274                         if (token.is(stop) && cursor) {
 275                             depth--;
 276                             if (depth == 0) return this.tokens[cursor];
 277                         }
 278                         cursor++;
 279                     }
 280                     return false;
 281                 }
 282
 283                 public Gee.ArrayList<Token> insertAhead(Token token) {
 284                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 285                 }
 286
 287                 public Gee.ArrayList<Token> remaining() {
 288                     var ret = new Gee.ArrayList<Token>();
 289                     while (true) {
 290                         var tok = this.look(1,true);
 291                         if (tok.is("VOID")) {
 292                             return ret;
 293                         }
 294                         ret.push(this.next(1));
 295                     }
 296                 }
 297
 298                 /*
 299                 arrayToString : function(ar) {
 300                     console.log(typeof(ar));
 301                     var ret = [];
 302                     ar.forEach(function(e) {
 303                         ret.push(e.data);
 304                     })
 305                     return ret.join('');
 306                 },
 307                 dump: function(start, end)
 308                 {
 309                     start = Math.max(start || 0, 0);
 310                     end = Math.min(end || this.tokens.length, this.tokens.length);
 311                     var out='';
 312                     for (var i =start;i < end; i++) {
 313
 314                         out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
 315                     };
 316                     print(out);
 317                 }
 318                 */
 319         }
 320 }
 321