git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         errordomain TokenStreamError {
  16             ArgumentError
  17     }
  18         public class TokenStream : Object
  19         {
  20
  21                 Gee.ArrayList<Token> tokens;
  22                 int cursor; // where are we in the stream.
  23
  24
  25                 public TokenStream(Gee.ArrayList<Token> tokens) {
  26
  27                         this.tokens = tokens;
  28
  29                         this.rewind();
  30                 }
  31
  32
  33
  34                 public void rewind() {
  35                     this.cursor = -1;
  36                 }
  37
  38                 /**
  39                     @type JSDOC.Token
  40                 */
  41                 public Token? look (int n, bool considerWhitespace)
  42                 {
  43
  44
  45                     if (considerWhitespace == true) {
  46
  47                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  48                             return new Token("", "VOID", "START_OF_STREAM");
  49                         }
  50                         return this.tokens.get(this.cursor+n);
  51                     }
  52
  53
  54                 var count = 0;
  55                 var i = this.cursor;
  56
  57                 while (true) {
  58                     if (i < 0) {
  59                                 return new Token("", "VOID", "START_OF_STREAM");
  60                         }
  61                     if (i > this.tokens.size) {
  62                                 return new Token("", "VOID", "END_OF_STREAM");
  63                         }
  64
  65                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  66                                 i += (n < 0) ? -1 : 1;
  67                         continue;
  68                     }
  69
  70                     if (count == n) {
  71                         return this.tokens.get(i);
  72                     }
  73                     count++;
  74                     i += (n < 0) ? -1 : 1;
  75                 }
  76
  77                 return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  78
  79                 }
  80
  81                 public int lookFor  (string data)
  82                 {
  83                     // non tree version..
  84                     var i = this.cursor < 0 ? 0 : this.cursor ;
  85
  86                     while (true) {
  87                         if (i >= this.tokens.size) {
  88                                 return -1;
  89                         }
  90                         if (this.tokens.get(i).data == data) {
  91                             return i;
  92                         }
  93                         i++;
  94
  95                     }
  96                     // should not get here!
  97                     return -1;
  98
  99                 }
 100
 101
 102                 /**
 103                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 104                  * ?? used any more?
 105                  */
 106                 public Token lookTok (int n) {
 107
 108
 109
 110                     var count = 0;
 111                     var i = this.cursor;
 112
 113                     while (true) {
 114                        // print(i);
 115                         if (i < 0) {
 116                             if (n > -1) {
 117                                 i = 0;
 118                                 count++;
 119                                 continue;
 120
 121                             }
 122                             return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124                         if (i > this.tokens.size) {
 125                                 return  new Token("", "VOID", "END_OF_STREAM");
 126                         }
 127
 128                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 129                             i += (n < 0) ? -1 : 1;
 130                             continue;
 131                         }
 132
 133                         if (count == n) {
 134                             return this.tokens.get(i);
 135                         }
 136                         count++;
 137                         i += (n < 0) ? -1 : 1;
 138                     }
 139                 // should never get here..
 140                     return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 141
 142                 }
 143
 144                 /**
 145                  *  @return {Token|null}
 146                  * next token (with white space)
 147                  */
 148
 149
 150                 public Token? next() {
 151
 152
 153                     //if (typeof howMany == "undefined") howMany = 1;
 154                     // if (howMany < 1) { return  null;                 }
 155
 156                     if (this.cursor+1 >= this.tokens.size) {
 157                         return null;
 158                 }
 159                     this.cursor++;
 160                     return this.tokens.get(this.cursor);
 161
 162                 }
 163
 164             public Gee.ArrayList<Token> nextM(uint howMany) {
 165
 166                     //if (typeof howMany == "undefined") howMany = 1;
 167                     if (howMany < 2) {
 168                                 throw new  TokenStreamError.ArgumentError("nextM called with wrong number : %d", (int)howMany);
 169                     }
 170                     var got = new Gee.ArrayList<Token>();
 171
 172                     for (var i = 1; i <= howMany; i++) {
 173                         if (this.cursor+i >= this.tokens.size) {
 174                             return null;
 175                         }
 176                         got.add(this.tokens.get(this.cursor+i));
 177                     }
 178                     this.cursor += howMany;
 179
 180                         return got;
 181                 }
 182
 183
 184
 185
 186                 // what about comments after 'function'...
 187                 // is this used ???
 188                 public Token? nextTok() {
 189                     return this.nextNonSpace();
 190                 }
 191
 192                 public Token? nextNonSpace ()
 193                 {
 194
 195                     while (true) {
 196                         tok = this.next();
 197                         if (tok == null) {
 198                             return null;
 199                         }
 200                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 201                             continue;
 202                         }
 203                         return tok;
 204                     }
 205                 }
 206
 207                 /**
 208                  *  balance
 209                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 210                  * @param start {String}  token name or data (eg. '{'
 211                  * @param stop {String} (Optional) token name or data (eg. '}'
 212                  */
 213                 public Gee.ArrayList<Token> balance (string start, string stop = "") {
 214
 215                     // accepts names or "{" etc..
 216
 217                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 218
 219                     if (stop=="") {
 220                                 stop = Lang.matching(start);
 221                         }
 222                         if (stop == null) {
 223                                 throw new JSDOC.TokenStreamError("balance called with invalid start/stop : %s",start);
 224                         }
 225
 226                     var depth = 0;
 227                     var got = new Gee.ArrayList<Token>();
 228                     var started = false;
 229                     //Seed.print("START:" + start);
 230                     //Seed.print("STOP:" + stop);
 231
 232                     while ((token = this.look())) {
 233                         if (token.is(start)) {
 234                       //      Seed.print("balance: START : " + depth + " " + token.data);
 235                             depth++;
 236                             started = true;
 237                         }
 238
 239                         if (started) {
 240                             got.push(token);
 241                         }
 242
 243                         if (token.is(stop)) {
 244                             depth--;
 245                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 246                             if (depth < 1) {
 247                                         return got;
 248                                 }
 249                         }
 250                         if (!this.next()) {
 251                                 break;
 252                         }
 253                     }
 254                     return new Gee.ArrayList<Token>();
 255                 }
 256
 257                 public Token getMatchingToken(string start, string stop) {
 258                     var depth = 0;
 259                     var cursor = this.cursor;
 260
 261                     if (!start) {
 262                         start = Lang.matching(stop);
 263                         depth = 1;
 264                     }
 265                     if (!stop) stop = Lang.matching(start);
 266
 267                     while ((token = this.tokens[cursor])) {
 268                         if (token.is(start)) {
 269                             depth++;
 270                         }
 271
 272                         if (token.is(stop) && cursor) {
 273                             depth--;
 274                             if (depth == 0) return this.tokens[cursor];
 275                         }
 276                         cursor++;
 277                     }
 278                     return false;
 279                 }
 280
 281                 public Gee.ArrayList<Token> insertAhead(Token token) {
 282                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 283                 }
 284
 285                 public Gee.ArrayList<Token> remaining() {
 286                     var ret = new Gee.ArrayList<Token>();
 287                     while (true) {
 288                         var tok = this.look(1,true);
 289                         if (tok.is("VOID")) {
 290                             return ret;
 291                         }
 292                         ret.push(this.next(1));
 293                     }
 294                 }
 295
 296                 /*
 297                 arrayToString : function(ar) {
 298                     console.log(typeof(ar));
 299                     var ret = [];
 300                     ar.forEach(function(e) {
 301                         ret.push(e.data);
 302                     })
 303                     return ret.join('');
 304                 },
 305                 dump: function(start, end)
 306                 {
 307                     start = Math.max(start || 0, 0);
 308                     end = Math.min(end || this.tokens.length, this.tokens.length);
 309                     var out='';
 310                     for (var i =start;i < end; i++) {
 311
 312                         out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
 313                     };
 314                     print(out);
 315                 }
 316                 */
 317         }
 318 }
 319