git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public class TokenStream : Object
  16         {
  17
  18                 Gee.ArrayList<Token> tokens;
  19                 int cursor; // where are we in the stream.
  20
  21
  22                 public TokenStream(Gee.ArrayList<Token> tokens) {
  23
  24                         this.tokens = tokens;
  25
  26                         this.rewind();
  27                 }
  28
  29
  30
  31                 public void rewind() {
  32                     this.cursor = -1;
  33                 }
  34
  35                 /**
  36                     @type JSDOC.Token
  37                 */
  38                 public Token? look (int n, bool considerWhitespace)
  39                 {
  40
  41
  42                     if (considerWhitespace == true) {
  43
  44                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  45                             return new Token("", "VOID", "START_OF_STREAM");
  46                         }
  47                         return this.tokens.get(this.cursor+n);
  48                     }
  49
  50
  51                 var count = 0;
  52                 var i = this.cursor;
  53
  54                 while (true) {
  55                     if (i < 0) {
  56                                 return new Token("", "VOID", "START_OF_STREAM");
  57                         }
  58                     if (i > this.tokens.size) {
  59                                 return new Token("", "VOID", "END_OF_STREAM");
  60                         }
  61
  62                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  63                                 i += (n < 0) ? -1 : 1;
  64                         continue;
  65                     }
  66
  67                     if (count == n) {
  68                         return this.tokens.get(i);
  69                     }
  70                     count++;
  71                     i += (n < 0) ? -1 : 1;
  72                 }
  73
  74                 return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  75
  76                 }
  77
  78                 public int lookFor  (string data)
  79                 {
  80                     // non tree version..
  81                     var i = this.cursor < 0 ? 0 : this.cursor ;
  82
  83                     while (true) {
  84                         if (i >= this.tokens.size) {
  85                                 return -1;
  86                         }
  87                         if (this.tokens.get(i).data == data) {
  88                             return i;
  89                         }
  90                         i++;
  91
  92                     }
  93                     // should not get here!
  94                     return -1;
  95
  96                 }
  97
  98
  99                 /**
 100                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 101                  * ?? used any more?
 102                  */
 103                 public Token lookTok (int n) {
 104
 105
 106
 107                     var count = 0;
 108                     var i = this.cursor;
 109
 110                     while (true) {
 111                        // print(i);
 112                         if (i < 0) {
 113                             if (n > -1) {
 114                                 i = 0;
 115                                 count++;
 116                                 continue;
 117
 118                             }
 119                             return  new Token("", "VOID", "END_OF_STREAM");
 120                         }
 121                         if (i > this.tokens.size) {
 122                                 return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124
 125                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 126                             i += (n < 0) ? -1 : 1;
 127                             continue;
 128                         }
 129
 130                         if (count == n) {
 131                             return this.tokens.get(i);
 132                         }
 133                         count++;
 134                         i += (n < 0) ? -1 : 1;
 135                     }
 136                 // should never get here..
 137                     return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 138
 139                 }
 140
 141                 /**
 142                  *  @return {Token|null}
 143                  * next token (with white space)
 144                  */
 145
 146
 147                 public Token? next() {
 148
 149
 150                     //if (typeof howMany == "undefined") howMany = 1;
 151                     // if (howMany < 1) { return  null;                 }
 152
 153                     if (this.cursor+1 >= this.tokens.size) {
 154                         return null;
 155                 }
 156                     this.cursor++;
 157                     return this.tokens.get(this.cursor);
 158
 159                 }
 160
 161             public Gee.ArrayList<Token> nextM(uint howMany) {
 162
 163                     //if (typeof howMany == "undefined") howMany = 1;
 164                     if (howMany < 2) {
 165                                 throw new JSDOC.TokenStreamError("nextM called with wrong number : %d",howMany);
 166                     }
 167                     var got = new Gee.ArrayList<Token>();
 168
 169                     for (var i = 1; i <= howMany; i++) {
 170                         if (this.cursor+i >= this.tokens.size) {
 171                             return null;
 172                         }
 173                         got.add(this.tokens.get(this.cursor+i));
 174                     }
 175                     this.cursor += howMany;
 176
 177                         return got;
 178                 }
 179
 180
 181
 182
 183                 // what about comments after 'function'...
 184                 // is this used ???
 185                 public Token? nextTok() {
 186                     return this.nextNonSpace();
 187                 }
 188
 189                 public Token? nextNonSpace ()
 190                 {
 191
 192                     while (true) {
 193                         tok = this.next();
 194                         if (tok == null) {
 195                             return null;
 196                         }
 197                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 198                             continue;
 199                         }
 200                         return tok;
 201                     }
 202                 }
 203
 204                 /**
 205                  *  balance
 206                  * -- returns all the tokens betweeen and including stop token eg.. from {... to  }
 207                  * @param start {String}  token name or data (eg. '{'
 208                  * @param stop {String} (Optional) token name or data (eg. '}'
 209                  */
 210                 public Gee.ArrayList<Token> balance (string start, string stop = "") {
 211
 212                     // accepts names or "{" etc..
 213
 214                     start = Lang.punc(start) == null ? start : Lang.punc(start);
 215
 216                     if (stop=="") {
 217                                 stop = Lang.matching(start);
 218                         }
 219                         if (stop == null) {
 220                                 throw new JSDOC.TokenStreamError("balance called with invalid start/stop : %s",start);
 221                         }
 222
 223                     var depth = 0;
 224                     var got = new Gee.ArrayList<Token>();
 225                     var started = false;
 226                     //Seed.print("START:" + start);
 227                     //Seed.print("STOP:" + stop);
 228
 229                     while ((token = this.look())) {
 230                         if (token.is(start)) {
 231                       //      Seed.print("balance: START : " + depth + " " + token.data);
 232                             depth++;
 233                             started = true;
 234                         }
 235
 236                         if (started) {
 237                             got.push(token);
 238                         }
 239
 240                         if (token.is(stop)) {
 241                             depth--;
 242                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 243                             if (depth < 1) {
 244                                         return got;
 245                                 }
 246                         }
 247                         if (!this.next()) {
 248                                 break;
 249                         }
 250                     }
 251                     return new Gee.ArrayList<Token>();
 252                 }
 253
 254                 public Token getMatchingToken(string start, string stop) {
 255                     var depth = 0;
 256                     var cursor = this.cursor;
 257
 258                     if (!start) {
 259                         start = Lang.matching(stop);
 260                         depth = 1;
 261                     }
 262                     if (!stop) stop = Lang.matching(start);
 263
 264                     while ((token = this.tokens[cursor])) {
 265                         if (token.is(start)) {
 266                             depth++;
 267                         }
 268
 269                         if (token.is(stop) && cursor) {
 270                             depth--;
 271                             if (depth == 0) return this.tokens[cursor];
 272                         }
 273                         cursor++;
 274                     }
 275                     return false;
 276                 }
 277
 278                 public Gee.ArrayList<Token> insertAhead(Token token) {
 279                     this.tokens.splice(this.cursor+1, 0, token); // fixme...
 280                 }
 281
 282                 public Gee.ArrayList<Token> remaining() {
 283                     var ret = new Gee.ArrayList<Token>();
 284                     while (true) {
 285                         var tok = this.look(1,true);
 286                         if (tok.is("VOID")) {
 287                             return ret;
 288                         }
 289                         ret.push(this.next(1));
 290                     }
 291                 }
 292
 293                 /*
 294                 arrayToString : function(ar) {
 295                     console.log(typeof(ar));
 296                     var ret = [];
 297                     ar.forEach(function(e) {
 298                         ret.push(e.data);
 299                     })
 300                     return ret.join('');
 301                 },
 302                 dump: function(start, end)
 303                 {
 304                     start = Math.max(start || 0, 0);
 305                     end = Math.min(end || this.tokens.length, this.tokens.length);
 306                     var out='';
 307                     for (var i =start;i < end; i++) {
 308
 309                         out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
 310                     };
 311                     print(out);
 312                 }
 313                 */
 314         }
 315 }
 316