git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public class TokenStream : Object
  16         {
  17
  18                 Gee.ArrayList<Token> tokens;
  19                 int cursor; // where are we in the stream.
  20
  21
  22                 public TokenStream(Gee.ArrayList<Token> tokens) {
  23
  24                         this.tokens = tokens;
  25
  26                         this.rewind();
  27                 }
  28
  29
  30
  31                 public void rewind() {
  32                     this.cursor = -1;
  33                 }
  34
  35                 /**
  36                     @type JSDOC.Token
  37                 */
  38                 public Token? look (int n, bool considerWhitespace)
  39                 {
  40
  41
  42                     if (considerWhitespace == true) {
  43
  44                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  45                             return new Token("", "VOID", "START_OF_STREAM");
  46                         }
  47                         return this.tokens.get(this.cursor+n);
  48                     }
  49
  50
  51                 var count = 0;
  52                 var i = this.cursor;
  53
  54                 while (true) {
  55                     if (i < 0) {
  56                                 return new Token("", "VOID", "START_OF_STREAM");
  57                         }
  58                     if (i > this.tokens.size) {
  59                                 return new Token("", "VOID", "END_OF_STREAM");
  60                         }
  61
  62                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  63                                 i += (n < 0) ? -1 : 1;
  64                         continue;
  65                     }
  66
  67                     if (count == n) {
  68                         return this.tokens.get(i);
  69                     }
  70                     count++;
  71                     i += (n < 0) ? -1 : 1;
  72                 }
  73
  74                 return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  75
  76                 }
  77
  78                 public int lookFor  (string data)
  79                 {
  80                     // non tree version..
  81                     var i = this.cursor < 0 ? 0 : this.cursor ;
  82
  83                     while (true) {
  84                         if (i >= this.tokens.size) {
  85                                 return -1;
  86                         }
  87                         if (this.tokens.get(i).data == data) {
  88                             return i;
  89                         }
  90                         i++;
  91
  92                     }
  93                     // should not get here!
  94                     return -1;
  95
  96                 }
  97
  98
  99                 /**
 100                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 101                  * ?? used any more?
 102                  */
 103                 public Token lookTok (int n) {
 104
 105
 106
 107                     var count = 0;
 108                     var i = this.cursor;
 109
 110                     while (true) {
 111                        // print(i);
 112                         if (i < 0) {
 113                             if (n > -1) {
 114                                 i = 0;
 115                                 count++;
 116                                 continue;
 117
 118                             }
 119                             return  new Token("", "VOID", "END_OF_STREAM");
 120                         }
 121                         if (i > this.tokens.length) {
 122                                 return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124
 125                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 126                             i += (n < 0) ? -1 : 1;
 127                             continue;
 128                         }
 129
 130                         if (count == n) {
 131                             return this.tokens.get(i);
 132                         }
 133                         count++;
 134                         i += (n < 0) ? -1 : 1;
 135                     }
 136                 // should never get here..
 137                     return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 138
 139                 }
 140
 141                 /**
 142                  *  @return {Token|null}
 143                  * next token (with white space)
 144                  */
 145
 146
 147                 public Token? next() {
 148
 149
 150                     //if (typeof howMany == "undefined") howMany = 1;
 151                     // if (howMany < 1) { return  null;                 }
 152
 153                     if (this.cursor+1 >= this.tokens.size) {
 154                         return null;
 155                 }
 156                     this.cursor++;
 157                     return this.tokens.get(this.cursor);
 158
 159                 }
 160
 161             public Gee.ArrayList<Token> nextM(uint howMany) {
 162
 163                     //if (typeof howMany == "undefined") howMany = 1;
 164                     if (howMany < 2) {
 165                                 throw new JSDOC.TokenStreamError("nextM called with wrong number : %d",howMany);
 166                     }
 167                     var got = new Gee.ArrayList<Token>();
 168
 169                     for (var i = 1; i <= howMany; i++) {
 170                         if (this.cursor+i >= this.tokens.size) {
 171                             return null;
 172                         }
 173                         got.add(this.tokens.get(this.cursor+i));
 174                     }
 175                     this.cursor += howMany;
 176
 177                         return got;
 178                 }
 179
 180
 181
 182
 183                 // what about comments after 'function'...
 184                 // is this used ???
 185                 public Token? nextTok() {
 186                     return this.nextNonSpace();
 187                 }
 188
 189                 public Token? nextNonSpace ()
 190                 {
 191
 192                     while (true) {
 193                         tok = this.next();
 194                         if (tok == null) {
 195                             return null;
 196                         }
 197                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 198                             continue;
 199                         }
 200                         return tok;
 201                     }
 202                 }
 203
 204                 /**
 205                  *    @type JSDOC.Token[]
 206                  * @param start {String}  token name or data (eg. '{'
 207                  * @param stop {String} (Optional) token name or data (eg. '}'
 208                  */
 209                 public void balance (string start, string stop = "") {
 210
 211                     // accepts names or "{" etc..
 212
 213                     start = Lang.punc(start)) == null ? start : Lang.punc(start);
 214
 215                     if (stop=="") {
 216                                 stop = Lang.matching(start);
 217                         }
 218
 219                     var depth = 0;
 220                     var got = [];
 221                     var started = false;
 222                     //Seed.print("START:" + start);
 223                     //Seed.print("STOP:" + stop);
 224                     while ((token = this.look())) {
 225                         if (token.is(start)) {
 226                       //      Seed.print("balance: START : " + depth + " " + token.data);
 227                             depth++;
 228                             started = true;
 229                         }
 230
 231                         if (started) {
 232                             got.push(token);
 233                         }
 234
 235                         if (token.is(stop)) {
 236                             depth--;
 237                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 238                             if (depth < 1) return got;
 239                         }
 240                         if (!this.next()) break;
 241                     }
 242                     return false;
 243                 },
 244
 245                 getMatchingToken : function(/**String*/start, /**String*/stop) {
 246                     var depth = 0;
 247                     var cursor = this.cursor;
 248
 249                     if (!start) {
 250                         start = Lang.matching(stop);
 251                         depth = 1;
 252                     }
 253                     if (!stop) stop = Lang.matching(start);
 254
 255                     while ((token = this.tokens[cursor])) {
 256                         if (token.is(start)) {
 257                             depth++;
 258                         }
 259
 260                         if (token.is(stop) && cursor) {
 261                             depth--;
 262                             if (depth == 0) return this.tokens[cursor];
 263                         }
 264                         cursor++;
 265                     }
 266                     return false;
 267                 },
 268
 269                 insertAhead : function(/**JSDOC.Token*/token) {
 270                     this.tokens.splice(this.cursor+1, 0, token);
 271                 },
 272
 273                 remaining : function() {
 274                     var ret = [];
 275                     while (true) {
 276                         var tok = this.look(1,true);
 277                         if (!tok || !tok.is || tok.is('VOID')) {
 278                             return ret;
 279                         }
 280                         ret.push(this.next(1));
 281                     }
 282                 },
 283
 284
 285                 arrayToString : function(ar) {
 286                     console.log(typeof(ar));
 287                     var ret = [];
 288                     ar.forEach(function(e) {
 289                         ret.push(e.data);
 290                     })
 291                     return ret.join('');
 292                 },
 293                 dump: function(start, end)
 294                 {
 295                     start = Math.max(start || 0, 0);
 296                     end = Math.min(end || this.tokens.length, this.tokens.length);
 297                     var out='';
 298                     for (var i =start;i < end; i++) {
 299
 300                         out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
 301                     };
 302                     print(out);
 303                 }
 304 });
 305