git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenStream.vala

   1
   2 /**
   3  * @class TokenStream
   4  *
   5  * BC notes:
   6  *
   7  * nextT => nextTok
   8  * lookT => lookTok
   9  *
  10  */
  11
  12
  13 namespace JSDOC {
  14
  15         public class TokenStream : Object
  16         {
  17
  18                 Gee.ArrayList<Token> tokens;
  19                 int cursor; // where are we in the stream.
  20
  21
  22                 public TokenStream(Gee.ArrayList<Token> tokens) {
  23
  24                         this.tokens = tokens;
  25
  26                         this.rewind();
  27                 }
  28
  29
  30
  31                 public void rewind() {
  32                     this.cursor = -1;
  33                 }
  34
  35                 /**
  36                     @type JSDOC.Token
  37                 */
  38                 public Token? look (int n, bool considerWhitespace)
  39                 {
  40
  41
  42                     if (considerWhitespace == true) {
  43
  44                         if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
  45                             return new Token("", "VOID", "START_OF_STREAM");
  46                         }
  47                         return this.tokens.get(this.cursor+n);
  48                     }
  49
  50
  51                 var count = 0;
  52                 var i = this.cursor;
  53
  54                 while (true) {
  55                     if (i < 0) {
  56                                 return new Token("", "VOID", "START_OF_STREAM");
  57                         }
  58                     if (i > this.tokens.size) {
  59                                 return new Token("", "VOID", "END_OF_STREAM");
  60                         }
  61
  62                     if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
  63                                 i += (n < 0) ? -1 : 1;
  64                         continue;
  65                     }
  66
  67                     if (count == n) {
  68                         return this.tokens.get(i);
  69                     }
  70                     count++;
  71                     i += (n < 0) ? -1 : 1;
  72                 }
  73
  74                 return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
  75
  76                 }
  77
  78                 public int lookFor  (string data)
  79                 {
  80                     // non tree version..
  81                     var i = this.cursor < 0 ? 0 : this.cursor ;
  82
  83                     while (true) {
  84                         if (i >= this.tokens.size) {
  85                                 return -1;
  86                         }
  87                         if (this.tokens.get(i).data == data) {
  88                             return i;
  89                         }
  90                         i++;
  91
  92                     }
  93                     // should not get here!
  94                     return -1;
  95
  96                 }
  97
  98
  99                 /**
 100                  * look ahead (or back) x number of tokens (which are not comment or whitespace)
 101                  * ?? used any more?
 102                  */
 103                 public Token lookTok (int n) {
 104
 105
 106
 107                     var count = 0;
 108                     var i = this.cursor;
 109
 110                     while (true) {
 111                        // print(i);
 112                         if (i < 0) {
 113                             if (n > -1) {
 114                                 i = 0;
 115                                 count++;
 116                                 continue;
 117
 118                             }
 119                             return  new Token("", "VOID", "END_OF_STREAM");
 120                         }
 121                         if (i > this.tokens.length) {
 122                                 return  new Token("", "VOID", "END_OF_STREAM");
 123                         }
 124
 125                         if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
 126                             i += (n < 0) ? -1 : 1;
 127                             continue;
 128                         }
 129
 130                         if (count == n) {
 131                             return this.tokens.get(i);
 132                         }
 133                         count++;
 134                         i += (n < 0) ? -1 : 1;
 135                     }
 136                 // should never get here..
 137                     return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
 138
 139                 }
 140
 141                 /**
 142                  *  @return {Token|null}
 143                  * next token (with white space)
 144                  */
 145
 146
 147                 public Token? next() {
 148
 149
 150                     //if (typeof howMany == "undefined") howMany = 1;
 151                     // if (howMany < 1) { return  null;                 }
 152
 153                     if (this.cursor+1 >= this.tokens.size) {
 154                         return null;
 155                 }
 156                     this.cursor++;
 157                     return this.tokens.get(this.cursor);
 158
 159                 }
 160
 161             public Gee.ArrayList<Token> nextM(uint howMany) {
 162
 163                     //if (typeof howMany == "undefined") howMany = 1;
 164                     if (howMany < 2) {
 165                                 throw new JSDOC.TokenStreamError("nextM called with wrong number : %d",howMany);
 166                     }
 167                     var got = new Gee.ArrayList<Token>();
 168
 169                     for (var i = 1; i <= howMany; i++) {
 170                         if (this.cursor+i >= this.tokens.size) {
 171                             return null;
 172                         }
 173                         got.add(this.tokens.get(this.cursor+i));
 174                     }
 175                     this.cursor += howMany;
 176
 177                         return got;
 178                 }
 179
 180
 181
 182
 183                 // what about comments after 'function'...
 184                 // is this used ???
 185                 public Token? nextTok() {
 186                     return this.nextNonSpace();
 187                 }
 188
 189                 public Token? nextNonSpace ()
 190                 {
 191
 192                     while (true) {
 193                         tok = this.next();
 194                         if (tok == null) {
 195                             return null;
 196                         }
 197                         if (tok.is("WHIT") ||  tok.is("COMM")) {
 198                             continue;
 199                         }
 200                         return tok;
 201                     }
 202                 }
 203
 204                 /**
 205                  *    @type JSDOC.Token[]
 206                  * @param start {String}  token name or data (eg. '{'
 207                  * @param stop {String} (Optional) token name or data (eg. '}'
 208                  */
 209                 public void balance (string start, string stop = "") {
 210
 211                     // accepts names or "{" etc..
 212
 213                     start = Lang.punc(start)) == null ? start : Lang.punc(start);
 214
 215                     if (!stop) stop = Lang.matching(start);
 216
 217                     var depth = 0;
 218                     var got = [];
 219                     var started = false;
 220                     //Seed.print("START:" + start);
 221                     //Seed.print("STOP:" + stop);
 222                     while ((token = this.look())) {
 223                         if (token.is(start)) {
 224                       //      Seed.print("balance: START : " + depth + " " + token.data);
 225                             depth++;
 226                             started = true;
 227                         }
 228
 229                         if (started) {
 230                             got.push(token);
 231                         }
 232
 233                         if (token.is(stop)) {
 234                             depth--;
 235                         //    Seed.print("balance: STOP: "  + depth + " " + token.data);
 236                             if (depth < 1) return got;
 237                         }
 238                         if (!this.next()) break;
 239                     }
 240                     return false;
 241                 },
 242
 243                 getMatchingToken : function(/**String*/start, /**String*/stop) {
 244                     var depth = 0;
 245                     var cursor = this.cursor;
 246
 247                     if (!start) {
 248                         start = Lang.matching(stop);
 249                         depth = 1;
 250                     }
 251                     if (!stop) stop = Lang.matching(start);
 252
 253                     while ((token = this.tokens[cursor])) {
 254                         if (token.is(start)) {
 255                             depth++;
 256                         }
 257
 258                         if (token.is(stop) && cursor) {
 259                             depth--;
 260                             if (depth == 0) return this.tokens[cursor];
 261                         }
 262                         cursor++;
 263                     }
 264                     return false;
 265                 },
 266
 267                 insertAhead : function(/**JSDOC.Token*/token) {
 268                     this.tokens.splice(this.cursor+1, 0, token);
 269                 },
 270
 271                 remaining : function() {
 272                     var ret = [];
 273                     while (true) {
 274                         var tok = this.look(1,true);
 275                         if (!tok || !tok.is || tok.is('VOID')) {
 276                             return ret;
 277                         }
 278                         ret.push(this.next(1));
 279                     }
 280                 },
 281
 282
 283                 arrayToString : function(ar) {
 284                     console.log(typeof(ar));
 285                     var ret = [];
 286                     ar.forEach(function(e) {
 287                         ret.push(e.data);
 288                     })
 289                     return ret.join('');
 290                 },
 291                 dump: function(start, end)
 292                 {
 293                     start = Math.max(start || 0, 0);
 294                     end = Math.min(end || this.tokens.length, this.tokens.length);
 295                     var out='';
 296                     for (var i =start;i < end; i++) {
 297
 298                         out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
 299                     };
 300                     print(out);
 301                 }
 302 });
 303