git.roojs.org Git - gnome.introspection-doc-generator/blob - JSDOC/TokenReader.js

   1 //<script type="text/javascript">
   2
   3
   4 XObject = imports.XObject.XObject;
   5 console = imports.console.console;
   6
   7
   8 Token   = imports.Token.Token;
   9 Lang    = imports.Lang.Lang;
  10
  11 /**
  12         @class Search a {@link JSDOC.TextStream} for language tokens.
  13 */
  14 TokenReader = XObject.define(
  15     function(o) {
  16
  17         this.keepDocs = true;
  18         this.keepWhite = false;
  19         this.keepComments = false;
  20         this.sepIdents = false; // seperate '.' in identifiers..
  21         XObject.extend(this, o || {});
  22
  23     },
  24     Object,
  25     {
  26         collapseWhite : false, // only reduces white space...
  27
  28         /**
  29          * tokenize a stream
  30          * @return {Array} of tokens
  31          *
  32          * ts = new TextStream(File.read(str));
  33          * tr = TokenReader({ keepComments : true, keepWhite : true });
  34          * tr.tokenize(ts)
  35          *
  36          */
  37
  38
  39
  40         tokenize : function(/**JSDOC.TextStream*/stream) {
  41             this.line =1;
  42             var tokens = [];
  43             /**@ignore*/ tokens.last    = function() { return tokens[tokens.length-1]; }
  44             /**@ignore*/ tokens.lastSym = function() {
  45                 for (var i = tokens.length-1; i >= 0; i--) {
  46                     if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i];
  47                 }
  48             }
  49
  50             while (!stream.look().eof) {
  51                 if (this.read_mlcomment(stream, tokens)) continue;
  52                 if (this.read_slcomment(stream, tokens)) continue;
  53                 if (this.read_dbquote(stream, tokens))   continue;
  54                 if (this.read_snquote(stream, tokens))   continue;
  55                 if (this.read_regx(stream, tokens))      continue;
  56                 if (this.read_numb(stream, tokens))      continue;
  57                 if (this.read_punc(stream, tokens))      continue;
  58                 if (this.read_newline(stream, tokens))   continue;
  59                 if (this.read_space(stream, tokens))     continue;
  60                 if (this.read_word(stream, tokens))      continue;
  61
  62                 // if execution reaches here then an error has happened
  63                 tokens.push(new Token(stream.next(), "TOKN", "UNKNOWN_TOKEN", this.line));
  64             }
  65
  66
  67
  68             return tokens;
  69         },
  70
  71         /**
  72             @returns {Boolean} Was the token found?
  73          */
  74         read_word : function(/**JSDOC.TokenStream*/stream, tokens) {
  75             var found = "";
  76             while (!stream.look().eof && Lang.isWordChar(stream.look())) {
  77                 found += stream.next();
  78             }
  79
  80             if (found === "") {
  81                 return false;
  82             }
  83             else {
  84                 var name;
  85                 if ((name = Lang.keyword(found))) {
  86                     tokens.push(new Token(found, "KEYW", name, this.line));
  87                     return true;
  88                 }
  89                 if (!this.sepIdents || found.indexOf('.') < 0 ) {
  90                     tokens.push(new Token(found, "NAME", "NAME", this.line));
  91                     return true;
  92                 }
  93                 var n = found.split('.');
  94                 var p = false;
  95                 var _this = this;
  96                 n.forEach(function(nm) {
  97                     if (p) {
  98                         tokens.push(new Token('.', "PUNC", "DOT", _this.line));
  99                     }
 100                     p=true;
 101                     tokens.push(new Token(nm, "NAME", "NAME", _this.line));
 102                 });
 103                 return true;
 104
 105             }
 106         },
 107
 108         /**
 109             @returns {Boolean} Was the token found?
 110          */
 111         read_punc : function(/**JSDOC.TokenStream*/stream, tokens) {
 112             var found = "";
 113             var name;
 114             while (!stream.look().eof && Lang.punc(found+stream.look())) {
 115                 found += stream.next();
 116             }
 117
 118             if (found === "") {
 119                 return false;
 120             }
 121             else {
 122                 tokens.push(new Token(found, "PUNC", Lang.punc(found), this.line));
 123                 return true;
 124             }
 125         },
 126
 127         /**
 128             @returns {Boolean} Was the token found?
 129          */
 130         read_space : function(/**JSDOC.TokenStream*/stream, tokens) {
 131             var found = "";
 132
 133             while (!stream.look().eof && Lang.isSpace(stream.look())) {
 134                 if (Lang.isNewline()) break;
 135                 found += stream.next();
 136             }
 137
 138             if (found === "") {
 139                 return false;
 140             }
 141             print("WHITE = " + JSON.stringify(found));
 142             if (this.collapseWhite) found = " ";
 143             if (this.keepWhite) tokens.push(new Token(found, "WHIT", "SPACE", this.line));
 144             return true;
 145
 146         },
 147
 148         /**
 149             @returns {Boolean} Was the token found?
 150          */
 151         read_newline : function(/**JSDOC.TokenStream*/stream, tokens) {
 152             var found = "";
 153
 154             while (!stream.look().eof && Lang.isNewline(stream.look())) {
 155                 this.line++;
 156                 found += stream.next();
 157             }
 158
 159             if (found === "") {
 160                 return false;
 161             }
 162             //this.line++;
 163             if (this.collapseWhite) {
 164                 found = "\n";
 165             }
 166             if (this.keepWhite) {
 167                 var last = tokens.pop();
 168                 if (last.name != "WHIT") {
 169                     tokens.push(last);
 170                 }
 171
 172                 tokens.push(new Token(found, "WHIT", "NEWLINE", this.line));
 173             }
 174             return true;
 175         },
 176
 177         /**
 178             @returns {Boolean} Was the token found?
 179          */
 180         read_mlcomment : function(/**JSDOC.TokenStream*/stream, tokens) {
 181             if (stream.look() == "/" && stream.look(1) == "*") {
 182                 var found = stream.next(2);
 183                 var c = '';
 184                 while (!stream.look().eof && !(stream.look(-1) == "/" && stream.look(-2) == "*")) {
 185                     c = stream.next();
 186                     if (c == "\n") this.line++;
 187                     found += c;
 188                 }
 189
 190                 // to start doclet we allow /** or /*** but not /**/ or /****
 191                 if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new Token(found, "COMM", "JSDOC", this.line));
 192                 else if (this.keepComments) tokens.push(new Token(found, "COMM", "MULTI_LINE_COMM", this.line));
 193                 return true;
 194             }
 195             return false;
 196         },
 197
 198         /**
 199             @returns {Boolean} Was the token found?
 200          */
 201         read_slcomment : function(/**JSDOC.TokenStream*/stream, tokens) {
 202             var found;
 203             if (
 204                 (stream.look() == "/" && stream.look(1) == "/" && (found=stream.next(2)))
 205                 ||
 206                 (stream.look() == "<" && stream.look(1) == "!" && stream.look(2) == "-" && stream.look(3) == "-" && (found=stream.next(4)))
 207             ) {
 208
 209                 while (!stream.look().eof && !Lang.isNewline(stream.look())) {
 210                     found += stream.next();
 211                 }
 212
 213                 if (this.keepComments) {
 214                     tokens.push(new Token(found, "COMM", "SINGLE_LINE_COMM", this.line));
 215                 }
 216                 this.line++;
 217                 return true;
 218             }
 219             return false;
 220         },
 221
 222         /**
 223             @returns {Boolean} Was the token found?
 224          */
 225         read_dbquote : function(/**JSDOC.TokenStream*/stream, tokens) {
 226             if (stream.look() == "\"") {
 227                 // find terminator
 228                 var string = stream.next();
 229
 230                 while (!stream.look().eof) {
 231                     if (stream.look() == "\\") {
 232                         if (Lang.isNewline(stream.look(1))) {
 233                             do {
 234                                 stream.next();
 235                             } while (!stream.look().eof && Lang.isNewline(stream.look()));
 236                             string += "\\\n";
 237                         }
 238                         else {
 239                             string += stream.next(2);
 240                         }
 241                     }
 242                     else if (stream.look() == "\"") {
 243                         string += stream.next();
 244                         tokens.push(new Token(string, "STRN", "DOUBLE_QUOTE", this.line));
 245                         return true;
 246                     }
 247                     else {
 248                         string += stream.next();
 249                     }
 250                 }
 251             }
 252             return false; // error! unterminated string
 253         },
 254
 255         /**
 256             @returns {Boolean} Was the token found?
 257          */
 258         read_snquote : function(/**JSDOC.TokenStream*/stream, tokens) {
 259             if (stream.look() == "'") {
 260                 // find terminator
 261                 var string = stream.next();
 262
 263                 while (!stream.look().eof) {
 264                     if (stream.look() == "\\") { // escape sequence
 265                         string += stream.next(2);
 266                     }
 267                     else if (stream.look() == "'") {
 268                         string += stream.next();
 269                         tokens.push(new Token(string, "STRN", "SINGLE_QUOTE", this.line));
 270                         return true;
 271                     }
 272                     else {
 273                         string += stream.next();
 274                     }
 275                 }
 276             }
 277             return false; // error! unterminated string
 278         },
 279
 280         /**
 281             @returns {Boolean} Was the token found?
 282          */
 283         read_numb : function(/**JSDOC.TokenStream*/stream, tokens) {
 284             if (stream.look() === "0" && stream.look(1) == "x") {
 285                 return this.read_hex(stream, tokens);
 286             }
 287
 288             var found = "";
 289
 290             while (!stream.look().eof && Lang.isNumber(found+stream.look())){
 291                 found += stream.next();
 292             }
 293
 294             if (found === "") {
 295                 return false;
 296             }
 297             else {
 298                 if (/^0[0-7]/.test(found)) tokens.push(new Token(found, "NUMB", "OCTAL", this.line));
 299                 else tokens.push(new Token(found, "NUMB", "DECIMAL", this.line));
 300                 return true;
 301             }
 302         },
 303         /*t:
 304             requires("../lib/JSDOC/TextStream.js");
 305             requires("../lib/JSDOC/Token.js");
 306             requires("../lib/JSDOC/Lang.js");
 307
 308             plan(3, "testing read_numb");
 309
 310             //// setup
 311             var src = "function foo(num){while (num+8.0 >= 0x20 && num < 0777){}}";
 312             var tr = new TokenReader();
 313             var tokens = tr.tokenize(new TextStream(src));
 314
 315             var hexToken, octToken, decToken;
 316             for (var i = 0; i < tokens.length; i++) {
 317                 if (tokens[i].name == "HEX_DEC") hexToken = tokens[i];
 318                 if (tokens[i].name == "OCTAL") octToken = tokens[i];
 319                 if (tokens[i].name == "DECIMAL") decToken = tokens[i];
 320             }
 321             ////
 322
 323             is(decToken.data, "8.0", "decimal number is found in source.");
 324             is(hexToken.data, "0x20", "hexdec number is found in source (issue #99).");
 325             is(octToken.data, "0777", "octal number is found in source.");
 326         */
 327
 328         /**
 329             @returns {Boolean} Was the token found?
 330          */
 331         read_hex : function(/**JSDOC.TokenStream*/stream, tokens) {
 332             var found = stream.next(2);
 333
 334             while (!stream.look().eof) {
 335                 if (Lang.isHexDec(found) && !Lang.isHexDec(found+stream.look())) { // done
 336                     tokens.push(new Token(found, "NUMB", "HEX_DEC", this.line));
 337                     return true;
 338                 }
 339                 else {
 340                     found += stream.next();
 341                 }
 342             }
 343             return false;
 344         },
 345
 346         /**
 347             @returns {Boolean} Was the token found?
 348          */
 349         read_regx : function(/**JSDOC.TokenStream*/stream, tokens) {
 350             var last;
 351             if (
 352                 stream.look() == "/"
 353                 &&
 354                 (
 355
 356                     (
 357                         !(last = tokens.lastSym()) // there is no last, the regex is the first symbol
 358                         ||
 359                         (
 360                                !last.is("NUMB")
 361                             && !last.is("NAME")
 362                             && !last.is("RIGHT_PAREN")
 363                             && !last.is("RIGHT_BRACKET")
 364                         )
 365                     )
 366                 )
 367             ) {
 368                 var regex = stream.next();
 369
 370                 while (!stream.look().eof) {
 371                     if (stream.look() == "\\") { // escape sequence
 372                         regex += stream.next(2);
 373                     }
 374                     else if (stream.look() == "/") {
 375                         regex += stream.next();
 376
 377                         while (/[gmi]/.test(stream.look())) {
 378                             regex += stream.next();
 379                         }
 380
 381                         tokens.push(new Token(regex, "REGX", "REGX", this.line));
 382                         return true;
 383                     }
 384                     else {
 385                         regex += stream.next();
 386                     }
 387                 }
 388                 // error: unterminated regex
 389             }
 390             return false;
 391         }
 392 });