git.roojs.org Git - roojspacker/blob - roojspacker/Token.vala

   1 /**
   2  *      @class Token
   3  *
   4  *  @prop data {String} raw value of token
   5  *  @prop type {String} type of token
   6  *     TOKN  (unknown)          - name is UNKNOWN_TOKEN
   7  *
   8  *     KEYW  (keyword)          - name is upper case version of keyword
   9  *     NAME  (name/identifier)  - name is NAME
  10  *     COMM  (comment)          - name is MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM
  11  *     PUNC  (puctuation)       - name is String description of punctionan (eg LEFTPARAM)
  12  *     WHIT  (white space)      - name is SPACE,NEWLINE
  13  *     STRN  (string)           - name is DOBULE_QUOTE, SINGLE_QUOTE
  14  *     NUMB  (number)           - name is OCTAL,DECIMAL,HEC_DEC
  15  *     REGX   (reg.expression)  - name is REGX
  16  *  @prop name {String} see type details above
  17  *  @prop identifier {Identifier} identifier class if relivant
  18  *
  19  *
  20  *
  21  * old mappings:
  22  *
  23  * Script.TOKidentifier  - type == 'NAME'
  24  * Script.TOKassign  = data == '='
  25  * Script.TOKsemicolon data == '';
  26  *
  27  *
  28  *
  29 */
  30 namespace JSDOC
  31 {
  32     int Token_id = 1;
  33         public enum TokenType {
  34                 TOKN, //  (unknown)          - name is UNKNOWN_TOKEN
  35                 KEYW, //  (keyword)          - name is upper case version of keyword
  36                 NAME, //  (name/identifier)  - name is NAME
  37                 COMM, //  (comment)          - name is MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM
  38                 PUNC, //  (puctuation)       - name is String description of punctionan (eg LEFTPARAM)
  39                 WHIT, //  (white space)      - name is SPACE,NEWLINE
  40                 STRN, //  (string)           - name is DOBULE_QUOTE, SINGLE_QUOTE
  41                 NUMB, //  (number)           - name is OCTAL,DECIMAL,HEC_DEC
  42                 REGX, //   (reg.expression)  - name is REGX
  43
  44                 VOID // BAD eof
  45         }
  46
  47
  48         public enum TokenName {
  49                 UNKNOWN_TOKEN,
  50
  51                 // keywords.
  52                         BREAK,
  53                         CASE,
  54                         CATCH,
  55                         CONST,
  56                         CONTINUE,
  57                         DEFAULT,
  58                         DELETE,
  59                         DO,
  60                         ELSE,
  61                         FALSE,
  62                         FINALLY,
  63                         FOR,
  64                         FUNCTION,
  65                         IF,
  66                         IN,
  67                         INSTANCEOF,
  68                         NEW,
  69                         NULL,
  70                         RETURN,
  71                         SWITCH,
  72                         THIS,
  73                         THROW,
  74                         TRUE,
  75                         TRY,
  76                         TYPEOF,
  77                         VOID,
  78                         WHILE,
  79                         WITH,
  80                         VAR,
  81                         EVAL,
  82
  83                 NAME,
  84
  85
  86                 MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM,
  87                 // punc
  88                         SEMICOLON,
  89                         COMMA,
  90                         HOOK,
  91                         COLON,
  92                         OR,
  93                         AND,
  94                         BITWISE_OR,
  95                         BITWISE_XOR,
  96                         BITWISE_AND,
  97                         STRICT_EQ,
  98                         EQ,
  99                         ASSIGN,
 100                         STRICT_NE,
 101                         NE,
 102                         LSH,
 103                         LE,
 104                         LT,
 105                         URSH,
 106                         RSH,
 107                         GE,
 108                         GT,
 109                         INCREMENT,
 110                         DECREMENT,
 111                         PLUS,
 112                         MINUS,
 113                         MUL,
 114                         DIV,
 115                         MOD,
 116                         NOT,
 117                         BITWISE_NOT,
 118                         DOT,
 119                         LEFT_BRACE,
 120                         RIGHT_BRACE,
 121                         LEFT_CURLY,
 122                         RIGHT_CURLY,
 123                         LEFT_PAREN, // (
 124                         RIGHT_PAREN,  // )
 125
 126
 127
 128                 SPACE,NEWLINE,
 129                 DOUBLE_QUOTE, SINGLE_QUOTE,
 130                 OCTAL,DECIMAL,HEX_DEC,
 131                 REGX,
 132
 133                 START_OF_STREAM,
 134                 END_OF_STREAM,
 135
 136                 UNKNOWN // we should change void/void to void/unknown.
 137         }
 138
 139         public class TokenKeyMap : Object {
 140                 public Token key;
 141                 public Gee.ArrayList<Token> vals;
 142
 143                 public TokenKeyMap()
 144                 {
 145                         this.key = new Token("",TokenType.VOID, TokenName.VOID);
 146                         this.vals = new  Gee.ArrayList<Token>();
 147                 }
 148
 149
 150         }
 151
 152
 153     public class Token : Object {
 154
 155         public int id;
 156
 157         public string data;
 158         public TokenType type;
 159         public TokenName name;
 160         public int line;
 161         public string prefix; // white space prefix... (when outputing with WS)
 162
 163         public string outData;
 164
 165         public Identifier identifier;
 166
 167
 168
 169          // used to stuff tokens together when building a tree..
 170         public Gee.ArrayList<Gee.ArrayList<Token>> items;
 171         // for a object definition, key -> array of tokens..
 172             public Gee.HashMap<string,TokenKeyMap> props;
 173         public Gee.ArrayList<string> keyseq;
 174         // props??? what's this???
 175
 176         public Token(string data, TokenType type, TokenName name, int line = -1)
 177         {
 178             this.data = data;
 179             this.type = type;
 180             this.name = name;
 181             this.line = line;
 182             this.prefix = "";
 183             this.outData = ""; // used by packer/scopeparser
 184             this.identifier = null; // used by scope
 185             this.id = Token_id++;
 186
 187             // should we initialize when needed...?? to keep the usage down..
 188             this.items = null;
 189             this.props = null;
 190             this.keyseq = null;
 191             if (name == TokenName.LEFT_BRACE ||
 192                         name == TokenName.LEFT_CURLY ||
 193                     name == TokenName.LEFT_PAREN ) {
 194
 195                         this.items = new Gee.ArrayList<Gee.ArrayList<Token>>();
 196                         this.props = new Gee.HashMap<string,TokenKeyMap>();
 197                         this.keyseq =  new Gee.ArrayList<string>();
 198                 }
 199
 200         }
 201
 202         public string asString()
 203         {
 204             if (this.name == TokenName.LEFT_BRACE ||
 205                         this.name == TokenName.LEFT_CURLY ||
 206                     this.name == TokenName.LEFT_PAREN ) {
 207
 208                         return "line: %s, %d, id %d, type %s, IS=%d,PS=%d,KS=%d, data : %s,  name %s, , outData: %s".printf(
 209                                         DocParser.currentSourceFile == null ? "??" : DocParser.currentSourceFile,
 210                                 this.line,
 211                                 this.id,
 212                                 this.type.to_string(),
 213                                 this.name == TokenName.LEFT_BRACE ? -1 : this.items.size,
 214                                 this.name == TokenName.LEFT_BRACE ? -1 : this.props.size,
 215                                 this.name == TokenName.LEFT_BRACE ? -1 : this.keyseq.size,
 216                                 this.data,
 217                                 this.name.to_string(),
 218                                 this.outData == null ? "" : this.outData
 219                         );
 220
 221
 222                         }
 223
 224
 225
 226             return "line:%s:%d, id %d, type %s, data : %s,  name %s, , outData: %s".printf(
 227                         DocParser.currentSourceFile == null ? "??" : DocParser.currentSourceFile,
 228                     this.line,
 229                     this.id,
 230                     this.type.to_string(),
 231
 232                     this.data,
 233                     this.name.to_string(),
 234                     this.outData == null ? "" : this.outData
 235             );
 236
 237         }
 238
 239
 240         public void dump(string indent)
 241                 {
 242                 print("%s%s\n",indent, this.asString());
 243                 if (this.items != null && this.items.size > 0) {
 244
 245                                 for (var i = 0;i < this.items.size; i++) {
 246                                 print("%s --ITEMS[%d] [ \n",indent,i);
 247                                         for (var j = 0;j < this.items[i].size; j++) {
 248                                                 this.items[i][j].dump(indent + "  ");
 249                                         }
 250                                 }
 251                         }
 252                         if (this.props != null && this.props.size > 0) {
 253                                 var m = this.props.map_iterator();
 254                                 while(m.next()) {
 255                                 print("%s --KEY %s ::  \n",indent,m.get_key());
 256                                 var vals = m.get_value().vals;
 257                                         for (var i = 0;i < vals.size; i++) {
 258
 259                                                 vals[i].dump(indent + "  ");
 260                                         }
 261                                 }
 262
 263
 264                         }
 265
 266                 }
 267
 268
 269         public string toRaw(int lvl = 0)
 270         {
 271
 272
 273             var ret =  this.data ;
 274
 275             foreach(var ai in this.items ) {
 276                 // supposed to iterate properties???
 277                 string str = "";
 278                 //foreach( var it in ai) {
 279                  //   str += it.toRaw(lvl+1);
 280                // }
 281                 ret += str;
 282             }
 283
 284             /* -- what is a prop..
 285             if (this.props) {
 286                 for (var i in this.props) {
 287                     ret += this.props[i].key.toRaw(lvl+1) + ' : ';
 288                     this.props[i].val.forEach( function(e) {
 289                         ret+=e.toRaw(lvl+1);
 290                     })
 291
 292                 }
 293             }
 294
 295             */
 296
 297             return this.prefix +   ret;
 298
 299         }
 300         /*
 301         toJS : function() {
 302
 303             try {
 304                 var _tmp = '';
 305                 eval( "_tmp = " + this.data);
 306                 return _tmp;
 307             } catch( e) {
 308                 return "ERROR unparsable" + this.data;
 309             }
 310         },
 311         */
 312
 313
 314         public bool isName(TokenName what) {
 315             return this.name == what;
 316         }
 317         public bool isType(TokenType what) {
 318             return  this.type == what;
 319         }
 320
 321     }
 322 }
 323