4 * @prop data {String} raw value of token
5 * @prop type {String} type of token
6 * TOKN (unknown) - name is UNKNOWN_TOKEN
8 * KEYW (keyword) - name is upper case version of keyword
9 * NAME (name/identifier) - name is NAME
10 * COMM (comment) - name is MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM
11 * PUNC (puctuation) - name is String description of punctionan (eg LEFTPARAM)
12 * WHIT (white space) - name is SPACE,NEWLINE
13 * STRN (string) - name is DOBULE_QUOTE, SINGLE_QUOTE
14 * NUMB (number) - name is OCTAL,DECIMAL,HEC_DEC
15 * REGX (reg.expression) - name is REGX
16 * @prop name {String} see type details above
17 * @prop identifier {Identifier} identifier class if relivant
23 * Script.TOKidentifier - type == 'NAME'
24 * Script.TOKassign = data == '='
25 * Script.TOKsemicolon data == '';
33 public enum TokenType {
34 TOKN, // (unknown) - name is UNKNOWN_TOKEN
35 KEYW, // (keyword) - name is upper case version of keyword
36 NAME, // (name/identifier) - name is NAME
37 COMM, // (comment) - name is MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM
38 PUNC, // (puctuation) - name is String description of punctionan (eg LEFTPARAM)
39 WHIT, // (white space) - name is SPACE,NEWLINE
40 STRN, // (string) - name is DOBULE_QUOTE, SINGLE_QUOTE
41 NUMB, // (number) - name is OCTAL,DECIMAL,HEC_DEC
42 REGX, // (reg.expression) - name is REGX
48 public enum TokenName {
86 MULTI_LINE_COMM, JSDOC, SINGLE_LINE_COMM,
129 DOUBLE_QUOTE, SINGLE_QUOTE,
130 OCTAL,DECIMAL,HEX_DEC,
136 UNKNOWN // we should change void/void to void/unknown.
139 public class TokenKeyMap : Object {
141 public Gee.ArrayList<Token> vals;
145 this.key = new Token("",TokenType.VOID, TokenName.VOID);
146 this.vals = new Gee.ArrayList<Token>();
153 public class Token : Object {
158 public TokenType type;
159 public TokenName name;
161 public string prefix; // white space prefix... (when outputing with WS)
163 public string outData;
165 public Identifier identifier;
169 // used to stuff tokens together when building a tree..
170 public Gee.ArrayList<Gee.ArrayList<Token>> items;
171 // for a object definition, key -> array of tokens..
172 public Gee.HashMap<string,TokenKeyMap> props;
173 public Gee.ArrayList<string> keyseq;
174 // props??? what's this???
176 public Token(string data, TokenType type, TokenName name, int line = -1)
183 this.outData = ""; // used by packer/scopeparser
184 this.identifier = null; // used by scope
185 this.id = Token_id++;
187 // should we initialize when needed...?? to keep the usage down..
191 if (name == TokenName.LEFT_BRACE ||
192 name == TokenName.LEFT_CURLY ||
193 name == TokenName.LEFT_PAREN ) {
195 this.items = new Gee.ArrayList<Gee.ArrayList<Token>>();
196 this.props = new Gee.HashMap<string,TokenKeyMap>();
197 this.keyseq = new Gee.ArrayList<string>();
202 public string asString()
204 if (this.name == TokenName.LEFT_BRACE ||
205 this.name == TokenName.LEFT_CURLY ||
206 this.name == TokenName.LEFT_PAREN ) {
208 return "line: %s, %d, id %d, type %s, IS=%d,PS=%d,KS=%d, data : %s, name %s, , outData: %s".printf(
209 DocParser.currentSourceFile == null ? "??" : DocParser.currentSourceFile,
212 this.type.to_string(),
213 this.name == TokenName.LEFT_BRACE ? -1 : this.items.size,
214 this.name == TokenName.LEFT_BRACE ? -1 : this.props.size,
215 this.name == TokenName.LEFT_BRACE ? -1 : this.keyseq.size,
217 this.name.to_string(),
218 this.outData == null ? "" : this.outData
226 return "line:%s:%d, id %d, type %s, data : %s, name %s, , outData: %s".printf(
227 DocParser.currentSourceFile == null ? "??" : DocParser.currentSourceFile,
230 this.type.to_string(),
233 this.name.to_string(),
234 this.outData == null ? "" : this.outData
240 public void dump(string indent)
242 print("%s%s\n",indent, this.asString());
243 if (this.items != null && this.items.size > 0) {
245 for (var i = 0;i < this.items.size; i++) {
246 print("%s --ITEMS[%d] [ \n",indent,i);
247 for (var j = 0;j < this.items[i].size; j++) {
248 this.items[i][j].dump(indent + " ");
252 if (this.props != null && this.props.size > 0) {
253 var m = this.props.map_iterator();
255 print("%s --KEY %s :: \n",indent,m.get_key());
256 var vals = m.get_value().vals;
257 for (var i = 0;i < vals.size; i++) {
259 vals[i].dump(indent + " ");
269 public string toRaw(int lvl = 0)
273 var ret = this.data ;
275 foreach(var ai in this.items ) {
276 // supposed to iterate properties???
278 //foreach( var it in ai) {
279 // str += it.toRaw(lvl+1);
284 /* -- what is a prop..
286 for (var i in this.props) {
287 ret += this.props[i].key.toRaw(lvl+1) + ' : ';
288 this.props[i].val.forEach( function(e) {
297 return this.prefix + ret;
305 eval( "_tmp = " + this.data);
308 return "ERROR unparsable" + this.data;
314 public bool isName(TokenName what) {
315 return this.name == what;
317 public bool isType(TokenType what) {
318 return this.type == what;