JSDOC/TokenStream.vala
[gnome.introspection-doc-generator] / JSDOC / TokenStream.vala
1
2 /**
3  * @class TokenStream
4  * 
5  * BC notes:
6  * 
7  * nextT => nextTok
8  * lookT => lookTok
9  * 
10  */
11         
12
13 namespace JSDOC {
14
15         public class TokenStream : Object
16         {
17                 
18                 Gee.ArrayList<Token> tokens;
19             int cursor; // where are we in the stream.          
20                 
21                 
22                 public TokenStream(Gee.ArrayList<Token> tokens) {
23                  
24                     this.tokens = tokens;
25
26                     this.rewind();
27                 }
28                 
29
30             
31             public void rewind() {
32                 this.cursor = -1;
33             }
34
35             /**
36                 @type JSDOC.Token
37             */
38             public Token? look (int n, bool considerWhitespace) 
39             {
40
41
42                 if (considerWhitespace == true) {
43                 
44                     if (this.cursor+n < 0 || this.cursor+n > (this.tokens.size -1)) {
45                         return new Token("", "VOID", "START_OF_STREAM");
46                     }
47                     return this.tokens.get(this.cursor+n);
48                 }
49                 
50
51             var count = 0;
52             var i = this.cursor;
53
54             while (true) {
55                 if (i < 0) {
56                         return new Token("", "VOID", "START_OF_STREAM");
57                         }
58                 if (i > this.tokens.size) {
59                         return new Token("", "VOID", "END_OF_STREAM");
60                         }
61
62                 if (i != this.cursor && this.tokens.get(i).is("WHIT")) {
63                         i += (n < 0) ? -1 : 1;
64                     continue;
65                 }
66                 
67                 if (count == n) {
68                     return this.tokens.get(i);
69                 }
70                 count++;
71                 i += (n < 0) ? -1 : 1;
72             }
73
74             return new Token("", "VOID", "STREAM_ERROR"); // because null isn't an object and caller always expects an object
75                 
76             }
77
78             public int lookFor  (string data)
79             {
80                 // non tree version..
81                 var i = this.cursor < 0 ? 0 : this.cursor ;
82                 
83                 while (true) {
84                     if (i >= this.tokens.size) {
85                                 return -1;
86                         }
87                     if (this.tokens.get(i).data == data) {
88                         return i;
89                     }
90                     i++;
91                     
92                 }
93                 // should not get here!
94                 return -1;
95
96             }
97
98
99             /**
100              * look ahead (or back) x number of tokens (which are not comment or whitespace)
101              * ?? used any more?
102              */
103             public Token lookTok (int n) {
104
105
106                 
107                 var count = 0;
108                 var i = this.cursor;
109
110                 while (true) {
111                    // print(i);
112                     if (i < 0) {
113                         if (n > -1) {
114                             i = 0; 
115                             count++;
116                             continue;
117                             
118                         }
119                         return  new Token("", "VOID", "END_OF_STREAM");
120                     }
121                     if (i > this.tokens.length) {
122                                 return  new Token("", "VOID", "END_OF_STREAM");
123                         }
124
125                     if (i != this.cursor && ( this.tokens.get(i).is("WHIT") || this.tokens.get(i).is("COMM"))) {
126                         i += (n < 0) ? -1 : 1;
127                         continue;
128                     }
129                     
130                     if (count == n) {
131                         return this.tokens.get(i);
132                     }
133                     count++;
134                     i += (n < 0) ? -1 : 1;
135                 }
136             // should never get here..
137                 return  new Token("", "VOID", "END_OF_STREAM");; // because null isn't an object and caller always expects an object;
138                 
139             }
140
141             /**
142              *  @return {Token|null}
143              * next token (with white space)
144              */
145                 
146                
147             public Token next(uint howMany) {
148                         
149                 //if (typeof howMany == "undefined") howMany = 1;
150                 //if (howMany < 1) { return  null;              }
151                 var got = [];
152
153                 for (var i = 1; i <= howMany; i++) {
154                     if (this.cursor+i >= this.tokens.length) {
155                         return null;
156                     }
157                     got.push(this.tokens[this.cursor+i]);
158                 }
159                 this.cursor += howMany;
160
161                 if (howMany == 1) {
162                     return got[0];
163                 }
164                 else return got;
165             },
166             // what about comments after 'function'...
167             // is this used ???
168             nextTok  : function() {
169                 return this.nextNonSpace();
170             },
171             nextNonSpace : function ()
172             {
173                 
174                 while (true) {
175                     tok = this.next(1);
176                     if (!tok) {
177                         return false;
178                     }
179                     if (tok.is('WHIT') ||  tok.is('COMM')) {
180                         continue;
181                     }
182                     return tok;
183                 }
184             },
185             /**
186              *    @type JSDOC.Token[]
187              * @param start {String}  token name or data (eg. '{'
188              * @param stop {String} (Optional) token name or data (eg. '}'
189              */
190             balance : function(/**String*/start, /**String*/stop) {
191                 
192                 
193                 start = typeof(Lang.punc(start)) == 'undefined' ? start : Lang.punc(start);
194                 
195                 if (!stop) stop = Lang.matching(start);
196                 
197                 var depth = 0;
198                 var got = [];
199                 var started = false;
200                 //Seed.print("START:" + start);
201                 //Seed.print("STOP:" + stop);
202                 while ((token = this.look())) {
203                     if (token.is(start)) {
204                   //      Seed.print("balance: START : " + depth + " " + token.data);
205                         depth++;
206                         started = true;
207                     }
208                     
209                     if (started) {
210                         got.push(token);
211                     }
212                     
213                     if (token.is(stop)) {
214                         depth--;
215                     //    Seed.print("balance: STOP: "  + depth + " " + token.data);
216                         if (depth < 1) return got;
217                     }
218                     if (!this.next()) break;
219                 }
220                 return false;
221             },
222
223             getMatchingToken : function(/**String*/start, /**String*/stop) {
224                 var depth = 0;
225                 var cursor = this.cursor;
226                 
227                 if (!start) {
228                     start = Lang.matching(stop);
229                     depth = 1;
230                 }
231                 if (!stop) stop = Lang.matching(start);
232                 
233                 while ((token = this.tokens[cursor])) {
234                     if (token.is(start)) {
235                         depth++;
236                     }
237                     
238                     if (token.is(stop) && cursor) {
239                         depth--;
240                         if (depth == 0) return this.tokens[cursor];
241                     }
242                     cursor++;
243                 }
244                 return false;
245             },
246
247             insertAhead : function(/**JSDOC.Token*/token) {
248                 this.tokens.splice(this.cursor+1, 0, token);
249             },
250              
251             remaining : function() {
252                 var ret = [];
253                 while (true) {
254                     var tok = this.look(1,true);
255                     if (!tok || !tok.is || tok.is('VOID')) {
256                         return ret;
257                     }
258                     ret.push(this.next(1));
259                 }
260             },
261              
262
263             arrayToString : function(ar) {
264                 console.log(typeof(ar));
265                 var ret = [];
266                 ar.forEach(function(e) {
267                     ret.push(e.data);
268                 })
269                 return ret.join('');
270             },
271             dump: function(start, end)
272             {
273                 start = Math.max(start || 0, 0);
274                 end = Math.min(end || this.tokens.length, this.tokens.length);
275                 var out='';
276                 for (var i =start;i < end; i++) {
277                     
278                     out += (this.tokens[i].outData == false) ? this.tokens[i].data : this.tokens[i].outData;
279                 };
280                 print(out);
281             }
282 });
283