Bug 555964 - Parse floating-point #defines
[gnome.gobject-introspection] / giscanner / transformer.py
1 # -*- Mode: Python -*-
2 # GObject-Introspection - a framework for introspecting GObject libraries
3 # Copyright (C) 2008  Johan Dahlin
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Lesser General Public License for more details.
14 #
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the
17 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 # Boston, MA 02111-1307, USA.
19 #
20
21 import os
22
23 from .ast import (Bitfield, Callback, Enum, Function, Namespace, Member,
24                   Parameter, Return, Struct, Field,
25                   Type, Array, Alias, Interface, Class, Node, Union,
26                   Varargs, Constant, type_name_from_ctype,
27                   type_names, TYPE_STRING, BASIC_GIR_TYPES)
28 from .config import DATADIR
29 from .glibast import GLibBoxed
30 from .girparser import GIRParser
31 from .odict import odict
32 from .sourcescanner import (
33     SourceSymbol, ctype_name, CTYPE_POINTER,
34     CTYPE_BASIC_TYPE, CTYPE_UNION, CTYPE_ARRAY, CTYPE_TYPEDEF,
35     CTYPE_VOID, CTYPE_ENUM, CTYPE_FUNCTION, CTYPE_STRUCT,
36     CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT,
37     CSYMBOL_TYPE_ENUM, CSYMBOL_TYPE_UNION, CSYMBOL_TYPE_OBJECT,
38     CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST,
39     TYPE_QUALIFIER_CONST)
40 from .utils import strip_common_prefix, to_underscores
41
42 _xdg_data_dirs = [x for x in os.environ.get('XDG_DATA_DIRS', '').split(':') \
43                       + [DATADIR, '/usr/share'] if x]
44
45
46 class SkipError(Exception):
47     pass
48
49
50 class Names(object):
51     names = property(lambda self: self._names)
52     aliases = property(lambda self: self._aliases)
53     type_names = property(lambda self: self._type_names)
54     ctypes = property(lambda self: self._ctypes)
55
56     def __init__(self):
57         super(Names, self).__init__()
58         self._names = odict() # Maps from GIName -> (namespace, node)
59         self._aliases = {} # Maps from GIName -> GIName
60         self._type_names = {} # Maps from GTName -> (namespace, node)
61         self._ctypes = {} # Maps from CType -> (namespace, node)
62
63
64 class Transformer(object):
65
66     def __init__(self, cachestore, namespace_name, namespace_version):
67         self._cachestore = cachestore
68         self.generator = None
69         self._namespace = Namespace(namespace_name, namespace_version)
70         self._names = Names()
71         self._pkg_config_packages = set()
72         self._typedefs_ns = {}
73         self._strip_prefix = ''
74         self._includes = set()
75         self._includepaths = []
76
77     def get_names(self):
78         return self._names
79
80     def get_includes(self):
81         return self._includes
82
83     def set_strip_prefix(self, strip_prefix):
84         self._strip_prefix = strip_prefix
85
86     def get_pkgconfig_packages(self):
87         return self._pkg_config_packages
88
89     def set_source_ast(self, src_ast):
90         self.generator = src_ast
91
92     def parse(self):
93         nodes = []
94         for symbol in self.generator.get_symbols():
95             try:
96                 node = self._traverse_one(symbol)
97             except SkipError:
98                 continue
99             self._add_node(node)
100         return self._namespace
101
102     def set_include_paths(self, paths):
103         self._includepaths = list(paths)
104
105     def register_include(self, include):
106         if include in self._includes:
107             return
108         filename = self._find_include(include)
109         self._parse_include(filename)
110         self._includes.add(include)
111
112     # Private
113
114     def _find_include(self, include):
115         searchdirs = self._includepaths[:]
116         for path in _xdg_data_dirs:
117             searchdirs.append(os.path.join(path, 'gir-1.0'))
118
119         girname = '%s-%s.gir' % (include.name, include.version)
120         for d in searchdirs:
121             path = os.path.join(d, girname)
122             if os.path.exists(path):
123                 return path
124         raise ValueError("Couldn't find include %r (search path: %r)"\
125                          % (girname, searchdirs))
126
127     def _parse_include(self, filename):
128         parser = self._cachestore.load(filename)
129         if parser is None:
130             parser = GIRParser()
131             parser.set_include_parsing(True)
132             parser.parse(filename)
133             self._cachestore.store(filename, parser)
134
135         for include in parser.get_includes():
136             self.register_include(include)
137
138         for pkg in parser.get_pkgconfig_packages():
139             self._pkg_config_packages.add(pkg)
140         namespace = parser.get_namespace()
141         nsname = namespace.name
142         for node in namespace.nodes:
143             if isinstance(node, Alias):
144                 self._names.aliases[node.name] = (nsname, node)
145             elif isinstance(node, (GLibBoxed, Interface, Class)):
146                 self._names.type_names[node.type_name] = (nsname, node)
147             giname = '%s.%s' % (nsname, node.name)
148             self._names.names[giname] = (nsname, node)
149             if hasattr(node, 'ctype'):
150                 self._names.ctypes[node.ctype] = (nsname, node)
151             elif hasattr(node, 'symbol'):
152                 self._names.ctypes[node.symbol] = (nsname, node)
153
154     def _add_node(self, node):
155         if node is None:
156             return
157         if node.name.startswith('_'):
158             return
159         self._namespace.nodes.append(node)
160         self._names.names[node.name] = (None, node)
161
162     def _strip_namespace_func(self, name):
163         prefix = self._namespace.name.lower() + '_'
164         if name.lower().startswith(prefix):
165             name = name[len(prefix):]
166         else:
167             prefix = to_underscores(self._namespace.name).lower() + '_'
168             if name.lower().startswith(prefix):
169                 name = name[len(prefix):]
170         return self.remove_prefix(name, isfunction=True)
171
172     def remove_prefix(self, name, isfunction=False):
173         # when --strip-prefix=g:
174         #   GHashTable -> HashTable
175         #   g_hash_table_new -> hash_table_new
176         prefix = self._strip_prefix.lower()
177         if isfunction:
178             prefix += '_'
179         if len(name) > len(prefix) and name.lower().startswith(prefix):
180             name = name[len(prefix):]
181
182         while name.startswith('_'):
183             name = name[1:]
184         return name
185
186     def _traverse_one(self, symbol, stype=None):
187         assert isinstance(symbol, SourceSymbol), symbol
188
189         if stype is None:
190             stype = symbol.type
191         if stype == CSYMBOL_TYPE_FUNCTION:
192             return self._create_function(symbol)
193         elif stype == CSYMBOL_TYPE_TYPEDEF:
194             return self._create_typedef(symbol)
195         elif stype == CSYMBOL_TYPE_STRUCT:
196             return self._create_struct(symbol)
197         elif stype == CSYMBOL_TYPE_ENUM:
198             return self._create_enum(symbol)
199         elif stype == CSYMBOL_TYPE_OBJECT:
200             return self._create_object(symbol)
201         elif stype == CSYMBOL_TYPE_MEMBER:
202             return self._create_member(symbol)
203         elif stype == CSYMBOL_TYPE_UNION:
204             return self._create_union(symbol)
205         elif stype == CSYMBOL_TYPE_CONST:
206             return self._create_const(symbol)
207         else:
208             raise NotImplementedError(
209                 'Transformer: unhandled symbol: %r' % (symbol, ))
210
211     def _create_enum(self, symbol):
212         members = []
213         for child in symbol.base_type.child_list:
214             name = strip_common_prefix(symbol.ident, child.ident).lower()
215             members.append(Member(name,
216                                   child.const_int,
217                                   child.ident))
218
219         enum_name = self.remove_prefix(symbol.ident)
220         if symbol.base_type.is_bitfield:
221             klass = Bitfield
222         else:
223             klass = Enum
224         node = klass(enum_name, symbol.ident, members)
225         self._names.type_names[symbol.ident] = (None, node)
226         return node
227
228     def _create_object(self, symbol):
229         return Member(symbol.ident, symbol.base_type.name,
230                       symbol.ident)
231
232     def _type_is_callback(self, type):
233         if (isinstance(type, Callback) or
234             isinstance(self._typedefs_ns.get(type.name), Callback)):
235             return True
236         return False
237
238     def _handle_closure(self, param, closure_idx, closure_param):
239         if (closure_param.type.name == 'any' and
240             closure_param.name == 'user_data'):
241             param.closure_name = closure_param.name
242             param.closure_index = closure_idx
243             return True
244         return False
245
246     def _handle_destroy(self, param, destroy_idx, destroy_param):
247         if ((self._namespace.name == 'GLib' and
248              destroy_param.type.name == 'DestroyNotify') or
249             destroy_param.type.name == 'GLib.DestroyNotify'):
250             param.destroy_name = destroy_param.name
251             param.destroy_index = destroy_idx
252             return True
253         return False
254
255     def _augment_callback_params(self, params):
256         for i, param in enumerate(params):
257             if not self._type_is_callback(param.type):
258                 continue
259
260             # j is the index where we look for closure/destroy to
261             # group with the callback param
262             j = i + 1
263             if j == len(params):
264                 continue # no more args -> nothing to group look
265             # at the param directly following for either a closure
266             # or a destroy; only one of these will fire
267             had_closure = self._handle_closure(param, j, params[j])
268             had_destroy = self._handle_destroy(param, j, params[j])
269             j += 1
270             # are we out of params, or did we find neither?
271             if j == len(params) or (not had_closure and not had_destroy):
272                 continue
273             # we found either a closure or a destroy; check the
274             # parameter following for the other
275             if not had_closure:
276                 self._handle_closure(param, j, params[j])
277             if not had_destroy:
278                 self._handle_destroy(param, j, params[j])
279
280     def _create_function(self, symbol):
281         parameters = list(self._create_parameters(symbol.base_type))
282         return_ = self._create_return(symbol.base_type.base_type)
283         self._augment_callback_params(parameters)
284         name = self._strip_namespace_func(symbol.ident)
285         func = Function(name, return_, parameters, symbol.ident)
286         return func
287
288     def _create_source_type(self, source_type):
289         if source_type is None:
290             return 'None'
291         if source_type.type == CTYPE_VOID:
292             value = 'void'
293         elif source_type.type == CTYPE_BASIC_TYPE:
294             value = source_type.name
295         elif source_type.type == CTYPE_TYPEDEF:
296             value = source_type.name
297         elif source_type.type == CTYPE_ARRAY:
298             return self._create_source_type(source_type.base_type)
299         elif source_type.type == CTYPE_POINTER:
300             value = self._create_source_type(source_type.base_type) + '*'
301         else:
302             value = 'any'
303         return value
304
305     def _create_parameters(self, base_type):
306
307         # warn if we see annotations for unknown parameters
308         param_names = set(child.ident for child in base_type.child_list)
309         for child in base_type.child_list:
310             yield self._create_parameter(child)
311
312     def _create_member(self, symbol):
313         source_type = symbol.base_type
314         if (source_type.type == CTYPE_POINTER and
315             symbol.base_type.base_type.type == CTYPE_FUNCTION):
316             node = self._create_callback(symbol)
317         elif source_type.type == CTYPE_STRUCT and source_type.name is None:
318             node = self._create_struct(symbol, anonymous=True)
319         elif source_type.type == CTYPE_UNION and source_type.name is None:
320             node = self._create_union(symbol, anonymous=True)
321         else:
322             # Special handling for fields; we don't have annotations on them
323             # to apply later, yet.
324             if source_type.type == CTYPE_ARRAY:
325                 ctype = self._create_source_type(source_type)
326                 canonical_ctype = self._canonicalize_ctype(ctype)
327                 if canonical_ctype[-1] == '*':
328                     derefed_name = canonical_ctype[:-1]
329                 else:
330                     derefed_name = canonical_ctype
331                 derefed_name = self.resolve_param_type(derefed_name)
332                 ftype = Array(ctype, self.parse_ctype(derefed_name))
333                 child_list = list(symbol.base_type.child_list)
334                 ftype.zeroterminated = False
335                 if child_list:
336                     ftype.size = '%d' % (child_list[0].const_int, )
337             else:
338                 ftype = self._create_type(symbol.base_type,
339                                           is_param=False, is_retval=False)
340             ftype = self.resolve_param_type(ftype)
341             # Fields are assumed to be read-write
342             # (except for Objects, see also glibtransformer.py)
343             node = Field(symbol.ident, ftype, ftype.name,
344                          readable=True, writable=True, bits=symbol.const_int)
345         return node
346
347     def _create_typedef(self, symbol):
348         ctype = symbol.base_type.type
349         if (ctype == CTYPE_POINTER and
350             symbol.base_type.base_type.type == CTYPE_FUNCTION):
351             node = self._create_typedef_callback(symbol)
352         elif (ctype == CTYPE_POINTER and
353             symbol.base_type.base_type.type == CTYPE_STRUCT):
354             node = self._create_typedef_struct(symbol, disguised=True)
355         elif ctype == CTYPE_STRUCT:
356             node = self._create_typedef_struct(symbol)
357         elif ctype == CTYPE_UNION:
358             node = self._create_typedef_union(symbol)
359         elif ctype == CTYPE_ENUM:
360             return self._create_enum(symbol)
361         elif ctype in (CTYPE_TYPEDEF,
362                        CTYPE_POINTER,
363                        CTYPE_BASIC_TYPE,
364                        CTYPE_VOID):
365             name = self.remove_prefix(symbol.ident)
366             if symbol.base_type.name:
367                 target = self.remove_prefix(symbol.base_type.name)
368             else:
369                 target = 'none'
370             if name in type_names:
371                 return None
372             return Alias(name, target, ctype=symbol.ident)
373         else:
374             raise NotImplementedError(
375                 "symbol %r of type %s" % (symbol.ident, ctype_name(ctype)))
376         return node
377
378     def _canonicalize_ctype(self, ctype):
379         # First look up the ctype including any pointers;
380         # a few type names like 'char*' have their own aliases
381         # and we need pointer information for those.
382         firstpass = type_name_from_ctype(ctype)
383
384         # If we have a particular alias for this, skip deep
385         # canonicalization to prevent changing
386         # e.g. char* -> int8*
387         if firstpass != ctype:
388             return firstpass
389
390         # We're also done if the type is already a fundamental
391         # known type, or there are no pointers.
392         if ctype in type_names or not firstpass.endswith('*'):
393             return firstpass
394
395         # We have a pointer type.
396         # Strip the end pointer, canonicalize our base type
397         base = firstpass[:-1]
398         canonical_base = self._canonicalize_ctype(base)
399
400         # Append the pointer again
401         canonical = canonical_base + '*'
402
403         return canonical
404
405     def parse_ctype(self, ctype, is_member=False):
406         canonical = self._canonicalize_ctype(ctype)
407
408         # Remove all pointers - we require standard calling
409         # conventions.  For example, an 'int' is always passed by
410         # value (unless it's out or inout).
411         derefed_typename = canonical.replace('*', '')
412
413         # Preserve "pointerness" of struct/union members
414         if (is_member and canonical.endswith('*') and
415             derefed_typename in BASIC_GIR_TYPES):
416             return 'any'
417         else:
418             return derefed_typename
419
420     def _create_type(self, source_type, is_param, is_retval):
421         ctype = self._create_source_type(source_type)
422         if ctype.startswith('va_list'):
423             raise SkipError()
424         # FIXME: FILE* should not be skipped, it should be handled
425         #        properly instead
426         elif ctype == 'FILE*':
427             raise SkipError
428
429         is_member = not (is_param or is_retval)
430         # Here we handle basic type parsing; most of the heavy lifting
431         # and inference comes in annotationparser.py when we merge
432         # in annotation data.
433         derefed_name = self.parse_ctype(ctype, is_member)
434         rettype = Type(derefed_name, ctype)
435         rettype.canonical = self._canonicalize_ctype(ctype)
436         derefed_ctype = ctype.replace('*', '')
437         rettype.derefed_canonical = self._canonicalize_ctype(derefed_ctype)
438
439         canontype = type_name_from_ctype(ctype)
440         if ((canontype == TYPE_STRING or
441              source_type.type == CTYPE_POINTER) and
442             source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST):
443             rettype.is_const = True
444         return rettype
445
446     def _create_parameter(self, symbol):
447         if symbol.type == CSYMBOL_TYPE_ELLIPSIS:
448             ptype = Varargs()
449         else:
450             ptype = self._create_type(symbol.base_type,
451                                       is_param=True, is_retval=False)
452             ptype = self.resolve_param_type(ptype)
453         return Parameter(symbol.ident, ptype)
454
455     def _create_return(self, source_type):
456         rtype = self._create_type(source_type,
457                                   is_param=False, is_retval=True)
458         rtype = self.resolve_param_type(rtype)
459         return_ = Return(rtype)
460         return return_
461
462     def _create_const(self, symbol):
463         # Don't create constants for non-public things
464         # http://bugzilla.gnome.org/show_bug.cgi?id=572790
465         if not symbol.source_filename.endswith('.h'):
466             return None
467         name = self.remove_prefix(symbol.ident)
468         if symbol.const_string is not None:
469             type_name = 'utf8'
470             value = symbol.const_string
471         elif symbol.const_int is not None:
472             type_name = 'int'
473             value = symbol.const_int
474         elif symbol.const_double is not None:
475             type_name = 'double'
476             value = symbol.const_double
477         else:
478             raise AssertionError()
479
480         const = Constant(name, type_name, value)
481         return const
482
483     def _create_typedef_struct(self, symbol, disguised=False):
484         name = self.remove_prefix(symbol.ident)
485         struct = Struct(name, symbol.ident, disguised)
486         self._typedefs_ns[symbol.ident] = struct
487         self._create_struct(symbol)
488         return struct
489
490     def _create_typedef_union(self, symbol):
491         name = self.remove_prefix(symbol.ident)
492         union = Union(name, symbol.ident)
493         self._typedefs_ns[symbol.ident] = union
494         self._create_union(symbol)
495         return union
496
497     def _create_typedef_callback(self, symbol):
498         callback = self._create_callback(symbol)
499         self._typedefs_ns[callback.name] = callback
500         return callback
501
502     def _create_compound(self, klass, symbol, anonymous):
503         if symbol.ident is None:
504             # the compound is an anonymous member of another union or a struct
505             assert anonymous
506             compound = klass(None, None)
507         else:
508             compound = self._typedefs_ns.get(symbol.ident, None)
509
510         if compound is None:
511             # This is a bit of a hack; really we should try
512             # to resolve through the typedefs to find the real
513             # name
514             if symbol.ident.startswith('_'):
515                 name = symbol.ident[1:]
516                 compound = self._typedefs_ns.get(name, None)
517             else:
518                 name = symbol.ident
519             if compound is None:
520                 name = self.remove_prefix(name)
521                 compound = klass(name, symbol.ident)
522
523         for child in symbol.base_type.child_list:
524             field = self._traverse_one(child)
525             if field:
526                 compound.fields.append(field)
527
528         return compound
529
530     def _create_struct(self, symbol, anonymous=False):
531         return self._create_compound(Struct, symbol, anonymous)
532
533     def _create_union(self, symbol, anonymous=False):
534         return self._create_compound(Union, symbol, anonymous)
535
536     def _create_callback(self, symbol):
537         parameters = self._create_parameters(symbol.base_type.base_type)
538         retval = self._create_return(symbol.base_type.base_type.base_type)
539         if symbol.ident.find('_') > 0:
540             name = self.remove_prefix(symbol.ident, True)
541         else:
542             name = self.remove_prefix(symbol.ident)
543         callback = Callback(name, retval, list(parameters), symbol.ident)
544
545         return callback
546
547     def _typepair_to_str(self, item):
548         nsname, item = item
549         if nsname is None:
550             return item.name
551         return '%s.%s' % (nsname, item.name)
552
553     def _resolve_type_name_1(self, type_name, ctype, names):
554         # First look using the built-in names
555         if ctype:
556             try:
557                 return type_names[ctype]
558             except KeyError, e:
559                 pass
560         try:
561             return type_names[type_name]
562         except KeyError, e:
563             pass
564
565         if ctype:
566             ctype = ctype.replace('*', '')
567             resolved = names.ctypes.get(ctype)
568             if resolved:
569                 return self._typepair_to_str(resolved)
570         type_name = self.remove_prefix(type_name)
571         resolved = names.aliases.get(type_name)
572         if resolved:
573             return self._typepair_to_str(resolved)
574         resolved = names.names.get(type_name)
575         if resolved:
576             return self._typepair_to_str(resolved)
577         resolved = names.type_names.get(type_name)
578         if resolved:
579             return self._typepair_to_str(resolved)
580         raise KeyError("failed to find %r" % (type_name, ))
581
582     def resolve_type_name_full(self, type_name, ctype,
583                                names, allow_invalid=True):
584         try:
585             return self._resolve_type_name_1(type_name, ctype, names)
586         except KeyError, e:
587             try:
588                 return self._resolve_type_name_1(type_name, ctype, self._names)
589             except KeyError, e:
590                 if not allow_invalid:
591                     raise
592                 return type_name
593
594     def resolve_type_name(self, type_name, ctype=None):
595         try:
596             return self.resolve_type_name_full(type_name, ctype, self._names)
597         except KeyError, e:
598             return type_name
599
600     def gtypename_to_giname(self, gtname, names):
601         resolved = names.type_names.get(gtname)
602         if resolved:
603             return self._typepair_to_str(resolved)
604         resolved = self._names.type_names.get(gtname)
605         if resolved:
606             return self._typepair_to_str(resolved)
607         raise KeyError("Failed to resolve GType name: %r" % (gtname, ))
608
609     def ctype_of(self, obj):
610         if hasattr(obj, 'ctype'):
611             return obj.ctype
612         elif hasattr(obj, 'symbol'):
613             return obj.symbol
614         else:
615             return None
616
617     def resolve_param_type_full(self, ptype, names, **kwargs):
618         if isinstance(ptype, Node):
619             ptype.name = self.resolve_type_name_full(ptype.name,
620                                                      self.ctype_of(ptype),
621                                                      names, **kwargs)
622         elif isinstance(ptype, basestring):
623             return self.resolve_type_name_full(ptype, None, names, **kwargs)
624         else:
625             raise AssertionError("Unhandled param: %r" % (ptype, ))
626         return ptype
627
628     def resolve_param_type(self, ptype):
629         try:
630             return self.resolve_param_type_full(ptype, self._names)
631         except KeyError, e:
632             return ptype
633
634     def follow_aliases(self, type_name, names):
635         while True:
636             resolved = names.aliases.get(type_name)
637             if resolved:
638                 (ns, alias) = resolved
639                 type_name = alias.target
640             else:
641                 break
642         return type_name