Submit after session in PEP8 interrogation room
[gnome.gobject-introspection] / giscanner / transformer.py
1 # -*- Mode: Python -*-
2 # GObject-Introspection - a framework for introspecting GObject libraries
3 # Copyright (C) 2008  Johan Dahlin
4 #
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 # 02110-1301, USA.
19 #
20
21 import os
22 import re
23
24 from giscanner.ast import (Callback, Enum, Function, Namespace, Member,
25                            Parameter, Return, Array, Struct, Field,
26                            Type, Alias, Interface, Class, Node, Union,
27                            List, Map, Varargs, Constant, type_name_from_ctype,
28                            type_names, default_array_types, default_out_types,
29                            TYPE_STRING)
30 from giscanner.config import DATADIR
31 from .glibast import GLibBoxed
32 from giscanner.sourcescanner import (
33     SourceSymbol, ctype_name, CTYPE_POINTER,
34     CTYPE_BASIC_TYPE, CTYPE_UNION, CTYPE_ARRAY, CTYPE_TYPEDEF,
35     CTYPE_VOID, CTYPE_ENUM, CTYPE_FUNCTION, CTYPE_STRUCT,
36     CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT,
37     CSYMBOL_TYPE_ENUM, CSYMBOL_TYPE_UNION, CSYMBOL_TYPE_OBJECT,
38     CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST,
39     TYPE_QUALIFIER_CONST)
40 from .odict import odict
41 from .utils import strip_common_prefix, to_underscores
42
43 _xdg_data_dirs = [x for x in os.environ.get('XDG_DATA_DIRS', '').split(':') \
44                       + [DATADIR, '/usr/share'] if x]
45
46
47 class SkipError(Exception):
48     pass
49
50
51 class Names(object):
52     names = property(lambda self: self._names)
53     aliases = property(lambda self: self._aliases)
54     type_names = property(lambda self: self._type_names)
55     ctypes = property(lambda self: self._ctypes)
56
57     def __init__(self):
58         super(Names, self).__init__()
59         self._names = odict() # Maps from GIName -> (namespace, node)
60         self._aliases = {} # Maps from GIName -> GIName
61         self._type_names = {} # Maps from GTName -> (namespace, node)
62         self._ctypes = {} # Maps from CType -> (namespace, node)
63
64
65 class Transformer(object):
66
67     def __init__(self, generator, namespace_name, namespace_version):
68         self.generator = generator
69         self._namespace = Namespace(namespace_name, namespace_version)
70         self._names = Names()
71         self._typedefs_ns = {}
72         self._strip_prefix = ''
73         self._includes = set()
74         self._includepaths = []
75
76         self._list_ctypes = []
77         self._map_ctypes = []
78
79     def get_names(self):
80         return self._names
81
82     def get_includes(self):
83         return self._includes
84
85     def set_container_types(self, list_ctypes, map_ctypes):
86         self._list_ctypes = list_ctypes
87         self._map_ctypes = map_ctypes
88
89     def set_strip_prefix(self, strip_prefix):
90         self._strip_prefix = strip_prefix
91
92     def parse(self):
93         nodes = []
94         for symbol in self.generator.get_symbols():
95             node = self._traverse_one(symbol)
96             self._add_node(node)
97         return self._namespace
98
99     def set_include_paths(self, paths):
100         self._includepaths = list(paths)
101
102     def register_include(self, include, path=None):
103         if path is None:
104             girname = '%s-%s.gir' % (include.name, include.version)
105             searchdirs = list(self._includepaths)
106             searchdirs.extend([os.path.join(d, 'gir')
107                                for d in _xdg_data_dirs])
108             for d in searchdirs:
109                 path = os.path.join(d, girname)
110                 if os.path.exists(path):
111                     break
112                 path = None
113             if not path:
114                 raise ValueError("Couldn't find include %r (search path: %r)"\
115                                      % (girname, searchdirs))
116         d = os.path.dirname(path)
117         self._includes.add(include)
118         from .girparser import GIRParser
119         parser = GIRParser(path)
120         for include in parser.get_includes():
121             self.register_include(include)
122         nsname = parser.get_namespace().name
123         for node in parser.get_namespace().nodes:
124             if isinstance(node, Alias):
125                 self._names.aliases[node.name] = (nsname, node)
126             elif isinstance(node, (GLibBoxed, Interface, Class)):
127                 self._names.type_names[node.type_name] = (nsname, node)
128             self._names.names[node.name] = (nsname, node)
129             if hasattr(node, 'ctype'):
130                 self._names.ctypes[node.ctype] = (nsname, node)
131             elif hasattr(node, 'symbol'):
132                 self._names.ctypes[node.symbol] = (nsname, node)
133
134     # Private
135
136     def _add_node(self, node):
137         if node is None:
138             return
139         if node.name.startswith('_'):
140             return
141         self._namespace.nodes.append(node)
142         self._names.names[node.name] = (None, node)
143
144     def _strip_namespace_func(self, name):
145         prefix = self._namespace.name.lower() + '_'
146         if name.lower().startswith(prefix):
147             name = name[len(prefix):]
148         else:
149             prefix = to_underscores(self._namespace.name).lower() + '_'
150             if name.lower().startswith(prefix):
151                 name = name[len(prefix):]
152         return self.remove_prefix(name, isfunction=True)
153
154     def remove_prefix(self, name, isfunction=False):
155         # when --strip-prefix=g:
156         #   GHashTable -> HashTable
157         #   g_hash_table_new -> hash_table_new
158         prefix = self._strip_prefix.lower()
159         if isfunction:
160             prefix += '_'
161         if len(name) > len(prefix) and name.lower().startswith(prefix):
162             name = name[len(prefix):]
163
164         while name.startswith('_'):
165             name = name[1:]
166         return name
167
168     def _traverse_one(self, symbol, stype=None):
169         assert isinstance(symbol, SourceSymbol), symbol
170
171         if stype is None:
172             stype = symbol.type
173         if stype == CSYMBOL_TYPE_FUNCTION:
174             try:
175                 return self._create_function(symbol)
176             except SkipError:
177                 return
178         elif stype == CSYMBOL_TYPE_TYPEDEF:
179             return self._create_typedef(symbol)
180         elif stype == CSYMBOL_TYPE_STRUCT:
181             return self._create_struct(symbol)
182         elif stype == CSYMBOL_TYPE_ENUM:
183             return self._create_enum(symbol)
184         elif stype == CSYMBOL_TYPE_OBJECT:
185             return self._create_object(symbol)
186         elif stype == CSYMBOL_TYPE_MEMBER:
187             return self._create_member(symbol)
188         elif stype == CSYMBOL_TYPE_UNION:
189             return self._create_union(symbol)
190         elif stype == CSYMBOL_TYPE_CONST:
191             return self._create_const(symbol)
192         else:
193             raise NotImplementedError(
194                 'Transformer: unhandled symbol: %r' % (symbol, ))
195
196     def _create_enum(self, symbol):
197         members = []
198         for child in symbol.base_type.child_list:
199             name = strip_common_prefix(symbol.ident, child.ident).lower()
200             members.append(Member(name,
201                                   child.const_int,
202                                   child.ident))
203
204         enum_name = self.remove_prefix(symbol.ident)
205         enum = Enum(enum_name, symbol.ident, members)
206         self._names.type_names[symbol.ident] = (None, enum)
207         return enum
208
209     def _create_object(self, symbol):
210         return Member(symbol.ident, symbol.base_type.name,
211                       symbol.ident)
212
213     def _parse_deprecated(self, node, directives):
214         deprecated = directives.get('deprecated', False)
215         if deprecated:
216             deprecated_value = deprecated[0]
217             if ':' in deprecated_value:
218                 # Split out gtk-doc version
219                 (node.deprecated_version, node.deprecated) = \
220                     [x.strip() for x in deprecated_value.split(':', 1)]
221             else:
222                 # No version, just include str
223                 node.deprecated = deprecated_value.strip()
224
225     def _pair_array(self, params, array):
226         if not array.type.length_param_name:
227             return
228         target_name = array.type.length_param_name
229         for i, param in enumerate(params):
230             if param.name == array.type.length_param_name:
231                 array.type.length_param_index = i
232                 return
233         raise ValueError("Unmatched length parameter name %r"\
234                              % (target_name, ))
235
236     def _pair_annotations(self, params):
237         names = {}
238         for param in params:
239             if param.name in names:
240                 raise ValueError("Duplicate parameter name %r"\
241                                      % (param.name, ))
242             names[param.name] = 1
243             if isinstance(param.type, Array):
244                 self._pair_array(params, param)
245
246     # We take the annotations from the parser as strings; here we
247     # want to split them into components, so:
248     # (transfer full) -> {'transfer' : [ 'full' ]}
249
250     def _parse_options(self, options):
251         ret = {}
252         ws_re = re.compile(r'\s+')
253         for opt in options:
254             items = ws_re.split(opt)
255             ret[items[0]] = items[1:]
256         return ret
257
258     def _create_function(self, symbol):
259         directives = symbol.directives()
260         parameters = list(self._create_parameters(
261             symbol.base_type, directives))
262         self._pair_annotations(parameters)
263         return_ = self._create_return(symbol.base_type.base_type,
264                                       directives.get('return', {}))
265         name = self._strip_namespace_func(symbol.ident)
266         func = Function(name, return_, parameters, symbol.ident)
267         self._parse_deprecated(func, directives)
268         return func
269
270     def _create_source_type(self, source_type):
271         if source_type is None:
272             return 'None'
273         if source_type.type == CTYPE_VOID:
274             value = 'void'
275         elif source_type.type == CTYPE_BASIC_TYPE:
276             value = source_type.name
277         elif source_type.type == CTYPE_TYPEDEF:
278             value = source_type.name
279         elif source_type.type == CTYPE_ARRAY:
280             return self._create_source_type(source_type.base_type)
281         elif source_type.type == CTYPE_POINTER:
282             value = self._create_source_type(source_type.base_type) + '*'
283         else:
284             print 'TRANSFORMER: Unhandled source type %r' % (
285                 source_type, )
286             value = 'any'
287         return value
288
289     def _create_parameters(self, base_type, directives=None):
290         if directives is None:
291             dirs = {}
292         else:
293             dirs = directives
294
295         # warn if we see annotations for unknown parameters
296         param_names = set(child.ident for child in base_type.child_list)
297         dirs_for = set(dirs)
298         dirs_for = dirs_for.difference(param_names)
299         dirs_for.discard('return')
300         if dirs_for:
301             print 'Unexpected annotations for %s, parameters are %s' % (
302                 list(dirs_for), list(param_names), )
303
304         for child in base_type.child_list:
305             yield self._create_parameter(
306                 child, dirs.get(child.ident, {}))
307
308     def _create_member(self, symbol):
309         ctype = symbol.base_type.type
310         if (ctype == CTYPE_POINTER and
311             symbol.base_type.base_type.type == CTYPE_FUNCTION):
312             node = self._create_callback(symbol)
313         else:
314             ftype = self._create_type(symbol.base_type, {})
315             # Fields are assumed to be read-write
316             # (except for Objects, see also glibtransformer.py)
317             node = Field(symbol.ident, ftype, symbol.ident,
318                        readable=True, writable=True, bits=symbol.const_int)
319         return node
320
321     def _create_typedef(self, symbol):
322         ctype = symbol.base_type.type
323         if (ctype == CTYPE_POINTER and
324             symbol.base_type.base_type.type == CTYPE_FUNCTION):
325             node = self._create_callback(symbol)
326         elif ctype == CTYPE_STRUCT:
327             node = self._create_typedef_struct(symbol)
328         elif ctype == CTYPE_UNION:
329             node = self._create_typedef_union(symbol)
330         elif ctype == CTYPE_ENUM:
331             return self._create_enum(symbol)
332         elif ctype in (CTYPE_TYPEDEF,
333                        CTYPE_POINTER,
334                        CTYPE_BASIC_TYPE,
335                        CTYPE_VOID):
336             name = self.remove_prefix(symbol.ident)
337             if symbol.base_type.name:
338                 target = self.remove_prefix(symbol.base_type.name)
339             else:
340                 target = 'none'
341             if name in type_names:
342                 return None
343             return Alias(name, target, ctype=symbol.ident)
344         else:
345             raise NotImplementedError(
346                 "symbol %r of type %s" % (symbol.ident, ctype_name(ctype)))
347         return node
348
349     def _parse_ctype(self, ctype):
350         # First look up the ctype including any pointers;
351         # a few type names like 'char*' have their own aliases
352         # and we need pointer information for those.
353         firstpass = type_name_from_ctype(ctype)
354
355         # Remove all pointers - we require standard calling
356         # conventions.  For example, an 'int' is always passed by
357         # value (unless it's out or inout).
358         derefed = firstpass.replace('*', '')
359
360         # Canonicalize our type again, this time without the pointer;
361         # this ensures we turn e.g. plain "guint" => "int"
362         return type_name_from_ctype(derefed)
363
364     def _create_type(self, source_type, options):
365         ctype = self._create_source_type(source_type)
366         if ctype == 'va_list':
367             raise SkipError()
368         # FIXME: FILE* should not be skipped, it should be handled
369         #        properly instead
370         elif ctype == 'FILE*':
371             raise SkipError
372
373         # Now check for a list/map/array type
374         if ctype in self._list_ctypes:
375             param = options.get('element-type')
376             if param:
377                 contained_type = self._parse_ctype(param[0])
378             else:
379                 contained_type = None
380             return List(ctype.replace('*', ''),
381                         ctype,
382                         contained_type)
383         elif ctype in self._map_ctypes:
384             param = options.get('element-type')
385             if param:
386                 key_type = self._parse_ctype(param[0])
387                 value_type = self._parse_ctype(param[1])
388             else:
389                 key_type = None
390                 value_type = None
391             return Map(ctype.replace('*', ''),
392                        ctype,
393                        key_type, value_type)
394         elif (ctype in default_array_types) or ('array' in options):
395             derefed = ctype[:-1] # strip the *
396             result = Array(ctype,
397                          self._parse_ctype(derefed))
398             array_opts = options.get('array')
399             if array_opts:
400                 (_, len_name) = array_opts[0].split('=')
401                 result.length_param_name = len_name
402             return result
403
404         # string memory management - we just look at 'const'
405         if (type_name_from_ctype(ctype) == TYPE_STRING
406             and 'transfer' not in options):
407             if source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST:
408                 options['transfer'] = ['none']
409             else:
410                 options['transfer'] = ['full']
411
412         derefed_name = self._parse_ctype(ctype)
413
414         # deduce direction for some types passed by reference
415         if (not ('out' in options or
416                  'in' in options or
417                  'inout' in options or
418                  'in-out' in options) and
419             source_type.type == CTYPE_POINTER and
420             derefed_name in default_out_types):
421             options['out'] = []
422
423         return Type(derefed_name, ctype)
424
425     def _handle_generic_param_options(self, param, options):
426         for option, data in options.iteritems():
427             if option == 'transfer':
428                 if data:
429                     depth = data[0]
430                     if depth not in ('none', 'container', 'full'):
431                         raise ValueError("Invalid transfer %r" % (depth, ))
432                 else:
433                     depth = 'full'
434                 param.transfer = depth
435
436     def _create_parameter(self, symbol, options):
437         options = self._parse_options(options)
438         if symbol.type == CSYMBOL_TYPE_ELLIPSIS:
439             ptype = Varargs()
440         else:
441             ptype = self._create_type(symbol.base_type, options)
442         param = Parameter(symbol.ident, ptype)
443         for option, data in options.iteritems():
444             if option in ['in-out', 'inout']:
445                 param.direction = 'inout'
446             elif option == 'in':
447                 param.direction = 'in'
448             elif option == 'out':
449                 param.direction = 'out'
450             elif option == 'allow-none':
451                 param.allow_none = True
452             elif option.startswith(('element-type', 'array')):
453                 pass
454             elif option == 'transfer':
455                 pass
456             else:
457                 print 'Unhandled parameter annotation option: %r' % (
458                     option, )
459         self._handle_generic_param_options(param, options)
460         return param
461
462     def _create_return(self, source_type, options=None):
463         if options is None:
464             options_map = {}
465         else:
466             options_map = self._parse_options(options)
467         rtype = self._create_type(source_type, options_map)
468         rtype = self.resolve_param_type(rtype)
469         return_ = Return(rtype)
470         self._handle_generic_param_options(return_, options_map)
471         for option, data in options_map.iteritems():
472             if option in ('transfer', 'element-type', 'out'):
473                 pass
474             else:
475                 print 'Unhandled return type annotation option: %r' % (
476                     option, )
477         return return_
478
479     def _create_const(self, symbol):
480         name = self.remove_prefix(symbol.ident)
481         if symbol.const_string is None:
482             type_name = 'int'
483             value = symbol.const_int
484         else:
485             type_name = 'utf8'
486             value = symbol.const_string
487         const = Constant(name, type_name, value)
488         return const
489
490     def _create_typedef_struct(self, symbol):
491         name = self.remove_prefix(symbol.ident)
492         struct = Struct(name, symbol.ident)
493         self._typedefs_ns[symbol.ident] = struct
494         self._create_struct(symbol)
495         return struct
496
497     def _create_typedef_union(self, symbol):
498         name = self.remove_prefix(symbol.ident)
499         union = Union(name, symbol.ident)
500         self._typedefs_ns[symbol.ident] = union
501         self._create_union(symbol)
502         return union
503
504     def _create_struct(self, symbol):
505         struct = self._typedefs_ns.get(symbol.ident, None)
506         if struct is None:
507             # This is a bit of a hack; really we should try
508             # to resolve through the typedefs to find the real
509             # name
510             if symbol.ident.startswith('_'):
511                 name = symbol.ident[1:]
512             else:
513                 name = symbol.ident
514             name = self.remove_prefix(name)
515             struct = Struct(name, symbol.ident)
516
517         for child in symbol.base_type.child_list:
518             field = self._traverse_one(child)
519             if field:
520                 struct.fields.append(field)
521
522         return struct
523
524     def _create_union(self, symbol):
525         union = self._typedefs_ns.get(symbol.ident, None)
526         if union is None:
527             # This is a bit of a hack; really we should try
528             # to resolve through the typedefs to find the real
529             # name
530             if symbol.ident.startswith('_'):
531                 name = symbol.ident[1:]
532             else:
533                 name = symbol.ident
534             name = self.remove_prefix(name)
535             union = Union(name, symbol.ident)
536
537         for child in symbol.base_type.child_list:
538             field = self._traverse_one(child)
539             if field:
540                 union.fields.append(field)
541
542         return union
543
544     def _create_callback(self, symbol):
545         parameters = self._create_parameters(symbol.base_type.base_type)
546         retval = self._create_return(symbol.base_type.base_type.base_type)
547         if symbol.ident.find('_') > 0:
548             name = self.remove_prefix(symbol.ident, True)
549         else:
550             name = self.remove_prefix(symbol.ident)
551         return Callback(name, retval, list(parameters), symbol.ident)
552
553     def _typepair_to_str(self, item):
554         nsname, item = item
555         if nsname is None:
556             return item.name
557         return '%s.%s' % (nsname, item.name)
558
559     def _resolve_type_name_1(self, type_name, ctype, names):
560         # First look using the built-in names
561         if ctype:
562             try:
563                 return type_names[ctype]
564             except KeyError, e:
565                 pass
566         try:
567             return type_names[type_name]
568         except KeyError, e:
569             pass
570         type_name = self.remove_prefix(type_name)
571         resolved = names.aliases.get(type_name)
572         if resolved:
573             return self._typepair_to_str(resolved)
574         resolved = names.names.get(type_name)
575         if resolved:
576             return self._typepair_to_str(resolved)
577         if ctype:
578             ctype = ctype.replace('*', '')
579             resolved = names.ctypes.get(ctype)
580             if resolved:
581                 return self._typepair_to_str(resolved)
582         resolved = names.type_names.get(type_name)
583         if resolved:
584             return self._typepair_to_str(resolved)
585         raise KeyError("failed to find %r" % (type_name, ))
586
587     def resolve_type_name_full(self, type_name, ctype,
588                                names, allow_invalid=True):
589         try:
590             return self._resolve_type_name_1(type_name, ctype, names)
591         except KeyError, e:
592             try:
593                 return self._resolve_type_name_1(type_name, ctype, self._names)
594             except KeyError, e:
595                 if not allow_invalid:
596                     raise
597                 return type_name
598
599     def resolve_type_name(self, type_name, ctype=None):
600         try:
601             return self.resolve_type_name_full(type_name, ctype, self._names)
602         except KeyError, e:
603             return type_name
604
605     def gtypename_to_giname(self, gtname, names):
606         resolved = names.type_names.get(gtname)
607         if resolved:
608             return self._typepair_to_str(resolved)
609         resolved = self._names.type_names.get(gtname)
610         if resolved:
611             return self._typepair_to_str(resolved)
612         raise KeyError("Failed to resolve GType name: %r" % (gtname, ))
613
614     def ctype_of(self, obj):
615         if hasattr(obj, 'ctype'):
616             return obj.ctype
617         elif hasattr(obj, 'symbol'):
618             return obj.symbol
619         else:
620             return None
621
622     def resolve_param_type_full(self, ptype, names, **kwargs):
623         if isinstance(ptype, Node):
624             ptype.name = self.resolve_type_name_full(ptype.name,
625                                                      self.ctype_of(ptype),
626                                                      names, **kwargs)
627             if isinstance(ptype, (Array, List)):
628                 if ptype.element_type is not None:
629                     ptype.element_type = \
630                         self.resolve_param_type_full(ptype.element_type,
631                                                      names, **kwargs)
632             if isinstance(ptype, Map):
633                 if ptype.key_type is not None:
634                     ptype.key_type = \
635                         self.resolve_param_type_full(ptype.key_type,
636                                                      names, **kwargs)
637                     ptype.value_type = \
638                         self.resolve_param_type_full(ptype.value_type,
639                                                      names, **kwargs)
640         elif isinstance(ptype, basestring):
641             return self.resolve_type_name_full(ptype, None, names, **kwargs)
642         else:
643             raise AssertionError("Unhandled param: %r" % (ptype, ))
644         return ptype
645
646     def resolve_param_type(self, ptype):
647         try:
648             return self.resolve_param_type_full(ptype, self._names)
649         except KeyError, e:
650             return ptype
651
652     def follow_aliases(self, type_name, names):
653         while True:
654             resolved = names.aliases.get(type_name)
655             if resolved:
656                 (ns, alias) = resolved
657                 type_name = alias.target
658             else:
659                 break
660         return type_name