2 # GObject-Introspection - a framework for introspecting GObject libraries
3 # Copyright (C) 2008 Johan Dahlin
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 # AnnotationParser - parses gtk-doc annotations
26 from .ast import (Array, Bitfield, Callback, Class, Enum, Field, Function,
27 Interface, List, Map, Parameter, Record, Return, Type, Union,
31 PARAM_DIRECTION_INOUT,
35 PARAM_TRANSFER_CONTAINER,
38 from .odict import odict
39 from .glibast import GLibBoxed
41 # All gtk-doc comments needs to start with this:
42 _COMMENT_HEADER = '*\n '
44 # Tags - annotations applyed to comment blocks
47 TAG_DEPRECATED = 'deprecated'
48 TAG_RETURNS = 'returns'
49 TAG_RETURNS_ALT = 'return value'
50 TAG_ATTRIBUTES = 'attributes'
51 TAG_RENAME_TO = 'rename to'
53 # Options - annotations for parameters and return values
54 OPT_ALLOW_NONE = 'allow-none'
56 OPT_ELEMENT_TYPE = 'element-type'
59 OPT_INOUT_ALT = 'in-out'
62 OPT_TRANSFER = 'transfer'
64 OPT_CLOSURE = 'closure'
65 OPT_DESTROY = 'destroy'
68 # Specific option values
69 OPT_VAL_BITFIELD = 'bitfield'
71 # Array options - array specific annotations
72 OPT_ARRAY_FIXED_SIZE = 'fixed-size'
73 OPT_ARRAY_LENGTH = 'length'
74 OPT_ARRAY_ZERO_TERMINATED = 'zero-terminated'
76 OPT_SCOPE_ASYNC = 'async'
77 OPT_SCOPE_CALL = 'call'
78 OPT_SCOPE_NOTIFIED = 'notified'
80 class InvalidAnnotationError(Exception):
84 class DocBlock(object):
86 def __init__(self, name, options):
88 self.options = options
94 return '<DocBlock %r %r>' % (self.name, self.options)
97 if name == TAG_RETURNS:
98 value = self.tags.get(name)
100 return self.tags.get(TAG_RETURNS_ALT)
104 return self.tags.get(name)
107 class DocTag(object):
109 def __init__(self, name):
115 return '<DocTag %r %r>' % (self.name, self.options)
117 class Option(object):
119 def __init__(self, option):
122 for p in option.split(' '):
124 name, value = p.split('=', 1)
128 self._dict[name] = value
130 self._array.append(name)
132 self._array.append((name, value))
135 return '<Option %r>' % (self._array, )
138 assert len(self._array) == 1
139 return self._array[0]
148 class AnnotationParser(object):
150 def __init__(self, namespace, source_scanner, transformer):
152 self._namespace = namespace
153 self._transformer = transformer
154 for comment in source_scanner.get_comments():
155 self._parse_comment(comment)
158 aa = AnnotationApplier(self._blocks, self._transformer)
159 aa.parse(self._namespace)
161 def _parse_comment(self, comment):
162 # We're looking for gtk-doc comments here, they look like this:
166 # Or, alternatively, with options:
168 # * symbol: (name value) ...
170 # symbol is currently one of:
171 # - function: gtk_widget_show
172 # - signal: GtkWidget::destroy
173 # - property: GtkWidget:visible
175 comment = comment.lstrip()
176 if not comment.startswith(_COMMENT_HEADER):
178 comment = comment[len(_COMMENT_HEADER):]
179 comment = comment.strip()
180 if not comment.startswith('* '):
182 comment = comment[2:]
184 pos = comment.find('\n ')
187 block_header = comment[:pos]
188 block_header = block_header.strip()
189 cpos = block_header.find(': ')
191 block_name = block_header[:cpos]
192 block_options, rest = self._parse_options(block_header[cpos+2:])
196 block_name, block_options = block_header, {}
197 block = DocBlock(block_name, block_options)
199 for line in comment[pos+1:].split('\n'):
201 line = line[2:].strip() # Skip ' *'
204 if line.startswith('@'):
206 elif not ': ' in line:
207 comment_lines.append(line)
209 tag_name, value = self._split_tag_namevalue(line)
210 canon_name = tag_name.lower()
211 if canon_name in block.tags:
212 print >> sys.stderr, (
213 "Symbol %s has multiple definition of tag %r" % (
214 block_name, canon_name, ))
215 block.tags[canon_name] = self._create_tag(canon_name, value)
216 block.comment = '\n'.join(comment_lines)
217 self._blocks[block.name] = block
219 def _split_tag_namevalue(self, raw):
220 """Split a line into tag name and value"""
221 parts = raw.split(': ', 1)
225 if tag_name.endswith(':'):
226 tag_name = tag_name[:-1]
228 tag_name, value = parts
229 return (tag_name, value)
231 def _create_tag(self, tag_name, value):
234 tag = DocTag(tag_name)
236 options, rest = self._parse_options(tag.value)
237 tag.options = options
241 def _parse_options(self, value):
247 for i, c in enumerate(value):
248 if c == '(' and opened == -1:
250 if c == ')' and opened != -1:
251 segment = value[opened:i]
252 parts = segment.split(' ', 1)
255 elif len(parts) == 1:
260 if option is not None:
261 option = Option(option)
262 options[name] = option
267 rest = value[last:].strip()
273 class AnnotationApplier(object):
275 def __init__(self, blocks, transformer):
276 self._blocks = blocks
277 self._transformer = transformer
279 def _get_tag(self, block, tag_name):
283 return block.get(tag_name)
285 def parse(self, namespace):
286 self._namespace = namespace
287 for node in namespace.nodes[:]:
288 self._parse_node(node)
291 # Boring parsing boilerplate.
293 def _parse_node(self, node):
294 if isinstance(node, Function):
295 self._parse_function(node)
296 elif isinstance(node, Enum):
297 self._parse_enum(node)
298 elif isinstance(node, Bitfield):
299 self._parse_bitfield(node)
300 elif isinstance(node, Class):
301 self._parse_class(node)
302 elif isinstance(node, Interface):
303 self._parse_interface(node)
304 elif isinstance(node, Callback):
305 self._parse_callback(node)
306 elif isinstance(node, Record):
307 self._parse_record(node)
308 elif isinstance(node, Union):
309 self._parse_union(node)
310 elif isinstance(node, GLibBoxed):
311 self._parse_boxed(node)
313 def _parse_class(self, class_):
314 block = self._blocks.get(class_.type_name)
315 self._parse_node_common(class_, block)
316 self._parse_constructors(class_.constructors)
317 self._parse_methods(class_, class_.methods)
318 self._parse_vfuncs(class_, class_.virtual_methods)
319 self._parse_methods(class_, class_.static_methods)
320 self._parse_properties(class_, class_.properties)
321 self._parse_signals(class_, class_.signals)
322 self._parse_fields(class_, class_.fields)
324 class_.doc = block.comment
326 def _parse_interface(self, interface):
327 block = self._blocks.get(interface.type_name)
328 self._parse_node_common(interface, block)
329 self._parse_methods(interface, interface.methods)
330 self._parse_vfuncs(interface, interface.virtual_methods)
331 self._parse_properties(interface, interface.properties)
332 self._parse_signals(interface, interface.signals)
333 self._parse_fields(interface, interface.fields)
335 interface.doc = block.comment
337 def _parse_record(self, record):
338 block = self._blocks.get(record.symbol)
339 self._parse_node_common(record, block)
340 self._parse_constructors(record.constructors)
341 self._parse_methods(record, record.methods)
342 self._parse_fields(record, record.fields, block)
344 record.doc = block.comment
346 def _parse_boxed(self, boxed):
347 block = self._blocks.get(boxed.name)
348 self._parse_node_common(boxed, block)
349 self._parse_constructors(boxed.constructors)
350 self._parse_methods(boxed, boxed.methods)
352 boxed.doc = block.comment
354 def _parse_union(self, union):
355 block = self._blocks.get(union.name)
356 self._parse_node_common(union, block)
357 self._parse_fields(union, union.fields, block)
358 self._parse_constructors(union.constructors)
359 self._parse_methods(union, union.methods)
361 union.doc = block.comment
363 def _parse_enum(self, enum):
364 block = self._blocks.get(enum.symbol)
365 self._parse_node_common(enum, block)
367 enum.doc = block.comment
368 type_opt = block.options.get(OPT_TYPE)
369 if type_opt and type_opt.one() == OPT_VAL_BITFIELD:
370 # This is hack, but hey, it works :-)
371 enum.__class__ = Bitfield
373 def _parse_bitfield(self, bitfield):
374 block = self._blocks.get(bitfield.symbol)
375 self._parse_node_common(bitfield, block)
377 bitfield.doc = block.comment
379 def _parse_constructors(self, constructors):
380 for ctor in constructors:
381 self._parse_function(ctor)
383 def _parse_fields(self, parent, fields, block=None):
385 self._parse_field(parent, field, block)
387 def _parse_properties(self, parent, properties):
388 for prop in properties:
389 self._parse_property(parent, prop)
391 def _parse_methods(self, parent, methods):
392 for method in methods:
393 self._parse_method(parent, method)
395 def _parse_vfuncs(self, parent, vfuncs):
397 self._parse_vfunc(parent, vfunc)
399 def _parse_signals(self, parent, signals):
400 for signal in signals:
401 self._parse_signal(parent, signal)
403 def _parse_property(self, parent, prop):
404 block = self._blocks.get('%s:%s' % (parent.type_name, prop.name))
405 self._parse_node_common(prop, block)
407 prop.doc = block.comment
409 def _parse_callback(self, callback):
410 block = self._blocks.get(callback.ctype)
411 self._parse_node_common(callback, block)
412 self._parse_params(callback, callback.parameters, block)
413 self._parse_return(callback, callback.retval, block)
415 callback.doc = block.comment
417 def _parse_callable(self, callable, block):
418 self._parse_node_common(callable, block)
419 for i, param in enumerate(callable.parameters):
420 if (param.type.ctype != 'GDestroyNotify' and
421 param.type.name != 'GLib.DestroyNotify'):
425 callback_param = callable.parameters[i-2]
426 if callback_param.closure_index != -1:
427 callback_param.scope = OPT_SCOPE_NOTIFIED
428 callback_param.transfer = PARAM_TRANSFER_NONE
430 self._parse_params(callable, callable.parameters, block)
431 self._parse_return(callable, callable.retval, block)
433 callable.doc = block.comment
435 def _parse_function(self, func):
436 block = self._blocks.get(func.symbol)
437 self._parse_callable(func, block)
438 self._parse_rename_to_func(func, block)
440 def _parse_signal(self, parent, signal):
441 block = self._blocks.get('%s::%s' % (parent.type_name, signal.name))
442 self._parse_node_common(signal, block)
443 # We're only attempting to name the signal parameters if
444 # the number of parameter tags (@foo) is the same or greater
445 # than the number of signal parameters
446 if block and len(block.tags) > len(signal.parameters):
447 names = block.tags.items()
450 for i, param in enumerate(signal.parameters):
452 name, tag = names[i+1]
454 options = getattr(tag, 'options', {})
455 param_type = options.get(OPT_TYPE)
457 param.type = self._resolve(param_type.one(), param.type)
460 self._parse_param(signal, param, tag)
461 self._parse_return(signal, signal.retval, block)
463 signal.doc = block.comment
465 def _parse_method(self, parent, meth):
466 block = self._blocks.get(meth.symbol)
467 self._parse_function(meth)
468 virtual = self._get_tag(block, TAG_VFUNC)
470 invoker_name = virtual.value
472 for vfunc in parent.virtual_methods:
473 if vfunc.name == invoker_name:
475 vfunc.invoker = meth.name
478 print "warning: unmatched virtual invoker %r for method %r" % \
479 (invoker_name, meth.symbol)
481 def _parse_vfunc(self, parent, vfunc):
482 key = '%s::%s' % (parent.type_name, vfunc.name)
483 self._parse_callable(vfunc, self._blocks.get(key))
485 def _parse_field(self, parent, field, block=None):
486 if isinstance(field, Callback):
487 self._parse_callback(field)
491 tag = block.get(field.name)
494 t = tag.options.get('type')
497 field.type.name = self._transformer.resolve_type_name(t.one())
499 def _parse_params(self, parent, params, block):
501 tag = self._get_tag(block, param.name)
502 self._parse_param(parent, param, tag)
504 def _parse_return(self, parent, return_, block):
505 tag = self._get_tag(block, TAG_RETURNS)
506 self._parse_param_ret_common(parent, return_, tag)
508 def _parse_param(self, parent, param, tag):
509 options = getattr(tag, 'options', {})
510 if isinstance(parent, Function):
511 scope = options.get(OPT_SCOPE)
513 param.scope = scope.one()
514 param.transfer = PARAM_TRANSFER_NONE
515 elif (param.type.ctype == 'GAsyncReadyCallback' or
516 param.type.name == 'Gio.AsyncReadyCallback'):
517 param.scope = OPT_SCOPE_ASYNC
518 param.transfer = PARAM_TRANSFER_NONE
520 destroy = options.get(OPT_DESTROY)
522 param.destroy_index = parent.get_parameter_index(destroy.one())
523 self._fixup_param_destroy(parent, param)
524 closure = options.get(OPT_CLOSURE)
526 param.closure_index = parent.get_parameter_index(closure.one())
527 self._fixup_param_closure(parent, param)
528 if isinstance(parent, Callback):
529 if OPT_CLOSURE in options:
530 param.closure_index = parent.get_parameter_index(param.name)
531 self._fixup_param_closure(parent, param)
533 self._parse_param_ret_common(parent, param, tag)
535 def _fixup_param_destroy(self, parent, param):
536 for p in parent.parameters:
537 if p is not param and p.destroy_index == param.destroy_index:
538 p.destroy_index = None
540 def _fixup_param_closure(self, parent, param):
541 for p in parent.parameters:
542 if p is not param and p.closure_index == param.closure_index:
543 p.closure_index = None
545 def _parse_param_ret_common(self, parent, node, tag):
546 options = getattr(tag, 'options', {})
547 node.direction = self._extract_direction(node, options)
548 container_type = self._extract_container_type(
549 parent, node, options)
550 if container_type is not None:
551 node.type = container_type
552 if node.direction is None:
553 node.direction = self._guess_direction(node)
554 node.transfer = self._extract_transfer(parent, node, options)
555 if OPT_ALLOW_NONE in options:
556 node.allow_none = True
557 param_type = options.get(OPT_TYPE)
559 node.type = self._resolve(param_type.one(), node.type)
561 assert node.transfer is not None
562 if tag is not None and tag.comment is not None:
563 node.doc = tag.comment
565 def _extract_direction(self, node, options):
566 if (OPT_INOUT in options or
567 OPT_INOUT_ALT in options):
568 direction = PARAM_DIRECTION_INOUT
569 elif OPT_OUT in options:
570 direction = PARAM_DIRECTION_OUT
571 elif OPT_IN in options:
572 direction = PARAM_DIRECTION_IN
574 direction = node.direction
577 def _guess_array(self, node):
578 ctype = node.type.ctype
581 if not ctype.endswith('*'):
583 if node.type.canonical in default_array_types:
587 def _extract_container_type(self, parent, node, options):
588 has_element_type = OPT_ELEMENT_TYPE in options
589 has_array = OPT_ARRAY in options
591 # FIXME: This is a hack :-(
592 if (not isinstance(node, Field) and
593 (not has_element_type and
594 (node.direction is None
595 or isinstance(node, Return)
596 or node.direction == PARAM_DIRECTION_IN))):
597 if self._guess_array(node):
601 container_type = self._parse_array(parent, node, options)
602 elif has_element_type:
603 container_type = self._parse_element_type(parent, node, options)
605 container_type = None
607 return container_type
609 def _parse_array(self, parent, node, options):
610 array_opt = options.get(OPT_ARRAY)
612 array_values = array_opt.all()
616 element_type = options.get(OPT_ELEMENT_TYPE)
617 if element_type is not None:
618 element_type_node = self._resolve(element_type.one())
620 element_type_node = Type(node.type.name) # erase ctype
622 container_type = Array(node.type.ctype,
624 container_type.is_const = node.type.is_const
625 if OPT_ARRAY_ZERO_TERMINATED in array_values:
626 container_type.zeroterminated = array_values.get(
627 OPT_ARRAY_ZERO_TERMINATED) == '1'
628 length = array_values.get(OPT_ARRAY_LENGTH)
629 if length is not None:
630 param_index = parent.get_parameter_index(length)
631 container_type.length_param_index = param_index
632 # For in parameters we're incorrectly deferring
633 # char/unsigned char to utf8 when a length annotation
635 if (isinstance(node, Parameter) and
636 node.type.name == 'utf8' and
637 self._guess_direction(node) == PARAM_DIRECTION_IN and
638 element_type is None):
639 # FIXME: unsigned char/guchar should be uint8
640 container_type.element_type = Type('int8')
641 container_type.size = array_values.get(OPT_ARRAY_FIXED_SIZE)
642 return container_type
644 def _resolve(self, type_str, orig_node=None):
645 def grab_one(type_str, resolver, top_combiner, combiner):
646 """Return a complete type, and the trailing string part after it.
647 Use resolver() on each identifier, and combiner() on the parts of
648 each complete type. (top_combiner is used on the top-most type.)"""
649 bits = re.split(r'([,<>])', type_str, 1)
650 first, sep, rest = [bits[0], '', ''] if (len(bits)==1) else bits
651 args = [resolver(first)]
654 next, rest = grab_one(rest, resolver, combiner, combiner)
656 sep, rest = rest[0], rest[1:]
659 return top_combiner(*args), rest
661 return self._transformer.resolve_param_type(Type(ident))
662 def combiner(base, *rest):
665 if (base.name in ['GLib.List', 'GLib.SList'] or
666 base.ctype in ['GList*', 'GSList*']) and len(rest)==1:
667 return List(base.name, base.ctype, *rest)
668 if (base.name in ['GLib.HashTable'] or
669 base.ctype in ['GHashTable*']) and len(rest)==2:
670 return Map(base.name, base.ctype, *rest)
671 print "WARNING: throwing away type parameters:", type_str
673 def top_combiner(base, *rest):
674 """For the top most type, recycle orig_node if possible."""
675 if orig_node is not None:
676 orig_node.name = base.name
677 base = orig_node # preserve other properties of orig_node
678 return combiner(base, *rest)
680 result, rest = grab_one(type_str, resolver, top_combiner, combiner)
682 print "WARNING: throwing away trailing part of type:", type_str
685 def _parse_element_type(self, parent, node, options):
686 element_type_opt = options.get(OPT_ELEMENT_TYPE)
687 element_type = element_type_opt.flat()
688 if (node.type.name in ['GLib.List', 'GLib.SList'] or
689 node.type.ctype in ['GList*', 'GSList*']):
690 assert len(element_type) == 1
691 container_type = List(
694 self._resolve(element_type[0]))
695 elif (node.type.name in ['GLib.HashTable'] or
696 node.type.ctype in ['GHashTable*']):
697 assert len(element_type) == 2
698 container_type = Map(
701 self._resolve(element_type[0]),
702 self._resolve(element_type[1]))
704 print 'FIXME: unhandled element-type container:', node
705 return container_type
707 def _extract_transfer(self, parent, node, options):
708 transfer_opt = options.get(OPT_TRANSFER)
709 if transfer_opt is None:
710 transfer = self._guess_transfer(node, options)
712 transfer = transfer_opt.one()
714 transfer = PARAM_TRANSFER_FULL
715 if transfer not in [PARAM_TRANSFER_NONE,
716 PARAM_TRANSFER_CONTAINER,
717 PARAM_TRANSFER_FULL]:
718 raise InvalidAnnotationError(
719 "transfer for %s of %r is invalid (%r), must be one of "
720 "none, container, full." % (node, parent.name, transfer))
723 def _parse_node_common(self, node, block):
724 self._parse_version(node, block)
725 self._parse_deprecated(node, block)
726 self._parse_attributes(node, block)
727 self._parse_skip(node, block)
729 def _parse_version(self, node, block):
730 since_tag = self._get_tag(block, TAG_SINCE)
731 if since_tag is None:
733 node.version = since_tag.value
735 def _parse_deprecated(self, node, block):
736 deprecated_tag = self._get_tag(block, TAG_DEPRECATED)
737 if deprecated_tag is None:
739 value = deprecated_tag.value
741 version, desc = value.split(': ')
745 node.deprecated = desc
746 if version is not None:
747 node.deprecated_version = version
749 def _parse_attributes(self, node, block):
750 annos_tag = self._get_tag(block, TAG_ATTRIBUTES)
751 if annos_tag is None:
753 for key, value in annos_tag.options.iteritems():
754 node.attributes.append((key, value.one()))
756 def _parse_skip(self, node, block):
757 if block is not None:
758 if OPT_SKIP in block.options:
761 def _parse_rename_to_func(self, node, block):
762 rename_to_tag = self._get_tag(block, TAG_RENAME_TO)
763 if rename_to_tag is None:
765 new_name = rename_to_tag.value
769 def shadowed_filter(n):
770 if isinstance(n, Function) and n.symbol == new_name:
775 self._namespace.remove_matching(shadowed_filter)
776 if len(shadowed) == 1:
777 # method override; use the same (stripped) name as the overloaded
779 # Note that 'g_timeout_add_full' may specify a new_name of
780 # 'g_timeout_add' but the *real* name desired is the stripped name
781 # of 'g_timeout_add' which is 'timeout_add' (for example).
782 node.name = shadowed[0].name
783 elif len(shadowed) == 0:
784 # literal rename, to force a particular prefix strip or whatever
785 # Example: the "nm-utils" package uses a "NM" prefix in most places
786 # but some functions have an "nm_utils_" prefix; the 'Rename To:'
787 # annotation in this case is used to strip the 'utils_' part off.
790 assert False # more than two shadowed methods? Shouldn't happen.
792 def _guess_direction(self, node):
794 return node.direction
797 is_pointer = '*' in node.type.ctype
799 if is_pointer and node.type.name in BASIC_GIR_TYPES:
800 return PARAM_DIRECTION_OUT
802 return PARAM_DIRECTION_IN
804 def _guess_transfer(self, node, options):
805 if node.transfer is not None:
808 # Anything with 'const' gets none
809 if node.type.is_const:
810 return PARAM_TRANSFER_NONE
811 elif node.type.name in [TYPE_NONE, TYPE_ANY]:
812 return PARAM_TRANSFER_NONE
813 elif isinstance(node.type, Varargs):
814 return PARAM_TRANSFER_NONE
815 elif isinstance(node, Parameter):
816 if node.direction in [PARAM_DIRECTION_INOUT,
817 PARAM_DIRECTION_OUT]:
818 return PARAM_TRANSFER_FULL
819 # This one is a hack for compatibility; the transfer
820 # for string parameters really has no defined meaning.
821 elif node.type.canonical == 'utf8':
822 return PARAM_TRANSFER_FULL
824 return PARAM_TRANSFER_NONE
825 elif isinstance(node, Return):
826 if (node.type.canonical in BASIC_GIR_TYPES or
827 (node.type.canonical in [TYPE_NONE, TYPE_ANY] and
828 node.type.is_const)):
829 return PARAM_TRANSFER_NONE
831 return PARAM_TRANSFER_FULL
832 elif isinstance(node, Field):
833 return PARAM_TRANSFER_NONE
835 raise AssertionError(node)