2 # GObject-Introspection - a framework for introspecting GObject libraries
3 # Copyright (C) 2008 Johan Dahlin
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 # AnnotationParser - parses gtk-doc annotations
25 from .ast import (Array, Bitfield, Callback, Class, Enum, Field, Function,
26 Interface, List, Map, Parameter, Record, Return, Type, Union,
30 PARAM_DIRECTION_INOUT,
34 PARAM_TRANSFER_CONTAINER,
37 from .odict import odict
38 from .glibast import GLibBoxed
40 # All gtk-doc comments needs to start with this:
41 _COMMENT_HEADER = '*\n '
43 # Tags - annotations applyed to comment blocks
46 TAG_DEPRECATED = 'deprecated'
47 TAG_RETURNS = 'returns'
48 TAG_RETURNS_ALT = 'return value'
49 TAG_ATTRIBUTES = 'attributes'
50 TAG_RENAME_TO = 'rename to'
52 # Options - annotations for parameters and return values
53 OPT_ALLOW_NONE = 'allow-none'
55 OPT_ELEMENT_TYPE = 'element-type'
58 OPT_INOUT_ALT = 'in-out'
61 OPT_TRANSFER = 'transfer'
63 OPT_CLOSURE = 'closure'
64 OPT_DESTROY = 'destroy'
66 # Specific option values
67 OPT_VAL_BITFIELD = 'bitfield'
69 # Array options - array specific annotations
70 OPT_ARRAY_FIXED_SIZE = 'fixed-size'
71 OPT_ARRAY_LENGTH = 'length'
72 OPT_ARRAY_ZERO_TERMINATED = 'zero-terminated'
75 class InvalidAnnotationError(Exception):
79 class DocBlock(object):
81 def __init__(self, name, options):
83 self.options = options
89 return '<DocBlock %r %r>' % (self.name, self.options)
92 if name == TAG_RETURNS:
93 value = self.tags.get(name)
95 return self.tags.get(TAG_RETURNS_ALT)
99 return self.tags.get(name)
102 class DocTag(object):
104 def __init__(self, name):
110 return '<DocTag %r %r>' % (self.name, self.options)
112 class Option(object):
114 def __init__(self, option):
117 for p in option.split(' '):
119 name, value = p.split('=', 1)
123 self._dict[name] = value
125 self._array.append(name)
127 self._array.append((name, value))
130 return '<Option %r>' % (self._array, )
133 assert len(self._array) == 1
134 return self._array[0]
143 class AnnotationParser(object):
145 def __init__(self, namespace, source_scanner, transformer):
147 self._namespace = namespace
148 self._transformer = transformer
149 for comment in source_scanner.get_comments():
150 self._parse_comment(comment)
153 aa = AnnotationApplier(self._blocks, self._transformer)
154 aa.parse(self._namespace)
156 def _parse_comment(self, comment):
157 # We're looking for gtk-doc comments here, they look like this:
161 # Or, alternatively, with options:
163 # * symbol: (name value) ...
165 # symbol is currently one of:
166 # - function: gtk_widget_show
167 # - signal: GtkWidget::destroy
168 # - property: GtkWidget:visible
170 comment = comment.lstrip()
171 if not comment.startswith(_COMMENT_HEADER):
173 comment = comment[len(_COMMENT_HEADER):]
174 comment = comment.strip()
175 if not comment.startswith('* '):
177 comment = comment[2:]
179 pos = comment.find('\n ')
182 block_header = comment[:pos]
183 block_header = block_header.strip()
184 cpos = block_header.find(': ')
186 block_name = block_header[:cpos]
187 block_options, rest = self._parse_options(block_header[cpos+2:])
191 block_name, block_options = block_header, {}
192 block = DocBlock(block_name, block_options)
194 for line in comment[pos+1:].split('\n'):
196 line = line[2:].strip() # Skip ' *'
199 if line.startswith('@'):
201 elif not ': ' in line:
202 comment_lines.append(line)
204 tag_name, value = self._split_tag_namevalue(line)
205 canon_name = tag_name.lower()
206 if canon_name in block.tags:
207 print >>sys.stderr, "Multiple definition of tag %r" \
209 block.tags[canon_name] = self._create_tag(canon_name, value)
210 block.comment = '\n'.join(comment_lines)
211 self._blocks[block.name] = block
213 def _split_tag_namevalue(self, raw):
214 """Split a line into tag name and value"""
215 parts = raw.split(': ', 1)
220 tag_name, value = parts
221 return (tag_name, value)
223 def _create_tag(self, tag_name, value):
226 tag = DocTag(tag_name)
228 options, rest = self._parse_options(tag.value)
229 tag.options = options
233 def _parse_options(self, value):
239 for i, c in enumerate(value):
240 if c == '(' and opened == -1:
242 if c == ')' and opened != -1:
243 segment = value[opened:i]
244 parts = segment.split(' ', 1)
247 elif len(parts) == 1:
252 if option is not None:
253 option = Option(option)
254 options[name] = option
259 rest = value[last:].strip()
265 class AnnotationApplier(object):
267 def __init__(self, blocks, transformer):
268 self._blocks = blocks
269 self._transformer = transformer
271 def _get_tag(self, block, tag_name):
275 return block.get(tag_name)
277 def parse(self, namespace):
278 self._namespace = namespace
279 for node in namespace.nodes[:]:
280 self._parse_node(node)
283 # Boring parsing boilerplate.
285 def _parse_node(self, node):
286 if isinstance(node, Function):
287 self._parse_function(node)
288 elif isinstance(node, Enum):
289 self._parse_enum(node)
290 elif isinstance(node, Bitfield):
291 self._parse_bitfield(node)
292 elif isinstance(node, Class):
293 self._parse_class(node)
294 elif isinstance(node, Interface):
295 self._parse_interface(node)
296 elif isinstance(node, Callback):
297 self._parse_callback(node)
298 elif isinstance(node, Record):
299 self._parse_record(node)
300 elif isinstance(node, Union):
301 self._parse_union(node)
302 elif isinstance(node, GLibBoxed):
303 self._parse_boxed(node)
305 def _parse_class(self, class_):
306 block = self._blocks.get(class_.type_name)
307 self._parse_node_common(class_, block)
308 self._parse_constructors(class_.constructors)
309 self._parse_methods(class_, class_.methods)
310 self._parse_vfuncs(class_, class_.virtual_methods)
311 self._parse_methods(class_, class_.static_methods)
312 self._parse_properties(class_, class_.properties)
313 self._parse_signals(class_, class_.signals)
314 self._parse_fields(class_, class_.fields)
316 class_.doc = block.comment
318 def _parse_interface(self, interface):
319 block = self._blocks.get(interface.type_name)
320 self._parse_node_common(interface, block)
321 self._parse_methods(interface, interface.methods)
322 self._parse_vfuncs(interface, interface.virtual_methods)
323 self._parse_properties(interface, interface.properties)
324 self._parse_signals(interface, interface.signals)
325 self._parse_fields(interface, interface.fields)
327 interface.doc = block.comment
329 def _parse_record(self, record):
330 block = self._blocks.get(record.symbol)
331 self._parse_node_common(record, block)
332 self._parse_constructors(record.constructors)
333 self._parse_methods(record, record.methods)
334 self._parse_fields(record, record.fields)
336 record.doc = block.comment
338 def _parse_boxed(self, boxed):
339 block = self._blocks.get(boxed.name)
340 self._parse_node_common(boxed, block)
341 self._parse_constructors(boxed.constructors)
342 self._parse_methods(boxed, boxed.methods)
344 boxed.doc = block.comment
346 def _parse_union(self, union):
347 block = self._blocks.get(union.name)
348 self._parse_node_common(union, block)
349 self._parse_fields(union, union.fields)
350 self._parse_constructors(union.constructors)
351 self._parse_methods(union, union.methods)
353 union.doc = block.comment
355 def _parse_enum(self, enum):
356 block = self._blocks.get(enum.symbol)
357 self._parse_node_common(enum, block)
359 enum.doc = block.comment
360 type_opt = block.options.get(OPT_TYPE)
361 if type_opt and type_opt.one() == OPT_VAL_BITFIELD:
362 # This is hack, but hey, it works :-)
363 enum.__class__ = Bitfield
365 def _parse_bitfield(self, bitfield):
366 block = self._blocks.get(bitfield.symbol)
367 self._parse_node_common(bitfield, block)
369 bitfield.doc = block.comment
371 def _parse_constructors(self, constructors):
372 for ctor in constructors:
373 self._parse_function(ctor)
375 def _parse_fields(self, parent, fields):
377 self._parse_field(parent, field)
379 def _parse_properties(self, parent, properties):
380 for prop in properties:
381 self._parse_property(parent, prop)
383 def _parse_methods(self, parent, methods):
384 for method in methods:
385 self._parse_method(parent, method)
387 def _parse_vfuncs(self, parent, vfuncs):
389 self._parse_vfunc(parent, vfunc)
391 def _parse_signals(self, parent, signals):
392 for signal in signals:
393 self._parse_signal(parent, signal)
395 def _parse_property(self, parent, prop):
396 block = self._blocks.get('%s:%s' % (parent.type_name, prop.name))
397 self._parse_node_common(prop, block)
399 prop.doc = block.comment
401 def _parse_callback(self, callback):
402 block = self._blocks.get(callback.ctype)
403 self._parse_node_common(callback, block)
404 self._parse_params(callback, callback.parameters, block)
405 self._parse_return(callback, callback.retval, block)
407 callback.doc = block.comment
409 def _parse_callable(self, callable, block):
410 self._parse_node_common(callable, block)
411 self._parse_params(callable, callable.parameters, block)
412 self._parse_return(callable, callable.retval, block)
414 callable.doc = block.comment
416 def _parse_function(self, func):
417 block = self._blocks.get(func.symbol)
418 self._parse_callable(func, block)
419 self._parse_rename_to_func(func, block)
421 def _parse_signal(self, parent, signal):
422 block = self._blocks.get('%s::%s' % (parent.type_name, signal.name))
423 self._parse_node_common(signal, block)
424 # We're only attempting to name the signal parameters if
425 # the number of parameter tags (@foo) is the same or greater
426 # than the number of signal parameters
427 resolve = self._transformer.resolve_param_type
428 if block and len(block.tags) > len(signal.parameters):
429 names = block.tags.items()
432 for i, param in enumerate(signal.parameters):
434 name, tag = names[i+1]
436 options = getattr(tag, 'options', {})
437 param_type = options.get(OPT_TYPE)
439 param.type.name = resolve(param_type.one())
442 self._parse_param(signal, param, tag)
443 self._parse_return(signal, signal.retval, block)
445 signal.doc = block.comment
447 def _parse_method(self, parent, meth):
448 block = self._blocks.get(meth.symbol)
449 self._parse_function(meth)
450 virtual = self._get_tag(block, TAG_VFUNC)
452 invoker_name = virtual.value
454 for vfunc in parent.virtual_methods:
455 if vfunc.name == invoker_name:
457 vfunc.invoker = meth.name
460 print "warning: unmatched virtual invoker %r for method %r" % \
461 (invoker_name, meth.symbol)
463 def _parse_vfunc(self, parent, vfunc):
464 key = '%s::%s' % (parent.type_name, vfunc.name)
465 self._parse_callable(vfunc, self._blocks.get(key))
467 def _parse_field(self, parent, field):
468 if isinstance(field, Callback):
469 self._parse_callback(field)
471 def _parse_params(self, parent, params, block):
473 tag = self._get_tag(block, param.name)
474 self._parse_param(parent, param, tag)
476 def _parse_return(self, parent, return_, block):
477 tag = self._get_tag(block, TAG_RETURNS)
478 self._parse_param_ret_common(parent, return_, tag)
480 def _parse_param(self, parent, param, tag):
481 options = getattr(tag, 'options', {})
482 if isinstance(parent, Function):
483 scope = options.get(OPT_SCOPE)
485 param.scope = scope.one()
486 param.transfer = PARAM_TRANSFER_NONE
487 destroy = options.get(OPT_DESTROY)
489 param.destroy_index = parent.get_parameter_index(destroy.one())
490 self._fixup_param_destroy(parent, param)
491 closure = options.get(OPT_CLOSURE)
493 param.closure_index = parent.get_parameter_index(closure.one())
494 self._fixup_param_closure(parent, param)
495 if isinstance(parent, Callback):
496 if OPT_CLOSURE in options:
497 param.closure_index = parent.get_parameter_index(param.name)
498 self._fixup_param_closure(parent, param)
500 self._parse_param_ret_common(parent, param, tag)
502 def _fixup_param_destroy(self, parent, param):
503 for p in parent.parameters:
504 if p is not param and p.destroy_index == param.destroy_index:
505 p.destroy_index = None
507 def _fixup_param_closure(self, parent, param):
508 for p in parent.parameters:
509 if p is not param and p.closure_index == param.closure_index:
510 p.closure_index = None
512 def _parse_param_ret_common(self, parent, node, tag):
513 options = getattr(tag, 'options', {})
514 node.direction = self._extract_direction(node, options)
515 container_type = self._extract_container_type(
516 parent, node, options)
517 if container_type is not None:
518 node.type = container_type
519 if node.direction is None:
520 node.direction = self._guess_direction(node)
521 node.transfer = self._extract_transfer(parent, node, options)
522 if OPT_ALLOW_NONE in options:
523 node.allow_none = True
524 param_type = options.get(OPT_TYPE)
526 resolve = self._transformer.resolve_param_type
527 node.type.name = resolve(param_type.one())
529 assert node.transfer is not None
530 if tag is not None and tag.comment is not None:
531 node.doc = tag.comment
533 def _extract_direction(self, node, options):
534 if (OPT_INOUT in options or
535 OPT_INOUT_ALT in options):
536 direction = PARAM_DIRECTION_INOUT
537 elif OPT_OUT in options:
538 direction = PARAM_DIRECTION_OUT
539 elif OPT_IN in options:
540 direction = PARAM_DIRECTION_IN
542 direction = node.direction
545 def _guess_array(self, node):
546 ctype = node.type.ctype
549 if not ctype.endswith('*'):
551 if node.type.canonical in default_array_types:
555 def _extract_container_type(self, parent, node, options):
556 has_element_type = OPT_ELEMENT_TYPE in options
557 has_array = OPT_ARRAY in options
559 # FIXME: This is a hack :-(
560 if (not isinstance(node, Field) and
561 (not has_element_type and
562 (node.direction is None
563 or node.direction == PARAM_DIRECTION_IN))):
564 if self._guess_array(node):
568 container_type = self._parse_array(parent, node, options)
569 elif has_element_type:
570 container_type = self._parse_element_type(parent, node, options)
572 container_type = None
574 return container_type
576 def _parse_array(self, parent, node, options):
577 array_opt = options.get(OPT_ARRAY)
579 array_values = array_opt.all()
583 element_type = options.get(OPT_ELEMENT_TYPE)
584 if element_type is not None:
585 element_type_name = element_type.one()
587 element_type_name = node.type.name
589 container_type = Array(node.type.ctype,
591 if OPT_ARRAY_ZERO_TERMINATED in array_values:
592 container_type.zeroterminated = array_values.get(
593 OPT_ARRAY_ZERO_TERMINATED) == '1'
594 length = array_values.get(OPT_ARRAY_LENGTH)
595 if length is not None:
596 param_index = parent.get_parameter_index(length)
597 container_type.length_param_index = param_index
598 # For in parameters we're incorrectly deferring
599 # char/unsigned char to utf8 when a length annotation
601 if (isinstance(node, Parameter) and
602 node.type.name == 'utf8' and
603 self._guess_direction(node) == PARAM_DIRECTION_IN):
604 # FIXME: unsigned char/guchar should be uint8
605 container_type.element_type = 'int8'
606 container_type.size = array_values.get(OPT_ARRAY_FIXED_SIZE)
607 return container_type
609 def _parse_element_type(self, parent, node, options):
610 element_type_opt = options.get(OPT_ELEMENT_TYPE)
611 element_type = element_type_opt.flat()
612 if node.type.name in ['GLib.List', 'GLib.SList']:
613 assert len(element_type) == 1
614 etype = Type(element_type[0])
615 container_type = List(
618 self._transformer.resolve_param_type(etype))
619 elif node.type.name in ['GLib.HashTable']:
620 assert len(element_type) == 2
621 key_type = Type(element_type[0])
622 value_type = Type(element_type[1])
623 container_type = Map(
626 self._transformer.resolve_param_type(key_type),
627 self._transformer.resolve_param_type(value_type))
629 print 'FIXME: unhandled element-type container:', node
630 return container_type
632 def _extract_transfer(self, parent, node, options):
633 transfer_opt = options.get(OPT_TRANSFER)
634 if transfer_opt is None:
635 transfer = self._guess_transfer(node, options)
637 transfer = transfer_opt.one()
639 transfer = PARAM_TRANSFER_FULL
640 if transfer not in [PARAM_TRANSFER_NONE,
641 PARAM_TRANSFER_CONTAINER,
642 PARAM_TRANSFER_FULL]:
643 raise InvalidAnnotationError(
644 "transfer for %s of %r is invalid (%r), must be one of "
645 "none, container, full." % (node, parent.name, transfer))
648 def _parse_node_common(self, node, block):
649 self._parse_version(node, block)
650 self._parse_deprecated(node, block)
651 self._parse_attributes(node, block)
653 def _parse_version(self, node, block):
654 since_tag = self._get_tag(block, TAG_SINCE)
655 if since_tag is None:
657 node.version = since_tag.value
659 def _parse_deprecated(self, node, block):
660 deprecated_tag = self._get_tag(block, TAG_DEPRECATED)
661 if deprecated_tag is None:
663 value = deprecated_tag.value
665 version, desc = value.split(': ')
669 node.deprecated = desc
670 if version is not None:
671 node.deprecated_version = version
673 def _parse_attributes(self, node, block):
674 annos_tag = self._get_tag(block, TAG_ATTRIBUTES)
675 if annos_tag is None:
677 for key, value in annos_tag.options.iteritems():
678 node.attributes.append((key, value.one()))
680 def _parse_rename_to_func(self, node, block):
681 rename_to_tag = self._get_tag(block, TAG_RENAME_TO)
682 if rename_to_tag is None:
684 new_name = rename_to_tag.value
688 def shadowed_filter(n):
689 if isinstance(n, Function) and n.symbol == new_name:
694 self._namespace.remove_matching(shadowed_filter)
695 if len(shadowed) == 1:
696 # method override; use the same (stripped) name as the overloaded
698 # Note that 'g_timeout_add_full' may specify a new_name of
699 # 'g_timeout_add' but the *real* name desired is the stripped name
700 # of 'g_timeout_add' which is 'timeout_add' (for example).
701 node.name = shadowed[0].name
702 elif len(shadowed) == 0:
703 # literal rename, to force a particular prefix strip or whatever
704 # Example: the "nm-utils" package uses a "NM" prefix in most places
705 # but some functions have an "nm_utils_" prefix; the 'Rename To:'
706 # annotation in this case is used to strip the 'utils_' part off.
709 assert False # more than two shadowed methods? Shouldn't happen.
711 def _guess_direction(self, node):
713 return node.direction
716 is_pointer = '*' in node.type.ctype
718 if is_pointer and node.type.name in BASIC_GIR_TYPES:
719 return PARAM_DIRECTION_OUT
721 return PARAM_DIRECTION_IN
723 def _guess_transfer(self, node, options):
724 if node.transfer is not None:
727 if isinstance(node.type, Array):
728 return PARAM_TRANSFER_NONE
729 # Anything with 'const' gets none
730 if node.type.is_const:
731 return PARAM_TRANSFER_NONE
733 elif node.type.name in [TYPE_NONE, TYPE_ANY]:
734 return PARAM_TRANSFER_NONE
735 elif isinstance(node.type, Varargs):
736 return PARAM_TRANSFER_NONE
737 elif isinstance(node, Parameter):
738 if node.direction in [PARAM_DIRECTION_INOUT,
739 PARAM_DIRECTION_OUT]:
740 return PARAM_TRANSFER_FULL
741 # This one is a hack for compatibility; the transfer
742 # for string parameters really has no defined meaning.
743 elif node.type.canonical == 'utf8':
744 return PARAM_TRANSFER_FULL
746 return PARAM_TRANSFER_NONE
747 elif isinstance(node, Return):
748 if (node.type.canonical in BASIC_GIR_TYPES or
749 (node.type.canonical in [TYPE_NONE, TYPE_ANY] and
750 node.type.is_const)):
751 return PARAM_TRANSFER_NONE
753 return PARAM_TRANSFER_FULL
754 elif isinstance(node, Field):
755 return PARAM_TRANSFER_NONE
757 raise AssertionError(node)