ast.py

Go to the documentation of this file.

#!/usr/bin/env python
#
# Copyright 2007 Neal Norwitz
# Portions Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
 
"""Generate an Abstract Syntax Tree (AST) for C++."""
 
# FIXME:
#  * Tokens should never be exported, need to convert to Nodes
#    (return types, parameters, etc.)
#  * Handle static class data for templatized classes
#  * Handle casts (both C++ and C-style)
#  * Handle conditions and loops (if/else, switch, for, while/do)
#
# TODO much, much later:
#  * Handle #define
#  * exceptions
 
 
try:
  # Python 3.x
  import builtins
except ImportError:
  # Python 2.x
  import __builtin__ as builtins
 
import collections
import sys
import traceback
 
from cpp import keywords
from cpp import tokenize
from cpp import utils
 
 
if not hasattr(builtins, 'reversed'):
  # Support Python 2.3 and earlier.
  def reversed(seq):
    for i in range(len(seq)-1, -1, -1):
      yield seq[i]
 
if not hasattr(builtins, 'next'):
  # Support Python 2.5 and earlier.
  def next(obj):
    return obj.next()
 
 
VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3)
 
FUNCTION_NONE = 0x00
FUNCTION_CONST = 0x01
FUNCTION_VIRTUAL = 0x02
FUNCTION_PURE_VIRTUAL = 0x04
FUNCTION_CTOR = 0x08
FUNCTION_DTOR = 0x10
FUNCTION_ATTRIBUTE = 0x20
FUNCTION_UNKNOWN_ANNOTATION = 0x40
FUNCTION_THROW = 0x80
FUNCTION_OVERRIDE = 0x100
 
"""
These are currently unused.  Should really handle these properly at some point.
 
TYPE_MODIFIER_INLINE   = 0x010000
TYPE_MODIFIER_EXTERN   = 0x020000
TYPE_MODIFIER_STATIC   = 0x040000
TYPE_MODIFIER_CONST    = 0x080000
TYPE_MODIFIER_REGISTER = 0x100000
TYPE_MODIFIER_VOLATILE = 0x200000
TYPE_MODIFIER_MUTABLE  = 0x400000
 
TYPE_MODIFIER_MAP = {
    'inline': TYPE_MODIFIER_INLINE,
    'extern': TYPE_MODIFIER_EXTERN,
    'static': TYPE_MODIFIER_STATIC,
    'const': TYPE_MODIFIER_CONST,
    'register': TYPE_MODIFIER_REGISTER,
    'volatile': TYPE_MODIFIER_VOLATILE,
    'mutable': TYPE_MODIFIER_MUTABLE,
    }
"""
 
_INTERNAL_TOKEN = 'internal'
_NAMESPACE_POP = 'ns-pop'
 
 
# TODO(nnorwitz): use this as a singleton for templated_types, etc
# where we don't want to create a new empty dict each time.  It is also const.
class _NullDict(object):
  __contains__ = lambda self: False
  keys = values = items = iterkeys = itervalues = iteritems = lambda self: ()
 
 
# TODO(nnorwitz): move AST nodes into a separate module.
class Node(object):
  """Base AST node."""
 
  def __init__(self, start, end):
    self.start = start
    self.end = end
 
  def IsDeclaration(self):
    """Returns bool if this node is a declaration."""
    return False
 
  def IsDefinition(self):
    """Returns bool if this node is a definition."""
    return False
 
  def IsExportable(self):
    """Returns bool if this node exportable from a header file."""
    return False
 
  def Requires(self, node):
    """Does this AST node require the definition of the node passed in?"""
    return False
 
  def XXX__str__(self):
    return self._StringHelper(self.__class__.__name__, '')
 
  def _StringHelper(self, name, suffix):
    if not utils.DEBUG:
      return '%s(%s)' % (name, suffix)
    return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix)
 
  def __repr__(self):
    return str(self)
 
 
class Define(Node):
  def __init__(self, start, end, name, definition):
    Node.__init__(self, start, end)
    self.name = name
    self.definition = definition
 
  def __str__(self):
    value = '%s %s' % (self.name, self.definition)
    return self._StringHelper(self.__class__.__name__, value)
 
 
class Include(Node):
  def __init__(self, start, end, filename, system):
    Node.__init__(self, start, end)
    self.filename = filename
    self.system = system
 
  def __str__(self):
    fmt = '"%s"'
    if self.system:
      fmt = '<%s>'
    return self._StringHelper(self.__class__.__name__, fmt % self.filename)
 
 
class Goto(Node):
  def __init__(self, start, end, label):
    Node.__init__(self, start, end)
    self.label = label
 
  def __str__(self):
    return self._StringHelper(self.__class__.__name__, str(self.label))
 
 
class Expr(Node):
  def __init__(self, start, end, expr):
    Node.__init__(self, start, end)
    self.expr = expr
 
  def Requires(self, node):
    # TODO(nnorwitz): impl.
    return False
 
  def __str__(self):
    return self._StringHelper(self.__class__.__name__, str(self.expr))
 
 
class Return(Expr):
  pass
 
 
class Delete(Expr):
  pass
 
 
class Friend(Expr):
  def __init__(self, start, end, expr, namespace):
    Expr.__init__(self, start, end, expr)
    self.namespace = namespace[:]
 
 
class Using(Node):
  def __init__(self, start, end, names):
    Node.__init__(self, start, end)
    self.names = names
 
  def __str__(self):
    return self._StringHelper(self.__class__.__name__, str(self.names))
 
 
class Parameter(Node):
  def __init__(self, start, end, name, parameter_type, default):
    Node.__init__(self, start, end)
    self.name = name
    self.type = parameter_type
    self.default = default
 
  def Requires(self, node):
    # TODO(nnorwitz): handle namespaces, etc.
    return self.type.name == node.name
 
  def __str__(self):
    name = str(self.type)
    suffix = '%s %s' % (name, self.name)
    if self.default:
      suffix += ' = ' + ''.join([d.name for d in self.default])
    return self._StringHelper(self.__class__.__name__, suffix)
 
 
class _GenericDeclaration(Node):
  def __init__(self, start, end, name, namespace):
    Node.__init__(self, start, end)
    self.name = name
    self.namespace = namespace[:]
 
  def FullName(self):
    prefix = ''
    if self.namespace and self.namespace[-1]:
      prefix = '::'.join(self.namespace) + '::'
    return prefix + self.name
 
  def _TypeStringHelper(self, suffix):
    if self.namespace:
      names = [n or '<anonymous>' for n in self.namespace]
      suffix += ' in ' + '::'.join(names)
    return self._StringHelper(self.__class__.__name__, suffix)
 
 
# TODO(nnorwitz): merge with Parameter in some way?
class VariableDeclaration(_GenericDeclaration):
  def __init__(self, start, end, name, var_type, initial_value, namespace):
    _GenericDeclaration.__init__(self, start, end, name, namespace)
    self.type = var_type
    self.initial_value = initial_value
 
  def Requires(self, node):
    # TODO(nnorwitz): handle namespaces, etc.
    return self.type.name == node.name
 
  def ToString(self):
    """Return a string that tries to reconstitute the variable decl."""
    suffix = '%s %s' % (self.type, self.name)
    if self.initial_value:
      suffix += ' = ' + self.initial_value
    return suffix
 
  def __str__(self):
    return self._StringHelper(self.__class__.__name__, self.ToString())
 
 
class Typedef(_GenericDeclaration):
  def __init__(self, start, end, name, alias, namespace):
    _GenericDeclaration.__init__(self, start, end, name, namespace)
    self.alias = alias
 
  def IsDefinition(self):
    return True
 
  def IsExportable(self):
    return True
 
  def Requires(self, node):
    # TODO(nnorwitz): handle namespaces, etc.
    name = node.name
    for token in self.alias:
      if token is not None and name == token.name:
        return True
    return False
 
  def __str__(self):
    suffix = '%s, %s' % (self.name, self.alias)
    return self._TypeStringHelper(suffix)
 
 
class _NestedType(_GenericDeclaration):
  def __init__(self, start, end, name, fields, namespace):
    _GenericDeclaration.__init__(self, start, end, name, namespace)
    self.fields = fields
 
  def IsDefinition(self):
    return True
 
  def IsExportable(self):
    return True
 
  def __str__(self):
    suffix = '%s, {%s}' % (self.name, self.fields)
    return self._TypeStringHelper(suffix)
 
 
class Union(_NestedType):
  pass
 
 
class Enum(_NestedType):
  pass
 
 
class Class(_GenericDeclaration):
  def __init__(self, start, end, name, bases, templated_types, body, namespace):
    _GenericDeclaration.__init__(self, start, end, name, namespace)
    self.bases = bases
    self.body = body
    self.templated_types = templated_types
 
  def IsDeclaration(self):
    return self.bases is None and self.body is None
 
  def IsDefinition(self):
    return not self.IsDeclarationIsDeclaration()
 
  def IsExportable(self):
    return not self.IsDeclarationIsDeclaration()
 
  def Requires(self, node):
    # TODO(nnorwitz): handle namespaces, etc.
    if self.bases:
      for token_list in self.bases:
        # TODO(nnorwitz): bases are tokens, do name comparison.
        for token in token_list:
          if token.name == node.name:
            return True
    # TODO(nnorwitz): search in body too.
    return False
 
  def __str__(self):
    name = self.name
    if self.templated_types:
      name += '<%s>' % self.templated_types
    suffix = '%s, %s, %s' % (name, self.bases, self.body)
    return self._TypeStringHelper(suffix)
 
 
class Struct(Class):
  pass
 
 
class Function(_GenericDeclaration):
  def __init__(self, start, end, name, return_type, parameters,
               modifiers, templated_types, body, namespace):
    _GenericDeclaration.__init__(self, start, end, name, namespace)
    converter = TypeConverter(namespace)
    self.return_type = converter.CreateReturnType(return_type)
    self.parameters = converter.ToParameters(parameters)
    self.modifiers = modifiers
    self.body = body
    self.templated_types = templated_types
 
  def IsDeclaration(self):
    return self.body is None
 
  def IsDefinition(self):
    return self.body is not None
 
  def IsExportable(self):
    if self.return_type and 'static' in self.return_type.modifiers:
      return False
    return None not in self.namespace
 
  def Requires(self, node):
    if self.parameters:
      # TODO(nnorwitz): parameters are tokens, do name comparison.
      for p in self.parameters:
        if p.name == node.name:
          return True
    # TODO(nnorwitz): search in body too.
    return False
 
  def __str__(self):
    # TODO(nnorwitz): add templated_types.
    suffix = ('%s %s(%s), 0x%02x, %s' %
              (self.return_type, self.name, self.parameters,
               self.modifiers, self.body))
    return self._TypeStringHelper(suffix)
 
 
class Method(Function):
  def __init__(self, start, end, name, in_class, return_type, parameters,
               modifiers, templated_types, body, namespace):
    Function.__init__(self, start, end, name, return_type, parameters,
                      modifiers, templated_types, body, namespace)
    # TODO(nnorwitz): in_class could also be a namespace which can
    # mess up finding functions properly.
    self.in_class = in_class
 
 
class Type(_GenericDeclaration):
  """Type used for any variable (eg class, primitive, struct, etc)."""
 
  def __init__(self, start, end, name, templated_types, modifiers,
               reference, pointer, array):
    """
        Args:
          name: str name of main type
          templated_types: [Class (Type?)] template type info between <>
          modifiers: [str] type modifiers (keywords) eg, const, mutable, etc.
          reference, pointer, array: bools
        """
    _GenericDeclaration.__init__(self, start, end, name, [])
    self.templated_types = templated_types
    if not name and modifiers:
      self.namename = modifiers.pop()
    self.modifiers = modifiers
    self.reference = reference
    self.pointer = pointer
    self.array = array
 
  def __str__(self):
    prefix = ''
    if self.modifiers:
      prefix = ' '.join(self.modifiers) + ' '
    name = str(self.namename)
    if self.templated_types:
      name += '<%s>' % self.templated_types
    suffix = prefix + name
    if self.reference:
      suffix += '&'
    if self.pointer:
      suffix += '*'
    if self.array:
      suffix += '[]'
    return self._TypeStringHelper(suffix)
 
  # By definition, Is* are always False.  A Type can only exist in
  # some sort of variable declaration, parameter, or return value.
  def IsDeclaration(self):
    return False
 
  def IsDefinition(self):
    return False
 
  def IsExportable(self):
    return False
 
 
class TypeConverter(object):
 
  def __init__(self, namespace_stack):
    self.namespace_stack = namespace_stack
 
  def _GetTemplateEnd(self, tokens, start):
    count = 1
    end = start
    while 1:
      token = tokens[end]
      end += 1
      if token.name == '<':
        count += 1
      elif token.name == '>':
        count -= 1
        if count == 0:
          break
    return tokens[start:end-1], end
 
  def ToType(self, tokens):
    """Convert [Token,...] to [Class(...), ] useful for base classes.
        For example, code like class Foo : public Bar<x, y> { ... };
        the "Bar<x, y>" portion gets converted to an AST.
 
        Returns:
          [Class(...), ...]
        """
    result = []
    name_tokens = []
    reference = pointer = array = False
 
    def AddType(templated_types):
      # Partition tokens into name and modifier tokens.
      names = []
      modifiers = []
      for t in name_tokens:
        if keywords.IsKeyword(t.name):
          modifiers.append(t.name)
        else:
          names.append(t.name)
      name = ''.join(names)
      if name_tokens:
        result.append(Type(name_tokens[0].start, name_tokens[-1].end,
                           name, templated_types, modifiers,
                           reference, pointer, array))
      del name_tokens[:]
 
    i = 0
    end = len(tokens)
    while i < end:
      token = tokens[i]
      if token.name == '<':
        new_tokens, new_end = self._GetTemplateEnd(tokens, i+1)
        AddType(self.ToType(new_tokens))
        # If there is a comma after the template, we need to consume
        # that here otherwise it becomes part of the name.
        i = new_end
        reference = pointer = array = False
      elif token.name == ',':
        AddType([])
        reference = pointer = array = False
      elif token.name == '*':
        pointer = True
      elif token.name == '&':
        reference = True
      elif token.name == '[':
        pointer = True
      elif token.name == ']':
        pass
      else:
        name_tokens.append(token)
      i += 1
 
    if name_tokens:
      # No '<' in the tokens, just a simple name and no template.
      AddType([])
    return result
 
  def DeclarationToParts(self, parts, needs_name_removed):
    name = None
    default = []
    if needs_name_removed:
      # Handle default (initial) values properly.
      for i, t in enumerate(parts):
        if t.name == '=':
          default = parts[i+1:]
          name = parts[i-1].name
          if name == ']' and parts[i-2].name == '[':
            name = parts[i-3].name
            i -= 1
          parts = parts[:i-1]
          break
      else:
        if parts[-1].token_type == tokenize.NAME:
          name = parts.pop().name
        else:
          # TODO(nnorwitz): this is a hack that happens for code like
          # Register(Foo<T>); where it thinks this is a function call
          # but it's actually a declaration.
          name = '???'
    modifiers = []
    type_name = []
    other_tokens = []
    templated_types = []
    i = 0
    end = len(parts)
    while i < end:
      p = parts[i]
      if keywords.IsKeyword(p.name):
        modifiers.append(p.name)
      elif p.name == '<':
        templated_tokens, new_end = self._GetTemplateEnd(parts, i+1)
        templated_types = self.ToType(templated_tokens)
        i = new_end - 1
        # Don't add a spurious :: to data members being initialized.
        next_index = i + 1
        if next_index < end and parts[next_index].name == '::':
          i += 1
      elif p.name in ('[', ']', '='):
        # These are handled elsewhere.
        other_tokens.append(p)
      elif p.name not in ('*', '&', '>'):
        # Ensure that names have a space between them.
        if (type_name and type_name[-1].token_type == tokenize.NAME and
                p.token_type == tokenize.NAME):
          type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
        type_name.append(p)
      else:
        other_tokens.append(p)
      i += 1
    type_name = ''.join([t.name for t in type_name])
    return name, type_name, templated_types, modifiers, default, other_tokens
 
  def ToParameters(self, tokens):
    if not tokens:
      return []
 
    result = []
    name = type_name = ''
    type_modifiers = []
    pointer = reference = array = False
    first_token = None
    default = []
 
    def AddParameter(end):
      if default:
        del default[0]  # Remove flag.
      parts = self.DeclarationToParts(type_modifiers, True)
      (name, type_name, templated_types, modifiers,
       unused_default, unused_other_tokens) = parts
      parameter_type = Type(first_token.start, first_token.end,
                            type_name, templated_types, modifiers,
                            reference, pointer, array)
      p = Parameter(first_token.start, end, name,
                    parameter_type, default)
      result.append(p)
 
    template_count = 0
    brace_count = 0
    for s in tokens:
      if not first_token:
        first_token = s
 
      # Check for braces before templates, as we can have unmatched '<>'
      # inside default arguments.
      if s.name == '{':
        brace_count += 1
      elif s.name == '}':
        brace_count -= 1
      if brace_count > 0:
        type_modifiers.append(s)
        continue
 
      if s.name == '<':
        template_count += 1
      elif s.name == '>':
        template_count -= 1
      if template_count > 0:
        type_modifiers.append(s)
        continue
 
      if s.name == ',':
        AddParameter(s.start)
        name = type_name = ''
        type_modifiers = []
        pointer = reference = array = False
        first_token = None
        default = []
      elif s.name == '*':
        pointer = True
      elif s.name == '&':
        reference = True
      elif s.name == '[':
        array = True
      elif s.name == ']':
        pass  # Just don't add to type_modifiers.
      elif s.name == '=':
        # Got a default value.  Add any value (None) as a flag.
        default.append(None)
      elif default:
        default.append(s)
      else:
        type_modifiers.append(s)
    AddParameter(tokens[-1].end)
    return result
 
  def CreateReturnType(self, return_type_seq):
    if not return_type_seq:
      return None
    start = return_type_seq[0].start
    end = return_type_seq[-1].end
    _, name, templated_types, modifiers, default, other_tokens = \
        self.DeclarationToParts(return_type_seq, False)
    names = [n.name for n in other_tokens]
    reference = '&' in names
    pointer = '*' in names
    array = '[' in names
    return Type(start, end, name, templated_types, modifiers,
                reference, pointer, array)
 
  def GetTemplateIndices(self, names):
    # names is a list of strings.
    start = names.index('<')
    end = len(names) - 1
    while end > 0:
      if names[end] == '>':
        break
      end -= 1
    return start, end+1
 
class AstBuilder(object):
  def __init__(self, token_stream, filename, in_class='', visibility=None,
               namespace_stack=[]):
    self.tokens = token_stream
    self.filename = filename
    # TODO(nnorwitz): use a better data structure (deque) for the queue.
    # Switching directions of the "queue" improved perf by about 25%.
    # Using a deque should be even better since we access from both sides.
    self.token_queue = []
    self.namespace_stack = namespace_stack[:]
    self.in_class = in_class
    if in_class is None:
      self.in_class_name_only = None
    else:
      self.in_class_name_only = in_class.split('::')[-1]
    self.visibility = visibility
    self.in_function = False
    self.current_token = None
    # Keep the state whether we are currently handling a typedef or not.
    self._handling_typedef = False
 
    self.converter = TypeConverter(self.namespace_stack)
 
  def HandleError(self, msg, token):
    printable_queue = list(reversed(self.token_queue[-20:]))
    sys.stderr.write('Got %s in %s @ %s %s\n' %
                     (msg, self.filename, token, printable_queue))
 
  def Generate(self):
    while 1:
      token = self._GetNextToken()
      if not token:
        break
 
      # Get the next token.
      self.current_token = token
 
      # Dispatch on the next token type.
      if token.token_type == _INTERNAL_TOKEN:
        if token.name == _NAMESPACE_POP:
          self.namespace_stack.pop()
        continue
 
      try:
        result = self._GenerateOne(token)
        if result is not None:
          yield result
      except:
        self.HandleError('exception', token)
        raise
 
  def _CreateVariable(self, pos_token, name, type_name, type_modifiers,
                      ref_pointer_name_seq, templated_types, value=None):
    reference = '&' in ref_pointer_name_seq
    pointer = '*' in ref_pointer_name_seq
    array = '[' in ref_pointer_name_seq
    var_type = Type(pos_token.start, pos_token.end, type_name,
                    templated_types, type_modifiers,
                    reference, pointer, array)
    return VariableDeclaration(pos_token.start, pos_token.end,
                               name, var_type, value, self.namespace_stack)
 
  def _GenerateOne(self, token):
    if token.token_type == tokenize.NAME:
      if (keywords.IsKeyword(token.name) and
          not keywords.IsBuiltinType(token.name)):
        if token.name == 'enum':
          # Pop the next token and only put it back if it's not
          # 'class'.  This allows us to support the two-token
          # 'enum class' keyword as if it were simply 'enum'.
          next = self._GetNextToken()
          if next.name != 'class':
            self._AddBackToken(next)
 
        method = getattr(self, 'handle_' + token.name)
        return method()
      elif token.name == self.in_class_name_only:
        # The token name is the same as the class, must be a ctor if
        # there is a paren.  Otherwise, it's the return type.
        # Peek ahead to get the next token to figure out which.
        next = self._GetNextToken()
        self._AddBackToken(next)
        if next.token_type == tokenize.SYNTAX and next.name == '(':
          return self._GetMethod([token], FUNCTION_CTOR, None, True)
        # Fall through--handle like any other method.
 
      # Handle data or function declaration/definition.
      syntax = tokenize.SYNTAX
      temp_tokens, last_token = \
          self._GetVarTokensUpToIgnoringTemplates(syntax,
                                                  '(', ';', '{', '[')
      temp_tokens.insert(0, token)
      if last_token.name == '(':
        # If there is an assignment before the paren,
        # this is an expression, not a method.
        expr = bool([e for e in temp_tokens if e.name == '='])
        if expr:
          new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
          temp_tokens.append(last_token)
          temp_tokens.extend(new_temp)
          last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
 
      if last_token.name == '[':
        # Handle array, this isn't a method, unless it's an operator.
        # TODO(nnorwitz): keep the size somewhere.
        # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
        temp_tokens.append(last_token)
        if temp_tokens[-2].name == 'operator':
          temp_tokens.append(self._GetNextToken())
        else:
          temp_tokens2, last_token = \
              self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
          temp_tokens.extend(temp_tokens2)
 
      if last_token.name == ';':
        # Handle data, this isn't a method.
        parts = self.converter.DeclarationToParts(temp_tokens, True)
        (name, type_name, templated_types, modifiers, default,
         unused_other_tokens) = parts
 
        t0 = temp_tokens[0]
        names = [t.name for t in temp_tokens]
        if templated_types:
          start, end = self.converter.GetTemplateIndices(names)
          names = names[:start] + names[end:]
        default = ''.join([t.name for t in default])
        return self._CreateVariable(t0, name, type_name, modifiers,
                                    names, templated_types, default)
      if last_token.name == '{':
        self._AddBackTokens(temp_tokens[1:])
        self._AddBackToken(last_token)
        method_name = temp_tokens[0].name
        method = getattr(self, 'handle_' + method_name, None)
        if not method:
          # Must be declaring a variable.
          # TODO(nnorwitz): handle the declaration.
          return None
        return method()
      return self._GetMethod(temp_tokens, 0, None, False)
    elif token.token_type == tokenize.SYNTAX:
      if token.name == '~' and self.in_class:
        # Must be a dtor (probably not in method body).
        token = self._GetNextToken()
        # self.in_class can contain A::Name, but the dtor will only
        # be Name.  Make sure to compare against the right value.
        if (token.token_type == tokenize.NAME and
                token.name == self.in_class_name_only):
          return self._GetMethod([token], FUNCTION_DTOR, None, True)
      # TODO(nnorwitz): handle a lot more syntax.
    elif token.token_type == tokenize.PREPROCESSOR:
      # TODO(nnorwitz): handle more preprocessor directives.
      # token starts with a #, so remove it and strip whitespace.
      name = token.name[1:].lstrip()
      if name.startswith('include'):
        # Remove "include".
        name = name[7:].strip()
        assert name
        # Handle #include <newline> "header-on-second-line.h".
        if name.startswith('\\'):
          name = name[1:].strip()
        assert name[0] in '<"', token
        assert name[-1] in '>"', token
        system = name[0] == '<'
        filename = name[1:-1]
        return Include(token.start, token.end, filename, system)
      if name.startswith('define'):
        # Remove "define".
        name = name[6:].strip()
        assert name
        value = ''
        for i, c in enumerate(name):
          if c.isspace():
            value = name[i:].lstrip()
            name = name[:i]
            break
        return Define(token.start, token.end, name, value)
      if name.startswith('if') and name[2:3].isspace():
        condition = name[3:].strip()
        if condition.startswith('0') or condition.startswith('(0)'):
          self._SkipIf0Blocks()
    return None
 
  def _GetTokensUpTo(self, expected_token_type, expected_token):
    return self._GetVarTokensUpTo(expected_token_type, expected_token)[0]
 
  def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens):
    last_token = self._GetNextToken()
    tokens = []
    while (last_token.token_type != expected_token_type or
           last_token.name not in expected_tokens):
      tokens.append(last_token)
      last_token = self._GetNextToken()
    return tokens, last_token
 
  # Same as _GetVarTokensUpTo, but skips over '<...>' which could contain an
  # expected token.
  def _GetVarTokensUpToIgnoringTemplates(self, expected_token_type,
                                         *expected_tokens):
    last_token = self._GetNextToken()
    tokens = []
    nesting = 0
    while (nesting > 0 or
           last_token.token_type != expected_token_type or
           last_token.name not in expected_tokens):
      tokens.append(last_token)
      last_token = self._GetNextToken()
      if last_token.name == '<':
        nesting += 1
      elif last_token.name == '>':
        nesting -= 1
    return tokens, last_token
 
  # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necessary.
  def _IgnoreUpTo(self, token_type, token):
    unused_tokens = self._GetTokensUpTo(token_type, token)
 
  def _SkipIf0Blocks(self):
    count = 1
    while 1:
      token = self._GetNextToken()
      if token.token_type != tokenize.PREPROCESSOR:
        continue
 
      name = token.name[1:].lstrip()
      if name.startswith('endif'):
        count -= 1
        if count == 0:
          break
      elif name.startswith('if'):
        count += 1
 
  def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None):
    if GetNextToken is None:
      GetNextToken = self._GetNextToken
    # Assumes the current token is open_paren and we will consume
    # and return up to the close_paren.
    count = 1
    token = GetNextToken()
    while 1:
      if token.token_type == tokenize.SYNTAX:
        if token.name == open_paren:
          count += 1
        elif token.name == close_paren:
          count -= 1
          if count == 0:
            break
      yield token
      token = GetNextToken()
    yield token
 
  def _GetParameters(self):
    return self._GetMatchingChar('(', ')')
 
  def GetScope(self):
    return self._GetMatchingChar('{', '}')
 
  def _GetNextToken(self):
    if self.token_queue:
      return self.token_queue.pop()
    try:
      return next(self.tokens)
    except StopIteration:
      return
 
  def _AddBackToken(self, token):
    if token.whence == tokenize.WHENCE_STREAM:
      token.whence = tokenize.WHENCE_QUEUE
      self.token_queue.insert(0, token)
    else:
      assert token.whence == tokenize.WHENCE_QUEUE, token
      self.token_queue.append(token)
 
  def _AddBackTokens(self, tokens):
    if tokens:
      if tokens[-1].whence == tokenize.WHENCE_STREAM:
        for token in tokens:
          token.whence = tokenize.WHENCE_QUEUE
        self.token_queue[:0] = reversed(tokens)
      else:
        assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
        self.token_queue.extend(reversed(tokens))
 
  def GetName(self, seq=None):
    """Returns ([tokens], next_token_info)."""
    GetNextToken = self._GetNextToken
    if seq is not None:
      it = iter(seq)
      GetNextToken = lambda: next(it)
    next_token = GetNextToken()
    tokens = []
    last_token_was_name = False
    while (next_token.token_type == tokenize.NAME or
           (next_token.token_type == tokenize.SYNTAX and
            next_token.name in ('::', '<'))):
      # Two NAMEs in a row means the identifier should terminate.
      # It's probably some sort of variable declaration.
      if last_token_was_name and next_token.token_type == tokenize.NAME:
        break
      last_token_was_name = next_token.token_type == tokenize.NAME
      tokens.append(next_token)
      # Handle templated names.
      if next_token.name == '<':
        tokens.extend(self._GetMatchingChar('<', '>', GetNextToken))
        last_token_was_name = True
      next_token = GetNextToken()
    return tokens, next_token
 
  def GetMethod(self, modifiers, templated_types):
    return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
    assert len(return_type_and_name) >= 1
    return self._GetMethod(return_type_and_name, modifiers, templated_types,
                           False)
 
  def _GetMethod(self, return_type_and_name, modifiers, templated_types,
                 get_paren):
    template_portion = None
    if get_paren:
      token = self._GetNextToken()
      assert token.token_type == tokenize.SYNTAX, token
      if token.name == '<':
        # Handle templatized dtors.
        template_portion = [token]
        template_portion.extend(self._GetMatchingChar('<', '>'))
        token = self._GetNextToken()
      assert token.token_type == tokenize.SYNTAX, token
      assert token.name == '(', token
 
    name = return_type_and_name.pop()
    # Handle templatized ctors.
    if name.name == '>':
      index = 1
      while return_type_and_name[index].name != '<':
        index += 1
      template_portion = return_type_and_name[index:] + [name]
      del return_type_and_name[index:]
      name = return_type_and_name.pop()
    elif name.name == ']':
      rt = return_type_and_name
      assert rt[-1].name == '[', return_type_and_name
      assert rt[-2].name == 'operator', return_type_and_name
      name_seq = return_type_and_name[-2:]
      del return_type_and_name[-2:]
      name = tokenize.Token(tokenize.NAME, 'operator[]',
                            name_seq[0].start, name.end)
      # Get the open paren so _GetParameters() below works.
      unused_open_paren = self._GetNextToken()
 
    # TODO(nnorwitz): store template_portion.
    return_type = return_type_and_name
    indices = name
    if return_type:
      indices = return_type[0]
 
    # Force ctor for templatized ctors.
    if name.name == self.in_class and not modifiers:
      modifiers |= FUNCTION_CTOR
    parameters = list(self._GetParameters())
    del parameters[-1]              # Remove trailing ')'.
 
    # Handling operator() is especially weird.
    if name.name == 'operator' and not parameters:
      token = self._GetNextToken()
      assert token.name == '(', token
      parameters = list(self._GetParameters())
      del parameters[-1]          # Remove trailing ')'.
 
    token = self._GetNextToken()
    while token.token_type == tokenize.NAME:
      modifier_token = token
      token = self._GetNextToken()
      if modifier_token.name == 'const':
        modifiers |= FUNCTION_CONST
      elif modifier_token.name == '__attribute__':
        # TODO(nnorwitz): handle more __attribute__ details.
        modifiers |= FUNCTION_ATTRIBUTE
        assert token.name == '(', token
        # Consume everything between the (parens).
        unused_tokens = list(self._GetMatchingChar('(', ')'))
        token = self._GetNextToken()
      elif modifier_token.name == 'throw':
        modifiers |= FUNCTION_THROW
        assert token.name == '(', token
        # Consume everything between the (parens).
        unused_tokens = list(self._GetMatchingChar('(', ')'))
        token = self._GetNextToken()
      elif modifier_token.name == 'override':
        modifiers |= FUNCTION_OVERRIDE
      elif modifier_token.name == modifier_token.name.upper():
        # HACK(nnorwitz):  assume that all upper-case names
        # are some macro we aren't expanding.
        modifiers |= FUNCTION_UNKNOWN_ANNOTATION
      else:
        self.HandleError('unexpected token', modifier_token)
 
    assert token.token_type == tokenize.SYNTAX, token
    # Handle ctor initializers.
    if token.name == ':':
      # TODO(nnorwitz): anything else to handle for initializer list?
      while token.name != ';' and token.name != '{':
        token = self._GetNextToken()
 
    # Handle pointer to functions that are really data but look
    # like method declarations.
    if token.name == '(':
      if parameters[0].name == '*':
        # name contains the return type.
        name = parameters.pop()
        # parameters contains the name of the data.
        modifiers = [p.name for p in parameters]
        # Already at the ( to open the parameter list.
        function_parameters = list(self._GetMatchingChar('(', ')'))
        del function_parameters[-1]  # Remove trailing ')'.
        # TODO(nnorwitz): store the function_parameters.
        token = self._GetNextToken()
        assert token.token_type == tokenize.SYNTAX, token
        assert token.name == ';', token
        return self._CreateVariable(indices, name.name, indices.name,
                                    modifiers, '', None)
      # At this point, we got something like:
      #  return_type (type::*name_)(params);
      # This is a data member called name_ that is a function pointer.
      # With this code: void (sq_type::*field_)(string&);
      # We get: name=void return_type=[] parameters=sq_type ... field_
      # TODO(nnorwitz): is return_type always empty?
      # TODO(nnorwitz): this isn't even close to being correct.
      # Just put in something so we don't crash and can move on.
      real_name = parameters[-1]
      modifiers = [p.name for p in self._GetParameters()]
      del modifiers[-1]           # Remove trailing ')'.
      return self._CreateVariable(indices, real_name.name, indices.name,
                                  modifiers, '', None)
 
    if token.name == '{':
      body = list(self.GetScope())
      del body[-1]                # Remove trailing '}'.
    else:
      body = None
      if token.name == '=':
        token = self._GetNextToken()
 
        if token.name == 'default' or token.name == 'delete':
          # Ignore explicitly defaulted and deleted special members
          # in C++11.
          token = self._GetNextToken()
        else:
          # Handle pure-virtual declarations.
          assert token.token_type == tokenize.CONSTANT, token
          assert token.name == '0', token
          modifiers |= FUNCTION_PURE_VIRTUAL
          token = self._GetNextToken()
 
      if token.name == '[':
        # TODO(nnorwitz): store tokens and improve parsing.
        # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N];
        tokens = list(self._GetMatchingChar('[', ']'))
        token = self._GetNextToken()
 
      assert token.name == ';', (token, return_type_and_name, parameters)
 
    # Looks like we got a method, not a function.
    if len(return_type) > 2 and return_type[-1].name == '::':
      return_type, in_class = \
          self._GetReturnTypeAndClassName(return_type)
      return Method(indices.start, indices.end, name.name, in_class,
                    return_type, parameters, modifiers, templated_types,
                    body, self.namespace_stack)
    return Function(indices.start, indices.end, name.name, return_type,
                    parameters, modifiers, templated_types, body,
                    self.namespace_stack)
 
  def _GetReturnTypeAndClassName(self, token_seq):
    # Splitting the return type from the class name in a method
    # can be tricky.  For example, Return::Type::Is::Hard::To::Find().
    # Where is the return type and where is the class name?
    # The heuristic used is to pull the last name as the class name.
    # This includes all the templated type info.
    # TODO(nnorwitz): if there is only One name like in the
    # example above, punt and assume the last bit is the class name.
 
    # Ignore a :: prefix, if exists so we can find the first real name.
    i = 0
    if token_seq[0].name == '::':
      i = 1
    # Ignore a :: suffix, if exists.
    end = len(token_seq) - 1
    if token_seq[end-1].name == '::':
      end -= 1
 
    # Make a copy of the sequence so we can append a sentinel
    # value. This is required for GetName will has to have some
    # terminating condition beyond the last name.
    seq_copy = token_seq[i:end]
    seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
    names = []
    while i < end:
      # Iterate through the sequence parsing out each name.
      new_name, next = self.GetName(seq_copy[i:])
      assert new_name, 'Got empty new_name, next=%s' % next
      # We got a pointer or ref.  Add it to the name.
      if next and next.token_type == tokenize.SYNTAX:
        new_name.append(next)
      names.append(new_name)
      i += len(new_name)
 
    # Now that we have the names, it's time to undo what we did.
 
    # Remove the sentinel value.
    names[-1].pop()
    # Flatten the token sequence for the return type.
    return_type = [e for seq in names[:-1] for e in seq]
    # The class name is the last name.
    class_name = names[-1]
    return return_type, class_name
 
  def handle_bool(self):
    pass
 
  def handle_char(self):
    pass
 
  def handle_int(self):
    pass
 
  def handle_long(self):
    pass
 
  def handle_short(self):
    pass
 
  def handle_double(self):
    pass
 
  def handle_float(self):
    pass
 
  def handle_void(self):
    pass
 
  def handle_wchar_t(self):
    pass
 
  def handle_unsigned(self):
    pass
 
  def handle_signed(self):
    pass
 
  def _GetNestedType(self, ctor):
    name = None
    name_tokens, token = self.GetName()
    if name_tokens:
      name = ''.join([t.name for t in name_tokens])
 
    # Handle forward declarations.
    if token.token_type == tokenize.SYNTAX and token.name == ';':
      return ctor(token.start, token.end, name, None,
                  self.namespace_stack)
 
    if token.token_type == tokenize.NAME and self._handling_typedef:
      self._AddBackToken(token)
      return ctor(token.start, token.end, name, None,
                  self.namespace_stack)
 
    # Must be the type declaration.
    fields = list(self._GetMatchingChar('{', '}'))
    del fields[-1]                  # Remove trailing '}'.
    if token.token_type == tokenize.SYNTAX and token.name == '{':
      next = self._GetNextToken()
      new_type = ctor(token.start, token.end, name, fields,
                      self.namespace_stack)
      # A name means this is an anonymous type and the name
      # is the variable declaration.
      if next.token_type != tokenize.NAME:
        return new_type
      name = new_type
      token = next
 
    # Must be variable declaration using the type prefixed with keyword.
    assert token.token_type == tokenize.NAME, token
    return self._CreateVariable(token, token.name, name, [], '', None)
 
  def handle_struct(self):
    # Special case the handling typedef/aliasing of structs here.
    # It would be a pain to handle in the class code.
    name_tokens, var_token = self.GetName()
    if name_tokens:
      next_token = self._GetNextToken()
      is_syntax = (var_token.token_type == tokenize.SYNTAX and
                   var_token.name[0] in '*&')
      is_variable = (var_token.token_type == tokenize.NAME and
                     next_token.name == ';')
      variable = var_token
      if is_syntax and not is_variable:
        variable = next_token
        temp = self._GetNextToken()
        if temp.token_type == tokenize.SYNTAX and temp.name == '(':
          # Handle methods declared to return a struct.
          t0 = name_tokens[0]
          struct = tokenize.Token(tokenize.NAME, 'struct',
                                  t0.start-7, t0.start-2)
          type_and_name = [struct]
          type_and_name.extend(name_tokens)
          type_and_name.extend((var_token, next_token))
          return self._GetMethod(type_and_name, 0, None, False)
        assert temp.name == ';', (temp, name_tokens, var_token)
      if is_syntax or (is_variable and not self._handling_typedef):
        modifiers = ['struct']
        type_name = ''.join([t.name for t in name_tokens])
        position = name_tokens[0]
        return self._CreateVariable(position, variable.name, type_name,
                                    modifiers, var_token.name, None)
      name_tokens.extend((var_token, next_token))
      self._AddBackTokens(name_tokens)
    else:
      self._AddBackToken(var_token)
    return self._GetClass(Struct, VISIBILITY_PUBLIC, None)
 
  def handle_union(self):
    return self._GetNestedType(Union)
 
  def handle_enum(self):
    return self._GetNestedType(Enum)
 
  def handle_auto(self):
    # TODO(nnorwitz): warn about using auto?  Probably not since it
    # will be reclaimed and useful for C++0x.
    pass
 
  def handle_register(self):
    pass
 
  def handle_const(self):
    pass
 
  def handle_inline(self):
    pass
 
  def handle_extern(self):
    pass
 
  def handle_static(self):
    pass
 
  def handle_virtual(self):
    # What follows must be a method.
    token = token2 = self._GetNextToken()
    if token.name == 'inline':
      # HACK(nnorwitz): handle inline dtors by ignoring 'inline'.
      token2 = self._GetNextToken()
    if token2.token_type == tokenize.SYNTAX and token2.name == '~':
      return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None)
    assert token.token_type == tokenize.NAME or token.name == '::', token
    return_type_and_name, _ = self._GetVarTokensUpToIgnoringTemplates(
        tokenize.SYNTAX, '(')  # )
    return_type_and_name.insert(0, token)
    if token2 is not token:
      return_type_and_name.insert(1, token2)
    return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL,
                           None, False)
 
  def handle_volatile(self):
    pass
 
  def handle_mutable(self):
    pass
 
  def handle_public(self):
    assert self.in_class
    self.visibility = VISIBILITY_PUBLIC
 
  def handle_protected(self):
    assert self.in_class
    self.visibility = VISIBILITY_PROTECTED
 
  def handle_private(self):
    assert self.in_class
    self.visibility = VISIBILITY_PRIVATE
 
  def handle_friend(self):
    tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
    assert tokens
    t0 = tokens[0]
    return Friend(t0.start, t0.end, tokens, self.namespace_stack)
 
  def handle_static_cast(self):
    pass
 
  def handle_const_cast(self):
    pass
 
  def handle_dynamic_cast(self):
    pass
 
  def handle_reinterpret_cast(self):
    pass
 
  def handle_new(self):
    pass
 
  def handle_delete(self):
    tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
    assert tokens
    return Delete(tokens[0].start, tokens[0].end, tokens)
 
  def handle_typedef(self):
    token = self._GetNextToken()
    if (token.token_type == tokenize.NAME and
            keywords.IsKeyword(token.name)):
      # Token must be struct/enum/union/class.
      method = getattr(self, 'handle_' + token.name)
      self._handling_typedef = True
      tokens = [method()]
      self._handling_typedef = False
    else:
      tokens = [token]
 
    # Get the remainder of the typedef up to the semi-colon.
    tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
 
    # TODO(nnorwitz): clean all this up.
    assert tokens
    name = tokens.pop()
    indices = name
    if tokens:
      indices = tokens[0]
    if not indices:
      indices = token
    if name.name == ')':
      # HACK(nnorwitz): Handle pointers to functions "properly".
      if (len(tokens) >= 4 and
              tokens[1].name == '(' and tokens[2].name == '*'):
        tokens.append(name)
        name = tokens[3]
    elif name.name == ']':
      # HACK(nnorwitz): Handle arrays properly.
      if len(tokens) >= 2:
        tokens.append(name)
        name = tokens[1]
    new_type = tokens
    if tokens and isinstance(tokens[0], tokenize.Token):
      new_type = self.converter.ToType(tokens)[0]
    return Typedef(indices.start, indices.end, name.name,
                   new_type, self.namespace_stack)
 
  def handle_typeid(self):
    pass  # Not needed yet.
 
  def handle_typename(self):
    pass  # Not needed yet.
 
  def _GetTemplatedTypes(self):
    result = collections.OrderedDict()
    tokens = list(self._GetMatchingChar('<', '>'))
    len_tokens = len(tokens) - 1    # Ignore trailing '>'.
    i = 0
    while i < len_tokens:
      key = tokens[i].name
      i += 1
      if keywords.IsKeyword(key) or key == ',':
        continue
      type_name = default = None
      if i < len_tokens:
        i += 1
        if tokens[i-1].name == '=':
          assert i < len_tokens, '%s %s' % (i, tokens)
          default, unused_next_token = self.GetName(tokens[i:])
          i += len(default)
        else:
          if tokens[i-1].name != ',':
            # We got something like: Type variable.
            # Re-adjust the key (variable) and type_name (Type).
            key = tokens[i-1].name
            type_name = tokens[i-2]
 
      result[key] = (type_name, default)
    return result
 
  def handle_template(self):
    token = self._GetNextToken()
    assert token.token_type == tokenize.SYNTAX, token
    assert token.name == '<', token
    templated_types = self._GetTemplatedTypes()
    # TODO(nnorwitz): for now, just ignore the template params.
    token = self._GetNextToken()
    if token.token_type == tokenize.NAME:
      if token.name == 'class':
        return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types)
      elif token.name == 'struct':
        return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types)
      elif token.name == 'friend':
        return self.handle_friend()
    self._AddBackToken(token)
    tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
    tokens.append(last)
    self._AddBackTokens(tokens)
    if last.name == '(':
      return self.GetMethod(FUNCTION_NONE, templated_types)
    # Must be a variable definition.
    return None
 
  def handle_true(self):
    pass  # Nothing to do.
 
  def handle_false(self):
    pass  # Nothing to do.
 
  def handle_asm(self):
    pass  # Not needed yet.
 
  def handle_class(self):
    return self._GetClass(Class, VISIBILITY_PRIVATE, None)
 
  def _GetBases(self):
    # Get base classes.
    bases = []
    while 1:
      token = self._GetNextToken()
      assert token.token_type == tokenize.NAME, token
      # TODO(nnorwitz): store kind of inheritance...maybe.
      if token.name not in ('public', 'protected', 'private'):
        # If inheritance type is not specified, it is private.
        # Just put the token back so we can form a name.
        # TODO(nnorwitz): it would be good to warn about this.
        self._AddBackToken(token)
      else:
        # Check for virtual inheritance.
        token = self._GetNextToken()
        if token.name != 'virtual':
          self._AddBackToken(token)
        else:
          # TODO(nnorwitz): store that we got virtual for this base.
          pass
      base, next_token = self.GetName()
      bases_ast = self.converter.ToType(base)
      assert len(bases_ast) == 1, bases_ast
      bases.append(bases_ast[0])
      assert next_token.token_type == tokenize.SYNTAX, next_token
      if next_token.name == '{':
        token = next_token
        break
      # Support multiple inheritance.
      assert next_token.name == ',', next_token
    return bases, token
 
  def _GetClass(self, class_type, visibility, templated_types):
    class_name = None
    class_token = self._GetNextToken()
    if class_token.token_type != tokenize.NAME:
      assert class_token.token_type == tokenize.SYNTAX, class_token
      token = class_token
    else:
      # Skip any macro (e.g. storage class specifiers) after the
      # 'class' keyword.
      next_token = self._GetNextToken()
      if next_token.token_type == tokenize.NAME:
        self._AddBackToken(next_token)
      else:
        self._AddBackTokens([class_token, next_token])
      name_tokens, token = self.GetName()
      class_name = ''.join([t.name for t in name_tokens])
    bases = None
    if token.token_type == tokenize.SYNTAX:
      if token.name == ';':
        # Forward declaration.
        return class_type(class_token.start, class_token.end,
                          class_name, None, templated_types, None,
                          self.namespace_stack)
      if token.name in '*&':
        # Inline forward declaration.  Could be method or data.
        name_token = self._GetNextToken()
        next_token = self._GetNextToken()
        if next_token.name == ';':
          # Handle data
          modifiers = ['class']
          return self._CreateVariable(class_token, name_token.name,
                                      class_name,
                                      modifiers, token.name, None)
        else:
          # Assume this is a method.
          tokens = (class_token, token, name_token, next_token)
          self._AddBackTokens(tokens)
          return self.GetMethod(FUNCTION_NONE, None)
      if token.name == ':':
        bases, token = self._GetBases()
 
    body = None
    if token.token_type == tokenize.SYNTAX and token.name == '{':
      assert token.token_type == tokenize.SYNTAX, token
      assert token.name == '{', token
 
      ast = AstBuilder(self.GetScope(), self.filename, class_name,
                       visibility, self.namespace_stack)
      body = list(ast.Generate())
 
      if not self._handling_typedef:
        token = self._GetNextToken()
        if token.token_type != tokenize.NAME:
          assert token.token_type == tokenize.SYNTAX, token
          assert token.name == ';', token
        else:
          new_class = class_type(class_token.start, class_token.end,
                                 class_name, bases, None,
                                 body, self.namespace_stack)
 
          modifiers = []
          return self._CreateVariable(class_token,
                                      token.name, new_class,
                                      modifiers, token.name, None)
    else:
      if not self._handling_typedef:
        self.HandleError('non-typedef token', token)
      self._AddBackToken(token)
 
    return class_type(class_token.start, class_token.end, class_name,
                      bases, templated_types, body, self.namespace_stack)
 
  def handle_namespace(self):
    # Support anonymous namespaces.
    name = None
    name_tokens, token = self.GetName()
    if name_tokens:
      name = ''.join([t.name for t in name_tokens])
    self.namespace_stack.append(name)
    assert token.token_type == tokenize.SYNTAX, token
    # Create an internal token that denotes when the namespace is complete.
    internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
                                    None, None)
    internal_token.whence = token.whence
    if token.name == '=':
      # TODO(nnorwitz): handle aliasing namespaces.
      name, next_token = self.GetName()
      assert next_token.name == ';', next_token
      self._AddBackToken(internal_token)
    else:
      assert token.name == '{', token
      tokens = list(self.GetScope())
      # Replace the trailing } with the internal namespace pop token.
      tokens[-1] = internal_token
      # Handle namespace with nothing in it.
      self._AddBackTokens(tokens)
    return None
 
  def handle_using(self):
    tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
    assert tokens
    return Using(tokens[0].start, tokens[0].end, tokens)
 
  def handle_explicit(self):
    assert self.in_class
    # Nothing much to do.
    # TODO(nnorwitz): maybe verify the method name == class name.
    # This must be a ctor.
    return self.GetMethod(FUNCTION_CTOR, None)
 
  def handle_this(self):
    pass  # Nothing to do.
 
  def handle_operator(self):
    # Pull off the next token(s?) and make that part of the method name.
    pass
 
  def handle_sizeof(self):
    pass
 
  def handle_case(self):
    pass
 
  def handle_switch(self):
    pass
 
  def handle_default(self):
    token = self._GetNextToken()
    assert token.token_type == tokenize.SYNTAX
    assert token.name == ':'
 
  def handle_if(self):
    pass
 
  def handle_else(self):
    pass
 
  def handle_return(self):
    tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
    if not tokens:
      return Return(self.current_token.start, self.current_token.end, None)
    return Return(tokens[0].start, tokens[0].end, tokens)
 
  def handle_goto(self):
    tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
    assert len(tokens) == 1, str(tokens)
    return Goto(tokens[0].start, tokens[0].end, tokens[0].name)
 
  def handle_try(self):
    pass  # Not needed yet.
 
  def handle_catch(self):
    pass  # Not needed yet.
 
  def handle_throw(self):
    pass  # Not needed yet.
 
  def handle_while(self):
    pass
 
  def handle_do(self):
    pass
 
  def handle_for(self):
    pass
 
  def handle_break(self):
    self._IgnoreUpTo(tokenize.SYNTAX, ';')
 
  def handle_continue(self):
    self._IgnoreUpTo(tokenize.SYNTAX, ';')
 
 
def BuilderFromSource(source, filename):
  """Utility method that returns an AstBuilder from source code.
 
    Args:
      source: 'C++ source code'
      filename: 'file1'
 
    Returns:
      AstBuilder
    """
  return AstBuilder(tokenize.GetTokens(source), filename)
 
 
def PrintIndentifiers(filename, should_print):
  """Prints all identifiers for a C++ source file.
 
    Args:
      filename: 'file1'
      should_print: predicate with signature: bool Function(token)
    """
  source = utils.ReadFile(filename, False)
  if source is None:
    sys.stderr.write('Unable to find: %s\n' % filename)
    return
 
  #print('Processing %s' % actual_filename)
  builder = BuilderFromSource(source, filename)
  try:
    for node in builder.Generate():
      if should_print(node):
        print(node.name)
  except KeyboardInterrupt:
    return
  except:
    pass
 
 
def PrintAllIndentifiers(filenames, should_print):
  """Prints all identifiers for each C++ source file in filenames.
 
    Args:
      filenames: ['file1', 'file2', ...]
      should_print: predicate with signature: bool Function(token)
    """
  for path in filenames:
    PrintIndentifiers(path, should_print)
 
 
def main(argv):
  for filename in argv[1:]:
    source = utils.ReadFile(filename)
    if source is None:
      continue
 
    print('Processing %s' % filename)
    builder = BuilderFromSource(source, filename)
    try:
      entire_ast = filter(None, builder.Generate())
    except KeyboardInterrupt:
      return
    except:
      # Already printed a warning, print the traceback and continue.
      traceback.print_exc()
    else:
      if utils.DEBUG:
        for ast in entire_ast:
          print(ast)
 
 
if __name__ == '__main__':
  main(sys.argv)