"""Parse a Python module and describe its classes and methods.
Parse enough of a Python file to recognize imports and class and
method definitions, and to find out the superclasses of a class.
The interface consists of a single function:
readmodule_ex(module [, path])
where module is the name of a Python module, and path is an optional
list of directories where the module is to be searched. If present,
path is prepended to the system search path sys.path. The return
value is a dictionary. The keys of the dictionary are the names of
the classes defined in the module (including classes that are defined
via the from XXX import YYY construct). The values are class
instances of the class Class defined here. One special key/value pair
is present for packages: the key '__path__' has a list as its value
which contains the package search path.
A class is described by the class Class in this module. Instances
of this class have the following instance variables:
module -- the module name
name -- the name of the class
super -- a list of super classes (Class instances)
methods -- a dictionary of methods
file -- the file in which the class was defined
lineno -- the line in the file on which the class statement occurred
The dictionary of methods uses the method names as keys and the line
numbers on which the method was defined as values.
If the name of a super class is not recognized, the corresponding
entry in the list of super classes is not a class instance but a
string giving the name of the super class. Since import statements
are recognized and imported modules are scanned as well, this
A function is described by the class Function in this module.
Instances of this class have the following instance variables:
module -- the module name
name -- the name of the class
file -- the file in which the class was defined
lineno -- the line in the file on which the class statement occurred
import tokenize
# Python tokenizer
from token
import NAME
, DEDENT
, NEWLINE
from operator
import itemgetter
__all__
= ["readmodule", "readmodule_ex", "Class", "Function"]
_modules
= {} # cache of modules we've seen
# each Python class is represented by an instance of this class
'''Class to represent a Python class.'''
def __init__(self
, module
, name
, super, file, lineno
):
def _addmethod(self
, name
, lineno
):
self
.methods
[name
] = lineno
'''Class to represent a top-level Python function'''
def __init__(self
, module
, name
, file, lineno
):
def readmodule(module
, path
=[]):
'''Backwards compatible interface.
Call readmodule_ex() and then only keep Class objects from the
dict = _readmodule(module
, path
)
for key
, value
in dict.items():
if isinstance(value
, Class
):
def readmodule_ex(module
, path
=[]):
'''Read a module file and return a dictionary of classes.
Search for MODULE in PATH and sys.path, read and parse the
module and return a dictionary with one entry for each class
If INPACKAGE is true, it must be the dotted name of the package in
which we are searching for a submodule, and then PATH must be the
package search path; otherwise, we are searching for a top-level
module, and PATH is combined with sys.path.
return _readmodule(module
, path
)
def _readmodule(module
, path
, inpackage
=None):
'''Do the hard work for readmodule[_ex].'''
# Compute the full module name (prepending inpackage if set)
fullmodule
= "%s.%s" % (inpackage
, module
)
if fullmodule
in _modules
:
return _modules
[fullmodule
]
# Initialize the dict for this module's contents
# Check if it is a built-in module; we don't do much for these
if module
in sys
.builtin_module_names
and not inpackage
:
# Check for a dotted module name
parent
= _readmodule(package
, path
, inpackage
)
package
= "%s.%s" % (inpackage
, package
)
return _readmodule(submodule
, parent
['__path__'], package
)
# Search the path for the module
f
, file, (suff
, mode
, type) = imp
.find_module(module
, path
)
f
, file, (suff
, mode
, type) = imp
.find_module(module
, path
+ sys
.path
)
if type == imp
.PKG_DIRECTORY
:
dict['__path__'] = [file]
f
, file, (suff
, mode
, type) = imp
.find_module('__init__', [file])
_modules
[fullmodule
] = dict
if type != imp
.PY_SOURCE
:
# not Python source, can't do anything with this module
stack
= [] # stack of (class, indent) pairs
g
= tokenize
.generate_tokens(f
.readline
)
for tokentype
, token
, start
, end
, line
in g
:
lineno
, thisindent
= start
# close nested classes and defs
while stack
and stack
[-1][1] >= thisindent
:
lineno
, thisindent
= start
# close previous nested classes and defs
while stack
and stack
[-1][1] >= thisindent
:
tokentype
, meth_name
, start
, end
, line
= g
.next()
if isinstance(cur_class
, Class
):
cur_class
._addmethod
(meth_name
, lineno
)
dict[meth_name
] = Function(module
, meth_name
, file, lineno
)
stack
.append((None, thisindent
)) # Marker for nested fns
lineno
, thisindent
= start
# close previous nested classes and defs
while stack
and stack
[-1][1] >= thisindent
:
tokentype
, class_name
, start
, end
, line
= g
.next()
# parse what follows the class name
tokentype
, token
, start
, end
, line
= g
.next()
names
= [] # List of superclasses
# there's a list of superclasses
super = [] # Tokens making up current superclass
tokentype
, token
, start
, end
, line
= g
.next()
if token
in (')', ',') and level
== 1:
# we know this super class
# super class is of the form
# module.class: look in module for
elif token
== ',' and level
== 1:
cur_class
= Class(fullmodule
, class_name
, inherit
, file, lineno
)
dict[class_name
] = cur_class
stack
.append((cur_class
, thisindent
))
elif token
== 'import' and start
[1] == 0:
modules
= _getnamelist(g
)
for mod
, mod2
in modules
:
# Recursively read the imported module
_readmodule(mod
, path
, inpackage
)
# If we can't find or parse the imported module,
# too bad -- don't die here.
elif token
== 'from' and start
[1] == 0:
if not mod
or token
!= "import":
# Recursively read the imported module
d
= _readmodule(mod
, path
, inpackage
)
# If we can't find or parse the imported module,
# too bad -- don't die here.
# add any classes that were defined in the imported module
# to our name space if they were mentioned in the list
# don't add names that start with _
# Helper to get a comma-separated list of dotted names plus 'as'
# clauses. Return a list of pairs (name, name2) where name2 is
# the 'as' name, or None if there is no 'as' clause.
name
, token
= _getname(g
)
name2
, token
= _getname(g
)
names
.append((name
, name2
))
while token
!= "," and "\n" not in token
:
tokentype
, token
, start
, end
, line
= g
.next()
# Helper to get a dotted name, return a pair (name, token) where
# name is the dotted name, or None if there was no dotted name,
# and token is the next input token.
tokentype
, token
, start
, end
, line
= g
.next()
if tokentype
!= NAME
and token
!= '*':
tokentype
, token
, start
, end
, line
= g
.next()
tokentype
, token
, start
, end
, line
= g
.next()
return (".".join(parts
), token
)
# Main program for testing.
path
= [os
.path
.dirname(mod
)]
mod
= os
.path
.basename(mod
)
if mod
.lower().endswith(".py"):
dict = readmodule_ex(mod
, path
)
objs
.sort(lambda a
, b
: cmp(getattr(a
, 'lineno', 0),
getattr(b
, 'lineno', 0)))
if isinstance(obj
, Class
):
print "class", obj
.name
, obj
.super, obj
.lineno
methods
= sorted(obj
.methods
.iteritems(), key
=itemgetter(1))
for name
, lineno
in methods
:
print " def", name
, lineno
elif isinstance(obj
, Function
):
print "def", obj
.name
, obj
.lineno
if __name__
== "__main__":