Commit | Line | Data |
---|---|---|
920dae64 AT |
1 | """Parse a Python module and describe its classes and methods. |
2 | ||
3 | Parse enough of a Python file to recognize imports and class and | |
4 | method definitions, and to find out the superclasses of a class. | |
5 | ||
6 | The interface consists of a single function: | |
7 | readmodule_ex(module [, path]) | |
8 | where module is the name of a Python module, and path is an optional | |
9 | list of directories where the module is to be searched. If present, | |
10 | path is prepended to the system search path sys.path. The return | |
11 | value is a dictionary. The keys of the dictionary are the names of | |
12 | the classes defined in the module (including classes that are defined | |
13 | via the from XXX import YYY construct). The values are class | |
14 | instances of the class Class defined here. One special key/value pair | |
15 | is present for packages: the key '__path__' has a list as its value | |
16 | which contains the package search path. | |
17 | ||
18 | A class is described by the class Class in this module. Instances | |
19 | of this class have the following instance variables: | |
20 | module -- the module name | |
21 | name -- the name of the class | |
22 | super -- a list of super classes (Class instances) | |
23 | methods -- a dictionary of methods | |
24 | file -- the file in which the class was defined | |
25 | lineno -- the line in the file on which the class statement occurred | |
26 | The dictionary of methods uses the method names as keys and the line | |
27 | numbers on which the method was defined as values. | |
28 | If the name of a super class is not recognized, the corresponding | |
29 | entry in the list of super classes is not a class instance but a | |
30 | string giving the name of the super class. Since import statements | |
31 | are recognized and imported modules are scanned as well, this | |
32 | shouldn't happen often. | |
33 | ||
34 | A function is described by the class Function in this module. | |
35 | Instances of this class have the following instance variables: | |
36 | module -- the module name | |
37 | name -- the name of the class | |
38 | file -- the file in which the class was defined | |
39 | lineno -- the line in the file on which the class statement occurred | |
40 | """ | |
41 | ||
42 | import sys | |
43 | import imp | |
44 | import tokenize # Python tokenizer | |
45 | from token import NAME, DEDENT, NEWLINE | |
46 | from operator import itemgetter | |
47 | ||
48 | __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] | |
49 | ||
50 | _modules = {} # cache of modules we've seen | |
51 | ||
52 | # each Python class is represented by an instance of this class | |
53 | class Class: | |
54 | '''Class to represent a Python class.''' | |
55 | def __init__(self, module, name, super, file, lineno): | |
56 | self.module = module | |
57 | self.name = name | |
58 | if super is None: | |
59 | super = [] | |
60 | self.super = super | |
61 | self.methods = {} | |
62 | self.file = file | |
63 | self.lineno = lineno | |
64 | ||
65 | def _addmethod(self, name, lineno): | |
66 | self.methods[name] = lineno | |
67 | ||
68 | class Function: | |
69 | '''Class to represent a top-level Python function''' | |
70 | def __init__(self, module, name, file, lineno): | |
71 | self.module = module | |
72 | self.name = name | |
73 | self.file = file | |
74 | self.lineno = lineno | |
75 | ||
76 | def readmodule(module, path=[]): | |
77 | '''Backwards compatible interface. | |
78 | ||
79 | Call readmodule_ex() and then only keep Class objects from the | |
80 | resulting dictionary.''' | |
81 | ||
82 | dict = _readmodule(module, path) | |
83 | res = {} | |
84 | for key, value in dict.items(): | |
85 | if isinstance(value, Class): | |
86 | res[key] = value | |
87 | return res | |
88 | ||
89 | def readmodule_ex(module, path=[]): | |
90 | '''Read a module file and return a dictionary of classes. | |
91 | ||
92 | Search for MODULE in PATH and sys.path, read and parse the | |
93 | module and return a dictionary with one entry for each class | |
94 | found in the module. | |
95 | ||
96 | If INPACKAGE is true, it must be the dotted name of the package in | |
97 | which we are searching for a submodule, and then PATH must be the | |
98 | package search path; otherwise, we are searching for a top-level | |
99 | module, and PATH is combined with sys.path. | |
100 | ''' | |
101 | return _readmodule(module, path) | |
102 | ||
103 | def _readmodule(module, path, inpackage=None): | |
104 | '''Do the hard work for readmodule[_ex].''' | |
105 | # Compute the full module name (prepending inpackage if set) | |
106 | if inpackage: | |
107 | fullmodule = "%s.%s" % (inpackage, module) | |
108 | else: | |
109 | fullmodule = module | |
110 | ||
111 | # Check in the cache | |
112 | if fullmodule in _modules: | |
113 | return _modules[fullmodule] | |
114 | ||
115 | # Initialize the dict for this module's contents | |
116 | dict = {} | |
117 | ||
118 | # Check if it is a built-in module; we don't do much for these | |
119 | if module in sys.builtin_module_names and not inpackage: | |
120 | _modules[module] = dict | |
121 | return dict | |
122 | ||
123 | # Check for a dotted module name | |
124 | i = module.rfind('.') | |
125 | if i >= 0: | |
126 | package = module[:i] | |
127 | submodule = module[i+1:] | |
128 | parent = _readmodule(package, path, inpackage) | |
129 | if inpackage: | |
130 | package = "%s.%s" % (inpackage, package) | |
131 | return _readmodule(submodule, parent['__path__'], package) | |
132 | ||
133 | # Search the path for the module | |
134 | f = None | |
135 | if inpackage: | |
136 | f, file, (suff, mode, type) = imp.find_module(module, path) | |
137 | else: | |
138 | f, file, (suff, mode, type) = imp.find_module(module, path + sys.path) | |
139 | if type == imp.PKG_DIRECTORY: | |
140 | dict['__path__'] = [file] | |
141 | path = [file] + path | |
142 | f, file, (suff, mode, type) = imp.find_module('__init__', [file]) | |
143 | _modules[fullmodule] = dict | |
144 | if type != imp.PY_SOURCE: | |
145 | # not Python source, can't do anything with this module | |
146 | f.close() | |
147 | return dict | |
148 | ||
149 | stack = [] # stack of (class, indent) pairs | |
150 | ||
151 | g = tokenize.generate_tokens(f.readline) | |
152 | try: | |
153 | for tokentype, token, start, end, line in g: | |
154 | if tokentype == DEDENT: | |
155 | lineno, thisindent = start | |
156 | # close nested classes and defs | |
157 | while stack and stack[-1][1] >= thisindent: | |
158 | del stack[-1] | |
159 | elif token == 'def': | |
160 | lineno, thisindent = start | |
161 | # close previous nested classes and defs | |
162 | while stack and stack[-1][1] >= thisindent: | |
163 | del stack[-1] | |
164 | tokentype, meth_name, start, end, line = g.next() | |
165 | if tokentype != NAME: | |
166 | continue # Syntax error | |
167 | if stack: | |
168 | cur_class = stack[-1][0] | |
169 | if isinstance(cur_class, Class): | |
170 | # it's a method | |
171 | cur_class._addmethod(meth_name, lineno) | |
172 | # else it's a nested def | |
173 | else: | |
174 | # it's a function | |
175 | dict[meth_name] = Function(module, meth_name, file, lineno) | |
176 | stack.append((None, thisindent)) # Marker for nested fns | |
177 | elif token == 'class': | |
178 | lineno, thisindent = start | |
179 | # close previous nested classes and defs | |
180 | while stack and stack[-1][1] >= thisindent: | |
181 | del stack[-1] | |
182 | tokentype, class_name, start, end, line = g.next() | |
183 | if tokentype != NAME: | |
184 | continue # Syntax error | |
185 | # parse what follows the class name | |
186 | tokentype, token, start, end, line = g.next() | |
187 | inherit = None | |
188 | if token == '(': | |
189 | names = [] # List of superclasses | |
190 | # there's a list of superclasses | |
191 | level = 1 | |
192 | super = [] # Tokens making up current superclass | |
193 | while True: | |
194 | tokentype, token, start, end, line = g.next() | |
195 | if token in (')', ',') and level == 1: | |
196 | n = "".join(super) | |
197 | if n in dict: | |
198 | # we know this super class | |
199 | n = dict[n] | |
200 | else: | |
201 | c = n.split('.') | |
202 | if len(c) > 1: | |
203 | # super class is of the form | |
204 | # module.class: look in module for | |
205 | # class | |
206 | m = c[-2] | |
207 | c = c[-1] | |
208 | if m in _modules: | |
209 | d = _modules[m] | |
210 | if c in d: | |
211 | n = d[c] | |
212 | names.append(n) | |
213 | super = [] | |
214 | if token == '(': | |
215 | level += 1 | |
216 | elif token == ')': | |
217 | level -= 1 | |
218 | if level == 0: | |
219 | break | |
220 | elif token == ',' and level == 1: | |
221 | pass | |
222 | else: | |
223 | super.append(token) | |
224 | inherit = names | |
225 | cur_class = Class(fullmodule, class_name, inherit, file, lineno) | |
226 | if not stack: | |
227 | dict[class_name] = cur_class | |
228 | stack.append((cur_class, thisindent)) | |
229 | elif token == 'import' and start[1] == 0: | |
230 | modules = _getnamelist(g) | |
231 | for mod, mod2 in modules: | |
232 | try: | |
233 | # Recursively read the imported module | |
234 | if not inpackage: | |
235 | _readmodule(mod, path) | |
236 | else: | |
237 | try: | |
238 | _readmodule(mod, path, inpackage) | |
239 | except ImportError: | |
240 | _readmodule(mod, []) | |
241 | except: | |
242 | # If we can't find or parse the imported module, | |
243 | # too bad -- don't die here. | |
244 | pass | |
245 | elif token == 'from' and start[1] == 0: | |
246 | mod, token = _getname(g) | |
247 | if not mod or token != "import": | |
248 | continue | |
249 | names = _getnamelist(g) | |
250 | try: | |
251 | # Recursively read the imported module | |
252 | d = _readmodule(mod, path, inpackage) | |
253 | except: | |
254 | # If we can't find or parse the imported module, | |
255 | # too bad -- don't die here. | |
256 | continue | |
257 | # add any classes that were defined in the imported module | |
258 | # to our name space if they were mentioned in the list | |
259 | for n, n2 in names: | |
260 | if n in d: | |
261 | dict[n2 or n] = d[n] | |
262 | elif n == '*': | |
263 | # don't add names that start with _ | |
264 | for n in d: | |
265 | if n[0] != '_': | |
266 | dict[n] = d[n] | |
267 | except StopIteration: | |
268 | pass | |
269 | ||
270 | f.close() | |
271 | return dict | |
272 | ||
273 | def _getnamelist(g): | |
274 | # Helper to get a comma-separated list of dotted names plus 'as' | |
275 | # clauses. Return a list of pairs (name, name2) where name2 is | |
276 | # the 'as' name, or None if there is no 'as' clause. | |
277 | names = [] | |
278 | while True: | |
279 | name, token = _getname(g) | |
280 | if not name: | |
281 | break | |
282 | if token == 'as': | |
283 | name2, token = _getname(g) | |
284 | else: | |
285 | name2 = None | |
286 | names.append((name, name2)) | |
287 | while token != "," and "\n" not in token: | |
288 | tokentype, token, start, end, line = g.next() | |
289 | if token != ",": | |
290 | break | |
291 | return names | |
292 | ||
293 | def _getname(g): | |
294 | # Helper to get a dotted name, return a pair (name, token) where | |
295 | # name is the dotted name, or None if there was no dotted name, | |
296 | # and token is the next input token. | |
297 | parts = [] | |
298 | tokentype, token, start, end, line = g.next() | |
299 | if tokentype != NAME and token != '*': | |
300 | return (None, token) | |
301 | parts.append(token) | |
302 | while True: | |
303 | tokentype, token, start, end, line = g.next() | |
304 | if token != '.': | |
305 | break | |
306 | tokentype, token, start, end, line = g.next() | |
307 | if tokentype != NAME: | |
308 | break | |
309 | parts.append(token) | |
310 | return (".".join(parts), token) | |
311 | ||
312 | def _main(): | |
313 | # Main program for testing. | |
314 | import os | |
315 | mod = sys.argv[1] | |
316 | if os.path.exists(mod): | |
317 | path = [os.path.dirname(mod)] | |
318 | mod = os.path.basename(mod) | |
319 | if mod.lower().endswith(".py"): | |
320 | mod = mod[:-3] | |
321 | else: | |
322 | path = [] | |
323 | dict = readmodule_ex(mod, path) | |
324 | objs = dict.values() | |
325 | objs.sort(lambda a, b: cmp(getattr(a, 'lineno', 0), | |
326 | getattr(b, 'lineno', 0))) | |
327 | for obj in objs: | |
328 | if isinstance(obj, Class): | |
329 | print "class", obj.name, obj.super, obj.lineno | |
330 | methods = sorted(obj.methods.iteritems(), key=itemgetter(1)) | |
331 | for name, lineno in methods: | |
332 | if name != "__path__": | |
333 | print " def", name, lineno | |
334 | elif isinstance(obj, Function): | |
335 | print "def", obj.name, obj.lineno | |
336 | ||
337 | if __name__ == "__main__": | |
338 | _main() |