"""Find modules used by a script, using introspection."""
# This module should be kept compatible with Python 2.2, see PEP 291.
if hasattr(sys
.__stdout
__, "newlines"):
READ_MODE
= "U" # universal line endings
# remain compatible with Python < 2.3
LOAD_CONST
= dis
.opname
.index('LOAD_CONST')
IMPORT_NAME
= dis
.opname
.index('IMPORT_NAME')
STORE_NAME
= dis
.opname
.index('STORE_NAME')
STORE_GLOBAL
= dis
.opname
.index('STORE_GLOBAL')
STORE_OPS
= [STORE_NAME
, STORE_GLOBAL
]
# Modulefinder does a good job at simulating Python's, but it can not
# handle __path__ modifications packages make at runtime. Therefore there
# is a mechanism whereby you can register extra paths in this map for a
# package, and it will be honored.
# Note this is a mapping is lists of paths.
def AddPackagePath(packagename
, path
):
paths
= packagePathMap
.get(packagename
, [])
packagePathMap
[packagename
] = paths
# This ReplacePackage mechanism allows modulefinder to work around the
# way the _xmlplus package injects itself under the name "xml" into
# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml")
# before running ModuleFinder.
def ReplacePackage(oldname
, newname
):
replacePackageMap
[oldname
] = newname
def __init__(self
, name
, file=None, path
=None):
# The set of global names that are assigned to in the module.
# This includes those names imported through starimports of
# The set of starimports this module did that could not be
# resolved, ie. a starimport from a non-Python module.
s
= "Module(%r" % (self
.__name
__,)
if self
.__file
__ is not None:
s
= s
+ ", %r" % (self
.__file
__,)
if self
.__path
__ is not None:
s
= s
+ ", %r" % (self
.__path
__,)
def __init__(self
, path
=None, debug
=0, excludes
=[], replace_paths
=[]):
self
.replace_paths
= replace_paths
self
.processed_paths
= [] # Used in debugging only
def msg(self
, level
, str, *args
):
for i
in range(self
.indent
):
self
.indent
= self
.indent
+ 1
self
.indent
= self
.indent
- 1
def run_script(self
, pathname
):
self
.msg(2, "run_script", pathname
)
fp
= open(pathname
, READ_MODE
)
stuff
= ("", "r", imp
.PY_SOURCE
)
self
.load_module('__main__', fp
, pathname
, stuff
)
def load_file(self
, pathname
):
dir, name
= os
.path
.split(pathname
)
name
, ext
= os
.path
.splitext(name
)
fp
= open(pathname
, READ_MODE
)
stuff
= (ext
, "r", imp
.PY_SOURCE
)
self
.load_module(name
, fp
, pathname
, stuff
)
def import_hook(self
, name
, caller
=None, fromlist
=None):
self
.msg(3, "import_hook", name
, caller
, fromlist
)
parent
= self
.determine_parent(caller
)
q
, tail
= self
.find_head_package(parent
, name
)
m
= self
.load_tail(q
, tail
)
self
.ensure_fromlist(m
, fromlist
)
def determine_parent(self
, caller
):
self
.msgin(4, "determine_parent", caller
)
self
.msgout(4, "determine_parent -> None")
parent
= self
.modules
[pname
]
self
.msgout(4, "determine_parent ->", parent
)
parent
= self
.modules
[pname
]
assert parent
.__name
__ == pname
self
.msgout(4, "determine_parent ->", parent
)
self
.msgout(4, "determine_parent -> None")
def find_head_package(self
, parent
, name
):
self
.msgin(4, "find_head_package", parent
, name
)
qname
= "%s.%s" % (parent
.__name
__, head
)
q
= self
.import_module(head
, qname
, parent
)
self
.msgout(4, "find_head_package ->", (q
, tail
))
q
= self
.import_module(head
, qname
, parent
)
self
.msgout(4, "find_head_package ->", (q
, tail
))
self
.msgout(4, "raise ImportError: No module named", qname
)
raise ImportError, "No module named " + qname
def load_tail(self
, q
, tail
):
self
.msgin(4, "load_tail", q
, tail
)
head
, tail
= tail
[:i
], tail
[i
+1:]
mname
= "%s.%s" % (m
.__name
__, head
)
m
= self
.import_module(head
, mname
, m
)
self
.msgout(4, "raise ImportError: No module named", mname
)
raise ImportError, "No module named " + mname
self
.msgout(4, "load_tail ->", m
)
def ensure_fromlist(self
, m
, fromlist
, recursive
=0):
self
.msg(4, "ensure_fromlist", m
, fromlist
, recursive
)
all
= self
.find_all_submodules(m
)
self
.ensure_fromlist(m
, all
, 1)
elif not hasattr(m
, sub
):
subname
= "%s.%s" % (m
.__name
__, sub
)
submod
= self
.import_module(sub
, subname
, m
)
raise ImportError, "No module named " + subname
def find_all_submodules(self
, m
):
# 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"].
# But we must also collect Python extension modules - although
# we cannot separate normal dlls from Python extensions.
for triple
in imp
.get_suffixes():
suffixes
.append(triple
[0])
self
.msg(2, "can't list directory", dir)
if mod
and mod
!= "__init__":
def import_module(self
, partname
, fqname
, parent
):
self
.msgin(3, "import_module", partname
, fqname
, parent
)
self
.msgout(3, "import_module ->", m
)
if self
.badmodules
.has_key(fqname
):
self
.msgout(3, "import_module -> None")
if parent
and parent
.__path
__ is None:
self
.msgout(3, "import_module -> None")
fp
, pathname
, stuff
= self
.find_module(partname
,
parent
and parent
.__path
__, parent
)
self
.msgout(3, "import_module ->", None)
m
= self
.load_module(fqname
, fp
, pathname
, stuff
)
setattr(parent
, partname
, m
)
self
.msgout(3, "import_module ->", m
)
def load_module(self
, fqname
, fp
, pathname
, (suffix
, mode
, type)):
self
.msgin(2, "load_module", fqname
, fp
and "fp", pathname
)
if type == imp
.PKG_DIRECTORY
:
m
= self
.load_package(fqname
, pathname
)
self
.msgout(2, "load_module ->", m
)
if type == imp
.PY_SOURCE
:
co
= compile(fp
.read()+'\n', pathname
, 'exec')
elif type == imp
.PY_COMPILED
:
if fp
.read(4) != imp
.get_magic():
self
.msgout(2, "raise ImportError: Bad magic number", pathname
)
raise ImportError, "Bad magic number in %s" % pathname
m
= self
.add_module(fqname
)
co
= self
.replace_paths_in_code(co
)
self
.msgout(2, "load_module ->", m
)
def _add_badmodule(self
, name
, caller
):
if name
not in self
.badmodules
:
self
.badmodules
[name
] = {}
self
.badmodules
[name
][caller
.__name
__] = 1
def _safe_import_hook(self
, name
, caller
, fromlist
):
# wrapper for self.import_hook() that won't raise ImportError
if name
in self
.badmodules
:
self
._add
_badmodule
(name
, caller
)
self
.import_hook(name
, caller
)
self
.msg(2, "ImportError:", str(msg
))
self
._add
_badmodule
(name
, caller
)
if sub
in self
.badmodules
:
self
._add
_badmodule
(sub
, caller
)
self
.import_hook(name
, caller
, [sub
])
self
.msg(2, "ImportError:", str(msg
))
fullname
= name
+ "." + sub
self
._add
_badmodule
(fullname
, caller
)
def scan_code(self
, co
, m
):
if op
>= dis
.HAVE_ARGUMENT
:
oparg
= ord(code
[i
]) + ord(code
[i
+1])*256
# An IMPORT_NAME is always preceded by a LOAD_CONST, it's
# a tuple of "from" names, or None for a regular import.
# The tuple may contain "*" for "from <mod> import *"
fromlist
= co
.co_consts
[oparg
]
assert fromlist
is None or type(fromlist
) is tuple
name
= co
.co_names
[oparg
]
fromlist
= [f
for f
in fromlist
if f
!= "*"]
self
._safe
_import
_hook
(name
, m
, fromlist
)
# We've encountered an "import *". If it is a Python module,
# the code has already been parsed and we can suck out the
# At this point we don't know whether 'name' is a
# submodule of 'm' or a global module. Let's just try
mm
= self
.modules
.get(m
.__name
__ + "." + name
)
mm
= self
.modules
.get(name
)
m
.globalnames
.update(mm
.globalnames
)
m
.starimports
.update(mm
.starimports
)
# keep track of all global names that are assigned to
name
= co
.co_names
[oparg
]
if isinstance(c
, type(co
)):
def load_package(self
, fqname
, pathname
):
self
.msgin(2, "load_package", fqname
, pathname
)
newname
= replacePackageMap
.get(fqname
)
m
= self
.add_module(fqname
)
# As per comment at top of file, simulate runtime __path__ additions.
m
.__path
__ = m
.__path
__ + packagePathMap
.get(fqname
, [])
fp
, buf
, stuff
= self
.find_module("__init__", m
.__path
__)
self
.load_module(fqname
, fp
, buf
, stuff
)
self
.msgout(2, "load_package ->", m
)
def add_module(self
, fqname
):
if self
.modules
.has_key(fqname
):
return self
.modules
[fqname
]
self
.modules
[fqname
] = m
= Module(fqname
)
def find_module(self
, name
, path
, parent
=None):
# assert path is not None
fullname
= parent
.__name
__+'.'+name
if fullname
in self
.excludes
:
self
.msgout(3, "find_module -> Excluded", fullname
)
if name
in sys
.builtin_module_names
:
return (None, None, ("", "", imp
.C_BUILTIN
))
return imp
.find_module(name
, path
)
"""Print a report to stdout, listing the found modules with their
paths, as well as modules that are missing, or seem to be missing.
print " %-25s %s" % ("Name", "File")
print " %-25s %s" % ("----", "----")
keys
= self
.modules
.keys()
print "%-25s" % key
, m
.__file
__ or ""
missing
, maybe
= self
.any_missing_maybe()
mods
= self
.badmodules
[name
].keys()
print "?", name
, "imported from", ', '.join(mods
)
# Print modules that may be missing, but then again, maybe not...
print "Submodules thay appear to be missing, but could also be",
print "global names in the parent package:"
mods
= self
.badmodules
[name
].keys()
print "?", name
, "imported from", ', '.join(mods
)
"""Return a list of modules that appear to be missing. Use
any_missing_maybe() if you want to know which modules are
certain to be missing, and which *may* be missing.
missing
, maybe
= self
.any_missing_maybe()
def any_missing_maybe(self
):
"""Return two lists, one with modules that are certainly missing
and one with modules that *may* be missing. The latter names could
either be submodules *or* just global names in the package.
The reason it can't always be determined is that it's impossible to
tell which names are imported when "from module import *" is done
with an extension module, short of actually importing it.
for name
in self
.badmodules
:
if name
in self
.excludes
:
pkg
= self
.modules
.get(pkgname
)
if pkgname
in self
.badmodules
[name
]:
# The package tried to import this module itself and
# failed. It's definitely missing.
elif subname
in pkg
.globalnames
:
# It's a global in the package: definitely not missing.
# It could be missing, but the package did an "import *"
# from a non-Python module, so we simply can't be sure.
# It's not a global in the package, the package didn't
# do funny star imports, it's very likely to be missing.
# The symbol could be inserted into the package from the
# outside, but since that's not good style we simply list
def replace_paths_in_code(self
, co
):
new_filename
= original_filename
= os
.path
.normpath(co
.co_filename
)
for f
, r
in self
.replace_paths
:
if original_filename
.startswith(f
):
new_filename
= r
+ original_filename
[len(f
):]
if self
.debug
and original_filename
not in self
.processed_paths
:
if new_filename
!= original_filename
:
self
.msgout(2, "co_filename %r changed to %r" \
% (original_filename
,new_filename
,))
self
.msgout(2, "co_filename %r remains unchanged" \
self
.processed_paths
.append(original_filename
)
consts
= list(co
.co_consts
)
for i
in range(len(consts
)):
if isinstance(consts
[i
], type(co
)):
consts
[i
] = self
.replace_paths_in_code(consts
[i
])
return new
.code(co
.co_argcount
, co
.co_nlocals
, co
.co_stacksize
,
co
.co_flags
, co
.co_code
, tuple(consts
), co
.co_names
,
co
.co_varnames
, new_filename
, co
.co_name
,
co
.co_firstlineno
, co
.co_lnotab
,
co
.co_freevars
, co
.co_cellvars
)
opts
, args
= getopt
.getopt(sys
.argv
[1:], "dmp:qx:")
except getopt
.error
, msg
:
addpath
= addpath
+ a
.split(os
.pathsep
)
# Provide default arguments
# Set the path based on sys.path and the script directory
path
[0] = os
.path
.dirname(script
)
# Create the module finder and turn its crank
mf
= ModuleFinder(path
, debug
, exclude
)
mf
.import_hook(arg
[:-2], None, ["*"])
return mf
# for -i debugging
if __name__
== '__main__':
except KeyboardInterrupt: