282 lines
9.4 KiB
Python
282 lines
9.4 KiB
Python
|
# Copyright 2019, David Wilson
|
||
|
|
#
|
||
|
|
# Redistribution and use in source and binary forms, with or without
|
||
|
|
# modification, are permitted provided that the following conditions are met:
|
||
|
|
#
|
||
|
|
# 1. Redistributions of source code must retain the above copyright notice,
|
||
|
|
# this list of conditions and the following disclaimer.
|
||
|
|
#
|
||
|
|
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||
|
|
# this list of conditions and the following disclaimer in the documentation
|
||
|
|
# and/or other materials provided with the distribution.
|
||
|
|
#
|
||
|
|
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||
|
|
# may be used to endorse or promote products derived from this software without
|
||
|
|
# specific prior written permission.
|
||
|
|
#
|
||
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||
|
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||
|
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||
|
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||
|
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||
|
|
# POSSIBILITY OF SUCH DAMAGE.
|
||
|
|
|
||
|
|
from __future__ import absolute_import, division, print_function
|
||
|
|
from __future__ import unicode_literals
|
||
|
|
__metaclass__ = type
|
||
|
|
|
||
|
|
import collections
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Python >= 3.4, PEP 451 ModuleSpec API
|
||
|
|
import importlib.machinery
|
||
|
|
import importlib.util
|
||
|
|
except ImportError:
|
||
|
|
# Python < 3.4, PEP 302 Import Hooks
|
||
|
|
import imp
|
||
|
|
|
||
|
|
import mitogen.master
|
||
|
|
|
||
|
|
|
||
|
|
LOG = logging.getLogger(__name__)
|
||
|
|
PREFIX = 'ansible.module_utils.'
|
||
|
|
|
||
|
|
|
||
|
|
# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
|
||
|
|
# name Unqualified name of the module.
|
||
|
|
# path Filesystem path of the module.
|
||
|
|
# kind One of the constants in `imp`, as returned in `imp.find_module()`
|
||
|
|
# parent `ansible_mitogen.module_finder.Module` of parent package (if any).
|
||
|
|
Module = collections.namedtuple('Module', 'name path kind parent')
|
||
|
|
|
||
|
|
|
||
|
|
def get_fullname(module):
|
||
|
|
"""
|
||
|
|
Reconstruct a Module's canonical path by recursing through its parents.
|
||
|
|
"""
|
||
|
|
bits = [str(module.name)]
|
||
|
|
while module.parent:
|
||
|
|
bits.append(str(module.parent.name))
|
||
|
|
module = module.parent
|
||
|
|
return '.'.join(reversed(bits))
|
||
|
|
|
||
|
|
|
||
|
|
def get_code(module):
|
||
|
|
"""
|
||
|
|
Compile and return a Module's code object.
|
||
|
|
"""
|
||
|
|
fp = open(module.path, 'rb')
|
||
|
|
try:
|
||
|
|
return compile(fp.read(), str(module.name), 'exec')
|
||
|
|
finally:
|
||
|
|
fp.close()
|
||
|
|
|
||
|
|
|
||
|
|
def is_pkg(module):
|
||
|
|
"""
|
||
|
|
Return :data:`True` if a Module represents a package.
|
||
|
|
"""
|
||
|
|
return module.kind == imp.PKG_DIRECTORY
|
||
|
|
|
||
|
|
|
||
|
|
def find(name, path=(), parent=None):
|
||
|
|
"""
|
||
|
|
Return a Module instance describing the first matching module found on the
|
||
|
|
search path.
|
||
|
|
|
||
|
|
:param str name:
|
||
|
|
Module name.
|
||
|
|
:param list path:
|
||
|
|
List of directory names to search for the module.
|
||
|
|
:param Module parent:
|
||
|
|
Optional module parent.
|
||
|
|
"""
|
||
|
|
assert isinstance(path, tuple)
|
||
|
|
head, _, tail = name.partition('.')
|
||
|
|
try:
|
||
|
|
tup = imp.find_module(head, list(path))
|
||
|
|
except ImportError:
|
||
|
|
return parent
|
||
|
|
|
||
|
|
fp, modpath, (suffix, mode, kind) = tup
|
||
|
|
if fp:
|
||
|
|
fp.close()
|
||
|
|
|
||
|
|
if parent and modpath == parent.path:
|
||
|
|
# 'from timeout import timeout', where 'timeout' is a function but also
|
||
|
|
# the name of the module being imported.
|
||
|
|
return None
|
||
|
|
|
||
|
|
if kind == imp.PKG_DIRECTORY:
|
||
|
|
modpath = os.path.join(modpath, '__init__.py')
|
||
|
|
|
||
|
|
module = Module(head, modpath, kind, parent)
|
||
|
|
# TODO: this code is entirely wrong on Python 3.x, but works well enough
|
||
|
|
# for Ansible. We need a new find_child() that only looks in the package
|
||
|
|
# directory, never falling back to the parent search path.
|
||
|
|
if tail and kind == imp.PKG_DIRECTORY:
|
||
|
|
return find_relative(module, tail, path)
|
||
|
|
return module
|
||
|
|
|
||
|
|
|
||
|
|
def find_relative(parent, name, path=()):
|
||
|
|
if parent.kind == imp.PKG_DIRECTORY:
|
||
|
|
path = (os.path.dirname(parent.path),) + path
|
||
|
|
return find(name, path, parent=parent)
|
||
|
|
|
||
|
|
|
||
|
|
def scan_fromlist(code):
|
||
|
|
"""Return an iterator of (level, name) for explicit imports in a code
|
||
|
|
object.
|
||
|
|
|
||
|
|
Not all names identify a module. `from os import name, path` generates
|
||
|
|
`(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
|
||
|
|
|
||
|
|
>>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
|
||
|
|
>>> code = compile(src, '<str>', 'exec')
|
||
|
|
>>> list(scan_fromlist(code))
|
||
|
|
[(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
|
||
|
|
"""
|
||
|
|
for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
|
||
|
|
for name in fromlist:
|
||
|
|
yield level, str('%s.%s' % (modname_s, name))
|
||
|
|
if not fromlist:
|
||
|
|
yield level, modname_s
|
||
|
|
|
||
|
|
|
||
|
|
def walk_imports(code, prefix=None):
|
||
|
|
"""Return an iterator of names for implicit parent imports & explicit
|
||
|
|
imports in a code object.
|
||
|
|
|
||
|
|
If a prefix is provided, then only children of that prefix are included.
|
||
|
|
Not all names identify a module. `from os import name, path` generates
|
||
|
|
`'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
|
||
|
|
|
||
|
|
>>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
|
||
|
|
>>> code = compile(source, '<str>', 'exec')
|
||
|
|
>>> list(walk_imports(code))
|
||
|
|
['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
|
||
|
|
>>> list(walk_imports(code, prefix='b'))
|
||
|
|
['b.c', 'b.d', 'b.d.e', 'b.d.f']
|
||
|
|
"""
|
||
|
|
if prefix is None:
|
||
|
|
prefix = ''
|
||
|
|
pattern = re.compile(r'(^|\.)(\w+)')
|
||
|
|
start = len(prefix)
|
||
|
|
for _, name, fromlist in mitogen.master.scan_code_imports(code):
|
||
|
|
if not name.startswith(prefix):
|
||
|
|
continue
|
||
|
|
for match in pattern.finditer(name, start):
|
||
|
|
yield name[:match.end()]
|
||
|
|
for leaf in fromlist:
|
||
|
|
yield str('%s.%s' % (name, leaf))
|
||
|
|
|
||
|
|
|
||
|
|
def scan(module_name, module_path, search_path):
|
||
|
|
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||
|
|
"""Return a list of (name, path, is_package) for ansible.module_utils
|
||
|
|
imports used by an Ansible module.
|
||
|
|
"""
|
||
|
|
log = LOG.getChild('scan')
|
||
|
|
log.debug('%r, %r, %r', module_name, module_path, search_path)
|
||
|
|
|
||
|
|
if sys.version_info >= (3, 4):
|
||
|
|
result = _scan_importlib_find_spec(
|
||
|
|
module_name, module_path, search_path,
|
||
|
|
)
|
||
|
|
log.debug('_scan_importlib_find_spec %r', result)
|
||
|
|
else:
|
||
|
|
result = _scan_imp_find_module(module_name, module_path, search_path)
|
||
|
|
log.debug('_scan_imp_find_module %r', result)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _scan_importlib_find_spec(module_name, module_path, search_path):
|
||
|
|
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||
|
|
module = importlib.machinery.ModuleSpec(
|
||
|
|
module_name, loader=None, origin=module_path,
|
||
|
|
)
|
||
|
|
prefix = importlib.machinery.ModuleSpec(
|
||
|
|
PREFIX.rstrip('.'), loader=None,
|
||
|
|
)
|
||
|
|
prefix.submodule_search_locations = search_path
|
||
|
|
queue = collections.deque([module])
|
||
|
|
specs = {prefix.name: prefix}
|
||
|
|
while queue:
|
||
|
|
spec = queue.popleft()
|
||
|
|
if spec.origin is None:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
with open(spec.origin, 'rb') as f:
|
||
|
|
code = compile(f.read(), spec.name, 'exec')
|
||
|
|
except Exception as exc:
|
||
|
|
raise ValueError((exc, module, spec, specs))
|
||
|
|
|
||
|
|
for name in walk_imports(code, prefix.name):
|
||
|
|
if name in specs:
|
||
|
|
continue
|
||
|
|
|
||
|
|
parent_name = name.rpartition('.')[0]
|
||
|
|
parent = specs[parent_name]
|
||
|
|
if parent is None or not parent.submodule_search_locations:
|
||
|
|
specs[name] = None
|
||
|
|
continue
|
||
|
|
|
||
|
|
child = importlib.util._find_spec(
|
||
|
|
name, parent.submodule_search_locations,
|
||
|
|
)
|
||
|
|
if child is None or child.origin is None:
|
||
|
|
specs[name] = None
|
||
|
|
continue
|
||
|
|
|
||
|
|
specs[name] = child
|
||
|
|
queue.append(child)
|
||
|
|
|
||
|
|
del specs[prefix.name]
|
||
|
|
return sorted(
|
||
|
|
(spec.name, spec.origin, spec.submodule_search_locations is not None)
|
||
|
|
for spec in specs.values() if spec is not None
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _scan_imp_find_module(module_name, module_path, search_path):
|
||
|
|
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||
|
|
module = Module(module_name, module_path, imp.PY_SOURCE, None)
|
||
|
|
stack = [module]
|
||
|
|
seen = set()
|
||
|
|
|
||
|
|
while stack:
|
||
|
|
module = stack.pop(0)
|
||
|
|
for level, fromname in scan_fromlist(get_code(module)):
|
||
|
|
if not fromname.startswith(PREFIX):
|
||
|
|
continue
|
||
|
|
|
||
|
|
imported = find(fromname[len(PREFIX):], search_path)
|
||
|
|
if imported is None or imported in seen:
|
||
|
|
continue
|
||
|
|
|
||
|
|
seen.add(imported)
|
||
|
|
stack.append(imported)
|
||
|
|
parent = imported.parent
|
||
|
|
while parent:
|
||
|
|
fullname = get_fullname(parent)
|
||
|
|
module = Module(fullname, parent.path, parent.kind, None)
|
||
|
|
if module not in seen:
|
||
|
|
seen.add(module)
|
||
|
|
stack.append(module)
|
||
|
|
parent = parent.parent
|
||
|
|
|
||
|
|
return sorted(
|
||
|
|
(PREFIX + get_fullname(module), module.path, is_pkg(module))
|
||
|
|
for module in seen
|
||
|
|
)
|