Refactor testpep561 and add test to address #5767

mypy/modulefinder.py

@@ -121,7 +121,7 @@ def _update_ns_ancestors(self, components: List[str], match: Tuple[str, bool]) -
         path, verify = match
         for i in range(1, len(components)):
             pkg_id = '.'.join(components[:-i])
-            if pkg_id not in self.ns_ancestors:
+            if pkg_id not in self.ns_ancestors and self.fscache.isdir(path):
                 self.ns_ancestors[pkg_id] = path
             path = os.path.dirname(path)
 

mypy/test/testpep561.py

@@ -1,8 +1,9 @@
 from contextlib import contextmanager
+from enum import Enum
 import os
 import sys
 import tempfile
-from typing import Tuple, List, Generator, Optional
+from typing import Tuple, List, Generator, Optional, Any
 from unittest import TestCase, main
 
 import mypy.api
@@ -21,9 +22,8 @@
 reveal_type(a)
 """
 
-NAMESPACE_PROGRAM = """
-from typedpkg_nested.nested_package.nested_module import nested_func
-from typedpkg_namespace.alpha.alpha_module import alpha_func
+_NAMESPACE_PROGRAM = """
+{import_style}
 
 nested_func("abc")
 alpha_func(False)
@@ -32,27 +32,100 @@
 alpha_func(2)
 """
 
+C_EXT_PROGRAM = """
+from typedpkg_c_ext.foo import speak
+from typedpkg_c_ext.hello import helloworld
+
+speak("abc")
+speak(1)
+"""
+
+
+class NamespaceProgramImportStyle(Enum):
+    from_import = """\
+from typedpkg_nested.nested_package.nested_module import nested_func
+from typedpkg_namespace.alpha.alpha_module import alpha_func"""
+    import_as = """\
+import typedpkg_nested.nested_package.nested_module as nm; nested_func = nm.nested_func
+import typedpkg_namespace.alpha.alpha_module as am; alpha_func = am.alpha_func"""
+    regular_import = """\
+import typedpkg_nested.nested_package.nested_module; \
+nested_func = typedpkg_nested.nested_package.nested_module.nested_func
+import typedpkg_namespace.alpha.alpha_module; \
+alpha_func = typedpkg_namespace.alpha.alpha_module.alpha_func"""
+
+
+class SimpleProgramMessage(Enum):
+    msg_dne = "{tempfile}:3: error: Module 'typedpkg' has no attribute 'dne'"
+    msg_list = "{tempfile}:5: error: Revealed type is 'builtins.list[builtins.str]'"
+    msg_tuple = "{tempfile}:5: error: Revealed type is 'builtins.tuple[builtins.str]'"
+
+
+class NamespaceProgramMessage(Enum):
+    bool_str = ('{tempfile}:8: error: Argument 1 to "nested_func" has incompatible type '
+                '"bool"; expected "str"')
+    int_bool = ('{tempfile}:9: error: Argument 1 to "alpha_func" has incompatible type '
+                '"int"; expected "bool"')
+
 
-def check_mypy_run(cmd_line: List[str],
-                   python_executable: str = sys.executable,
-                   expected_out: str = '',
-                   expected_err: str = '',
-                   expected_returncode: int = 1,
-                   venv_dir: Optional[str] = None) -> None:
-    """Helper to run mypy and check the output."""
-    if venv_dir is not None:
-        old_dir = os.getcwd()
-        os.chdir(venv_dir)
-    try:
-        if python_executable != sys.executable:
-            cmd_line.append('--python-executable={}'.format(python_executable))
-        out, err, returncode = mypy.api.run(cmd_line)
-        assert out == expected_out, err
-        assert err == expected_err, out
-        assert returncode == expected_returncode, returncode
-    finally:
+class C_ExtProgramMessage(Enum):
+    cannot_find = '{tempfile}:3: error: Cannot find module named \'typedpkg_c_ext.hello\''
+    help_msg = ('{tempfile}:3: note: (Perhaps setting MYPYPATH or '
+                'using the "--ignore-missing-imports" flag would help)')
+    int_str = ('{tempfile}:6: error: Argument 1 to "speak" has incompatible type '
+               '"int"; expected "str"')
+
+
+def create_namespace_program_source(import_style: NamespaceProgramImportStyle) -> str:
+    return _NAMESPACE_PROGRAM.format(import_style=import_style.value)
+
+
+class ExampleProgram(object):
+    _fname = 'test_program.py'
+
+    def __init__(self, source_code: str) -> None:
+        self._source_code = source_code
+
+        self._temp_dir = None  # type: Optional[tempfile.TemporaryDirectory[Any]]
+        self._full_fname = ''
+
+    def init(self) -> None:
+        self._temp_dir = tempfile.TemporaryDirectory()
+        self._full_fname = os.path.join(self._temp_dir.name, self._fname)
+        with open(self._full_fname, 'w+') as f:
+            f.write(self._source_code)
+
+    def cleanup(self) -> None:
+        if self._temp_dir:
+            self._temp_dir.cleanup()
+
+    def build_msg(self, *msgs: Enum) -> str:
+        return '\n'.join(
+            msg.value.format(tempfile=self._full_fname)
+            for msg in msgs
+        ) + '\n'
+
+    def check_mypy_run(self,
+                       python_executable: str,
+                       expected_out: List[Enum],
+                       expected_err: str = '',
+                       expected_returncode: int = 1,
+                       venv_dir: Optional[str] = None) -> None:
+        """Helper to run mypy and check the output."""
+        cmd_line = [self._full_fname]
         if venv_dir is not None:
-            os.chdir(old_dir)
+            old_dir = os.getcwd()
+            os.chdir(venv_dir)
+        try:
+            if python_executable != sys.executable:
+                cmd_line.append('--python-executable={}'.format(python_executable))
+            out, err, returncode = mypy.api.run(cmd_line)
+            assert out == self.build_msg(*expected_out), err
+            assert err == expected_err, out
+            assert returncode == expected_returncode, returncode
+        finally:
+            if venv_dir is not None:
+                os.chdir(old_dir)
 
 
 class TestPEP561(TestCase):
@@ -102,127 +175,162 @@ def install_package(self, pkg: str,
             self.fail('\n'.join(lines))
 
     def setUp(self) -> None:
-        self.temp_file_dir = tempfile.TemporaryDirectory()
-        self.tempfile = os.path.join(self.temp_file_dir.name, 'simple.py')
-        with open(self.tempfile, 'w+') as file:
-            file.write(SIMPLE_PROGRAM)
-        self.namespace_tempfile = os.path.join(self.temp_file_dir.name, 'namespace_program.py')
-        with open(self.namespace_tempfile, 'w+') as file:
-            file.write(NAMESPACE_PROGRAM)
-
-        self.msg_dne = \
-            "{}:3: error: Module 'typedpkg' has no attribute 'dne'\n".format(self.tempfile)
-        self.msg_list = \
-            "{}:5: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile)
-        self.msg_tuple = \
-            "{}:5: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile)
-
-        self.namespace_msg_bool_str = (
-            '{0}:8: error: Argument 1 to "nested_func" has incompatible type "bool"; '
-            'expected "str"\n'.format(self.namespace_tempfile))
-        self.namespace_msg_int_bool = (
-            '{0}:9: error: Argument 1 to "alpha_func" has incompatible type "int"; '
-            'expected "bool"\n'.format(self.namespace_tempfile))
+        self.simple_example_program = ExampleProgram(SIMPLE_PROGRAM)
+        self.from_namespace_example_program = ExampleProgram(
+            create_namespace_program_source(NamespaceProgramImportStyle.from_import))
+        self.import_as_namespace_example_program = ExampleProgram(
+            create_namespace_program_source(NamespaceProgramImportStyle.from_import))
+        self.regular_import_namespace_example_program = ExampleProgram(
+            create_namespace_program_source(NamespaceProgramImportStyle.from_import))
+        self.c_ext_example_program = ExampleProgram(C_EXT_PROGRAM)
 
     def tearDown(self) -> None:
-        self.temp_file_dir.cleanup()
+        self.simple_example_program.cleanup()
+        self.from_namespace_example_program.cleanup()
+        self.import_as_namespace_example_program.cleanup()
+        self.regular_import_namespace_example_program.cleanup()
+        self.c_ext_example_program.cleanup()
 
     def test_get_pkg_dirs(self) -> None:
         """Check that get_package_dirs works."""
         dirs = get_site_packages_dirs(sys.executable)
         assert dirs
 
     def test_typedpkg_stub_package(self) -> None:
+        self.simple_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg-stubs', python_executable)
-            check_mypy_run(
-                [self.tempfile],
+            self.simple_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.msg_dne + self.msg_list,
+                [SimpleProgramMessage.msg_dne,
+                 SimpleProgramMessage.msg_list],
                 venv_dir=venv_dir,
             )
 
     def test_typedpkg(self) -> None:
+        self.simple_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg', python_executable)
-            check_mypy_run(
-                [self.tempfile],
+            self.simple_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.msg_tuple,
+                [SimpleProgramMessage.msg_tuple],
                 venv_dir=venv_dir,
             )
 
     def test_stub_and_typed_pkg(self) -> None:
+        self.simple_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg', python_executable)
             self.install_package('typedpkg-stubs', python_executable)
-            check_mypy_run(
-                [self.tempfile],
+            self.simple_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.msg_list,
+                [SimpleProgramMessage.msg_list],
                 venv_dir=venv_dir,
             )
 
     def test_typedpkg_stubs_python2(self) -> None:
+        self.simple_example_program.init()
         python2 = try_find_python2_interpreter()
         if python2:
             with self.virtualenv(python2) as venv:
                 venv_dir, py2 = venv
                 self.install_package('typedpkg-stubs', py2)
-                check_mypy_run(
-                    [self.tempfile],
+                self.simple_example_program.check_mypy_run(
                     py2,
-                    expected_out=self.msg_dne + self.msg_list,
+                    [SimpleProgramMessage.msg_dne,
+                     SimpleProgramMessage.msg_list],
                     venv_dir=venv_dir,
                 )
 
     def test_typedpkg_python2(self) -> None:
+        self.simple_example_program.init()
         python2 = try_find_python2_interpreter()
         if python2:
             with self.virtualenv(python2) as venv:
                 venv_dir, py2 = venv
                 self.install_package('typedpkg', py2)
-                check_mypy_run(
-                    [self.tempfile],
+                self.simple_example_program.check_mypy_run(
                     py2,
-                    expected_out=self.msg_tuple,
+                    [SimpleProgramMessage.msg_tuple],
                     venv_dir=venv_dir,
                 )
 
     def test_typedpkg_egg(self) -> None:
+        self.simple_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg', python_executable, use_pip=False)
-            check_mypy_run(
-                [self.tempfile],
+            self.simple_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.msg_tuple,
+                [SimpleProgramMessage.msg_tuple],
                 venv_dir=venv_dir,
             )
 
     def test_typedpkg_editable(self) -> None:
+        self.simple_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg', python_executable, editable=True)
-            check_mypy_run(
-                [self.tempfile],
+            self.simple_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.msg_tuple,
+                [SimpleProgramMessage.msg_tuple],
                 venv_dir=venv_dir,
             )
 
-    def test_nested_and_namespace(self) -> None:
+    def test_nested_and_namespace_from_import(self) -> None:
+        self.from_namespace_example_program.init()
         with self.virtualenv() as venv:
             venv_dir, python_executable = venv
             self.install_package('typedpkg_nested', python_executable)
             self.install_package('typedpkg_namespace-alpha', python_executable)
-            check_mypy_run(
-                [self.namespace_tempfile],
+            self.from_namespace_example_program.check_mypy_run(
+                python_executable,
+                [NamespaceProgramMessage.bool_str,
+                 NamespaceProgramMessage.int_bool],
+                venv_dir=venv_dir,
+            )
+
+    def test_nested_and_namespace_import_as(self) -> None:
+        self.import_as_namespace_example_program.init()
+        with self.virtualenv() as venv:
+            venv_dir, python_executable = venv
+            self.install_package('typedpkg_nested', python_executable)
+            self.install_package('typedpkg_namespace-alpha', python_executable)
+            self.import_as_namespace_example_program.check_mypy_run(
+                python_executable,
+                [NamespaceProgramMessage.bool_str,
+                 NamespaceProgramMessage.int_bool],
+                venv_dir=venv_dir,
+            )
+
+    def test_nested_and_namespace_regular_import(self) -> None:
+        # This test case addresses https://github.com/python/mypy/issues/5767
+        self.regular_import_namespace_example_program.init()
+        with self.virtualenv() as venv:
+            venv_dir, python_executable = venv
+            self.install_package('typedpkg_nested', python_executable)
+            self.install_package('typedpkg_namespace-alpha', python_executable)
+            self.regular_import_namespace_example_program.check_mypy_run(
+                python_executable,
+                [NamespaceProgramMessage.bool_str,
+                 NamespaceProgramMessage.int_bool],
+                venv_dir=venv_dir,
+            )
+
+    def test_c_ext_from_import(self) -> None:
+        # This test case addresses https://github.com/python/mypy/issues/5784
+        self.c_ext_example_program.init()
+        with self.virtualenv() as venv:
+            venv_dir, python_executable = venv
+            self.install_package('typedpkg_c_ext', python_executable)
+            self.c_ext_example_program.check_mypy_run(
                 python_executable,
-                expected_out=self.namespace_msg_bool_str + self.namespace_msg_int_bool,
+                [C_ExtProgramMessage.cannot_find,
+                 C_ExtProgramMessage.help_msg,
+                 C_ExtProgramMessage.int_str],
                 venv_dir=venv_dir,
             )
 

test-data/packages/typedpkg_c_ext/hello.c

@@ -0,0 +1,35 @@
+/*******************************************************************
+ * This code was copied from the article written by Elliot Forbes at
+ * https://tutorialedge.net/python/python-c-extensions-tutorial/
+ ******************************************************************/
+#include <Python.h>
+
+// ot ForbesFunction 1: A simple 'hello world' function
+static PyObject* helloworld(PyObject* self, PyObject* args)
+{
+    printf("Hello World\n");
+    return Py_None;
+}
+
+// Our Module's Function Definition struct
+// We require this `NULL` to signal the end of our method
+// definition
+static PyMethodDef myMethods[] = {
+    { "helloworld", helloworld, METH_NOARGS, "Prints Hello World" },
+    { NULL, NULL, 0, NULL }
+};
+
+// Our Module Definition struct
+static struct PyModuleDef myModule = {
+    PyModuleDef_HEAD_INIT,
+    "hello",
+    "Test Module",
+    -1,
+    myMethods
+};
+
+// Initializes our module using our above struct
+PyMODINIT_FUNC PyInit_hello(void)
+{
+    return PyModule_Create(&myModule);
+}