diff --git a/.github/workflows/linux-x64-cpu-clang-python.yml b/.github/workflows/linux-x64-cpu-clang-python.yml index 53d6810d2eb..50358cf8039 100644 --- a/.github/workflows/linux-x64-cpu-clang-python.yml +++ b/.github/workflows/linux-x64-cpu-clang-python.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - name: cancel-previous-runs @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest setuptools wheel twine - name: configure env: CC: clang @@ -38,3 +38,13 @@ jobs: run: cd python && pip install . - name: test run: cd python && pytest tests + - name: build and publish + if: startsWith(github.ref, 'refs/tags') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" + run: | + cd python + python setup.py bdist_wheel + twine upload dist/* \ No newline at end of file diff --git a/.github/workflows/macos-x64-cpu-python.yml b/.github/workflows/macos-x64-cpu-python.yml index b64ecade41d..f3d3eb8d750 100644 --- a/.github/workflows/macos-x64-cpu-python.yml +++ b/.github/workflows/macos-x64-cpu-python.yml @@ -7,7 +7,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - name: cancel-previous-runs uses: styfle/cancel-workflow-action@0.7.0 @@ -23,7 +23,7 @@ jobs: - name: install dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest setuptools wheel twine - name: protobuf run: brew install protobuf opencv3 - name: cache-openmp @@ -63,4 +63,14 @@ jobs: - name: install python run: cd python && pip install . - name: test - run: cd python && pytest tests \ No newline at end of file + run: cd python && pytest tests + - name: build and publish + if: startsWith(github.ref, 'refs/tags') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" + run: | + cd python + python setup.py bdist_wheel + twine upload dist/* \ No newline at end of file diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml new file mode 100644 index 00000000000..1b4a9b1099b --- /dev/null +++ b/.github/workflows/release-python.yml @@ -0,0 +1,85 @@ +name: release-python +on: + push: + tags: + - '*' + +jobs: + build_sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - uses: actions/setup-python@v2 + + - name: Install deps + run: python -m pip install twine build + + - name: Build SDist + run: python -m build -s + + - name: Check metadata + run: twine check dist/* + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + + build_wheels: + name: Wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + + - name: Build wheel + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_SKIP: "cp27-* pp27-*" # skip Python 2.7 wheels + + - name: Show files + run: ls -lh wheelhouse + shell: bash + + - name: Verify clean directory + run: git diff --exit-code + shell: bash + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + upload_all: + name: Upload if release + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/setup-python@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@v1.4.1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/windows-x64-cpu-vs2019-python.yml b/.github/workflows/windows-x64-cpu-vs2019-python.yml index 802f235e94c..908f2a91a68 100644 --- a/.github/workflows/windows-x64-cpu-vs2019-python.yml +++ b/.github/workflows/windows-x64-cpu-vs2019-python.yml @@ -47,6 +47,7 @@ jobs: - name: test run: cd python && pytest tests - name: build and publish + if: startsWith(github.ref, 'refs/tags') env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..84bca11705f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "ninja; sys_platform != 'win32'", + "cmake>=3.12", +] +build-backend = "setuptools.build_meta" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 8566bc2cd26..140555ce706 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -10,10 +10,12 @@ set( CMAKE_CXX_STANDARD_REQUIRED ON ) add_subdirectory(pybind11) -if(MSVC OR CMAKE_GENERATOR STREQUAL "Xcode") - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}/ncnn/) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}/ncnn/) -endif(MSVC OR CMAKE_GENERATOR STREQUAL "Xcode") +if("${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" STREQUAL "") + if(MSVC OR CMAKE_GENERATOR STREQUAL "Xcode") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}/ncnn/) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}/ncnn/) + endif(MSVC OR CMAKE_GENERATOR STREQUAL "Xcode") +endif("${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" STREQUAL "") include_directories(${pybind11_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS}) pybind11_add_module(pyncnn src/main.cpp) @@ -21,8 +23,10 @@ set_target_properties(pyncnn PROPERTIES OUTPUT_NAME "ncnn") target_link_libraries(pyncnn PUBLIC ncnn) set_target_properties(pyncnn PROPERTIES PREFIX "" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/ncnn") set_property(TARGET pyncnn PROPERTY FOLDER "python") -add_custom_command(TARGET pyncnn POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/ncnn/ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION} - ${PROJECT_SOURCE_DIR}/ncnn/ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION}) +if("${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" STREQUAL "") + add_custom_command(TARGET pyncnn POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/ncnn/ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION} + ${PROJECT_SOURCE_DIR}/ncnn/ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION}) +endif("${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" STREQUAL "") configure_file(setup.py.i ${PROJECT_SOURCE_DIR}/setup.py) diff --git a/python/README.md b/python/README.md index a1e640c6e1f..786e837d7f1 100644 --- a/python/README.md +++ b/python/README.md @@ -1,6 +1,21 @@ # ncnn python wrapper of ncnn with [pybind11](https://github.com/pybind/pybind11), only support python3.x now. + +Install from pip +================== + +ncnn is available as wheel packages for macOS, Windows and Linux distributions, you can install with pip: + +``` +python -m pip install -U pip +python -m pip install -U ncnn +``` + +# Build from source + +If you want to build ncnn with some options not as default, or just like to build everything yourself, it is not difficult to build ncnn from source. + ## Prerequisites **On Unix (Linux, OS X)** @@ -8,6 +23,11 @@ python wrapper of ncnn with [pybind11](https://github.com/pybind/pybind11), only * A compiler with C++11 support * CMake >= 3.4 +**On Mac** + +* A compiler with C++11 support +* CMake >= 3.4 + **On Windows** * Visual Studio 2015 or higher @@ -79,6 +99,26 @@ import ncnn.model_zoo as model_zoo print(model_zoo.get_model_list()) ``` +models now in model zoo are as list below: +```bash +mobilenet_yolov2 +mobilenetv2_yolov3 +yolov4_tiny +yolov4 +yolov5s +yolact +mobilenet_ssd +squeezenet_ssd +mobilenetv2_ssdlite +mobilenetv3_ssdlite +squeezenet +faster_rcnn +peleenet_ssd +retinaface +rfcn +shufflenetv2 +simplepose +``` all model in model zoo has example in ncnn/python/examples folder # Custom Layer diff --git a/python/requirements.txt b/python/requirements.txt index ecccd013ff2..ac832c3f88f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,4 @@ +numpy tqdm requests portalocker diff --git a/python/setup.py.i b/python/setup.py.i index 962c0935cd5..1070ad51a3b 100644 --- a/python/setup.py.i +++ b/python/setup.py.i @@ -1,44 +1,42 @@ from setuptools import setup, find_packages -import sys -if sys.version_info < (3,0): - sys.exit("Sorry, Python < 3.0 is not supported") - try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + class bdist_wheel(_bdist_wheel): def finalize_options(self): _bdist_wheel.finalize_options(self) self.root_is_pure = False + + except ImportError: bdist_wheel = None -requirements = [ - "numpy", - #"tqdm", - #"requests", - #"portalocker", - #"opencv-python" -] +if sys.version_info < (3, 0): + sys.exit("Sorry, Python < 3.0 is not supported") + +requirements = ["numpy", "tqdm", "requests", "portalocker", "opencv-python"] setup( - name = "ncnn", - version = "${PACKAGE_VERSION}", - author = "nihui", - author_email = "nihuini@tencent.com", - maintainer = "caishanli", - maintainer_email = "caishanli25@gmail.com", - description = "ncnn is a high-performance neural network inference framework optimized for the mobile platform", - url = "https://github.com/Tencent/ncnn", - classifiers = [ - "Programming Language :: Python :: 3", + name="ncnn", + version="${PACKAGE_VERSION}", + author="nihui", + author_email="nihuini@tencent.com", + maintainer="caishanli", + maintainer_email="caishanli25@gmail.com", + description="ncnn is a high-performance neural network inference framework optimized for the mobile platform", + url="https://github.com/Tencent/ncnn", + classifiers=[ + "Programming Language :: C++", + "Programming Language :: Python 3", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", - ], - python_requires = ">=3.5", - packages = find_packages(), - package_dir = {"": "."}, - package_data = {"ncnn": ["ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION}"]}, - install_requires = requirements, - cmdclass = {"bdist_wheel": bdist_wheel}, + ], + license="BSD-3", + python_requires=">=3.5", + packages=find_packages(), + package_dir={"": "."}, + package_data={"ncnn": ["ncnn${PYTHON_MODULE_PREFIX}${PYTHON_MODULE_EXTENSION}"]}, + install_requires=requirements, + cmdclass={"bdist_wheel": bdist_wheel}, ) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..07d2e4f717e --- /dev/null +++ b/setup.py @@ -0,0 +1,156 @@ +import io +import os +import sys +import time +import re +import subprocess + +from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import build_ext + + +def find_version(): + with io.open("CMakeLists.txt", encoding="utf8") as f: + version_file = f.read() + + version_major = re.findall(r"NCNN_VERSION_MAJOR (.+?)", version_file) + version_minor = re.findall(r"NCNN_VERSION_MINOR (.+?)", version_file) + + if version_major and version_minor: + if sys.platform == "darwin": + ncnn_version = time.strftime("%Y.%m.%d", time.localtime()) + else: + ncnn_version = time.strftime("%Y%m%d", time.localtime()) + + return version_major[0] + "." + version_minor[0] + "." + ncnn_version + raise RuntimeError("Unable to find version string.") + + +# Convert distutils Windows platform specifiers to CMake -A arguments +PLAT_TO_CMAKE = { + "win32": "Win32", + "win-amd64": "x64", + "win-arm32": "ARM", + "win-arm64": "ARM64", +} + +# A CMakeExtension needs a sourcedir instead of a file list. +# The name must be the _single_ output extension from the CMake build. +# If you need multiple extensions, see scikit-build. +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeBuild(build_ext): + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + extdir = os.path.join(extdir, "ncnn") + + # required for auto-detection of auxiliary "native" libs + if not extdir.endswith(os.path.sep): + extdir += os.path.sep + + cfg = "Debug" if self.debug else "Release" + + # CMake lets you override the generator - we need to check this. + # Can be set with Conda-Build, for example. + cmake_generator = os.environ.get("CMAKE_GENERATOR", "") + + # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON + # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code + # from Python. + cmake_args = [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir), + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE={}".format(extdir), + "-DPYTHON_EXECUTABLE={}".format(sys.executable), + "-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm + "-DNCNN_PYTHON=ON", + "-DNCNN_BUILD_BENCHMARK=OFF", + "-DNCNN_BUILD_EXAMPLES=OFF", + "-DNCNN_BUILD_TOOLS=OFF", + ] + build_args = [] + + if self.compiler.compiler_type != "msvc": + # Using Ninja-build since it a) is available as a wheel and b) + # multithreads automatically. MSVC would require all variables be + # exported for Ninja to pick it up, which is a little tricky to do. + # Users can override the generator with CMAKE_GENERATOR in CMake + # 3.15+. + if not cmake_generator: + cmake_args += ["-GNinja"] + else: + # Single config generators are handled "normally" + single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) + + # CMake allows an arch-in-generator style for backward compatibility + contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) + + # Specify the arch if using MSVC generator, but only if it doesn't + # contain a backward-compatibility arch spec already in the + # generator name. + if not single_config and not contains_arch: + cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] + + # Multi-config generators have a different way to specify configs + if not single_config: + cmake_args += [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) + ] + build_args += ["--config", cfg] + + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level + # across all generators. + if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: + # self.parallel is a Python 3 only way to set parallel jobs by hand + # using -j in the build_ext call, not supported by pip or PyPA-build. + if hasattr(self, "parallel") and self.parallel: + # CMake 3.12+ only. + build_args += ["-j{}".format(self.parallel)] + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + subprocess.check_call( + ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp + ) + subprocess.check_call( + ["cmake", "--build", "."] + build_args, cwd=self.build_temp + ) + + +if sys.version_info < (3, 0): + sys.exit("Sorry, Python < 3.0 is not supported") + +requirements = ["numpy", "tqdm", "requests", "portalocker", "opencv-python"] + +with io.open("README.md", encoding="utf-8") as h: + long_description = h.read() + +setup( + name="ncnn", + version=find_version(), + author="nihui", + author_email="nihuini@tencent.com", + maintainer="caishanli", + maintainer_email="caishanli25@gmail.com", + description="ncnn is a high-performance neural network inference framework optimized for the mobile platform", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/Tencent/ncnn", + classifiers=[ + "Programming Language :: C++", + "Programming Language :: Python 3", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + ], + license="BSD-3", + python_requires=">=3.5", + packages=find_packages("python"), + package_dir={"": "python"}, + install_requires=requirements, + ext_modules=[CMakeExtension("ncnn")], + cmdclass={"build_ext": CMakeBuild}, +)