Cython compiles python project

Cython compiles python project

Introduction

cythonizeis a command-line tool for the Cython compiler that simplifies the building process of Cython extension modules. Cython is a tool for writing Python extension modules that allows the syntax and performance of C to be used in Python while maintaining the flexibility and ease of use of Python.

Specifically,cythonize is used to compile Cython source files (.pyx) into C code and ultimately build them into shared libraries or extension modules for use by Python. The goal of this tool is to improve performance while maintaining the simplicity and readability of Python coding.

Install

• python install cython

  pip install cpython
  

• Linux installation: python-devel, gcc

  yum install python-devel

  yum install gcc

Single file compilation

• Command line compilation

  cythonize -i 文件名
  

• Based on setup.py

   from setuptools import setup
   from Cython.Build import cythonize
  
   setup(
       name = 'test',
       ext_modules = cythonize("test.pyx"),
       zip_safe = False,
   )
  

  python setup.py build_ext --inplace
  

Compile the entire project

# -*- coding: utf-8 -*-
"""

执行前提：
     系统安装python-devel 和 gcc
     Python安装cython
 
编译整个当前目录：
     python py-setup.py
编译某个文件夹：
     python py-setup.py BigoModel
 
生成结果：
     目录 build 下
 
生成完成后：
     启动文件还需要py/pyc担当，须将启动的py/pyc拷贝到编译目录并删除so文件
 
"""
 
import sys, os, shutil, time
from distutils.core import setup
from Cython.Build import cythonize
 
starttime = time.time()
currdir = os.path.abspath('.')
if len(sys.argv) >= 2:    
    path_list = sys.argv[1:]
else:
    path_list = ['.']
print("======")
print("build path:", path_list)
print("======")
#parentpath = sys.argv[1] if len(sys.argv)>1 else ""
#if parentpath == '':
#    print("Error:CopyDir is null!")
#    sys.exit(1)

setupfile= os.path.join(os.path.abspath('.'), __file__)
build_dir = "build"
build_tmp_dir = build_dir + "/temp"
 
def getpy(parentpath='', name='', excepts=(), copyOther=False,delC=False, except_fname=()):
    """
    获取py文件的路径
    :param parentpath: 父路径
    :param name: 文件/夹
    :param excepts: 排除文件
    :param copyOther: 是否copy其他文件
    :param delC: 是否删除C文件
    :param except_fname: 排除的文件名
    :return: py文件的迭代器
    """
    fullpath = os.path.join(parentpath, name)
    for fname in os.listdir(fullpath):
        if fname in except_fname:
            continue
        ffile = os.path.join(fullpath, fname)
        #print('{}\n{}\n{}'.format(parentpath, name, fname))
        #print('-----')
        if os.path.isdir(ffile) and fname != build_dir and not fname.startswith('.'):
            if fname == "__pycache__":
                continue
            for f in getpy(os.path.join(parentpath, name), fname, excepts, copyOther, delC, except_fname):
                yield f
        elif os.path.isfile(ffile):
            ext = os.path.splitext(fname)[1]
            if ext == ".c":
                if delC and os.stat(ffile).st_mtime > starttime:
                    os.remove(ffile)
            elif ffile not in excepts and os.path.splitext(fname)[1] not in('.pyc', '.pyx'):
                if os.path.splitext(fname)[1] in('.py', '.pyx') and not fname.startswith('__'):
                    yield os.path.join(parentpath, name, fname)
                if fname == '__init__.py' and copyOther:
                    dstdir = os.path.join(build_dir, parentpath, name)
                    if not os.path.isdir(dstdir): 
                        os.makedirs(dstdir)
                    shutil.copyfile(ffile, os.path.join(dstdir, fname))
            if copyOther:
                if ffile in excepts:
                    continue
                exclude_suffix = ('.pyc', '.pyx', '.py', '.c')
                if os.path.splitext(fname)[1] in exclude_suffix\
                    or fname.startswith('__'):
                    continue
                dstdir = os.path.join(build_dir, parentpath, name)
                if not os.path.isdir(dstdir): 
                    os.makedirs(dstdir)
                shutil.copyfile(ffile, os.path.join(dstdir, fname))
        else:
            print('#pass [%s]...'%(ffile))
            pass


if __name__ == "__main__": 
    
    for parentpath in path_list:
        #获取py列表
        module_list = list(getpy(parentpath=parentpath, excepts=(setupfile), except_fname=("jupyter_config.py")))
        print(module_list)
        try:
            setup(ext_modules = cythonize(module_list, language_level=3),
               script_args=["build_ext", "-b", build_dir, "-t", build_tmp_dir]
               )
        except Exception as e:
             print (e)
        else:
             module_list = list(getpy(parentpath=parentpath, excepts=(setupfile), copyOther=True))
        module_list = list(getpy(parentpath=parentpath, excepts=(setupfile), delC=True))
        if os.path.exists(build_tmp_dir): 
            shutil.rmtree(build_tmp_dir)
        
        print('## setup [{}] over ...'.format(parentpath))
                
    print("complate! time:", time.time()-starttime, 's')

After compilation is complete, make the call.

import main

app = main.app
app.run(host='0.0.0.0',port=8037)