多进程运行因子

from _config import config
from factor_generator import FactorGenerator
from con_factor import *
import warnings
from alphas_101 import *
from multiprocessing import  Pool, Process

# pd.set_option('display.max_rows', None)
warnings.filterwarnings('ignore')
def read_data():
    generator = FactorGenerator(config)
    #读取历史数据
    df = generator.load_data()
    data_map, date_range_index, range_columns \
        = generator.transform(df,'2018-1-1', '2022-1-1')
    #读取聚宽的数据
    jk_df = generator.load_jk_data()
    jk_data_map = generator.jk_transform(jk_df, date_range_index, range_columns)
    conv = ConFactor(data_map,jk_data_map=jk_data_map)
    return conv

def calculate_alphas(name,instance):
    start = time.time()
    print('Task %s started'% name)
    conv = instance
    df = eval('conv'+'.'+name+'()')
    # alpha_dfs[name] =df
    end = time.time()
    # print(df)
    print('Task %s end in  %0.2f seconds.' % (name, (end - start)))
    return (name,df)

def err_call_back(err):
        print(f'出错啦~ error:{str(err)}')   

def func_call_back(res):
    # global alpha_dfs
    # alpha_dfs={}
    # alpha_dfs[res[0]]=res[1]
    res[1].to_csv(os.path.join(r'C:\Users\Meeting\Documents\StockData_6_7\alpha_raw_data'\
                               ,res[0]+'.csv'))

if __name__=="__main__":
    alpha_dfs={}
    conv = read_data()
    alpha_names = [i for i in dir(ConFactor) if i.startswith('alpha')]

    pool = Pool(5) #创建一个5个进程的进程池

    for name in alpha_names:
        pool.apply_async(func=calculate_alphas, args=(name,conv),\
                         error_callback=err_call_back,callback=func_call_back)

    pool.close()
    pool.join()
    # print(alpha_names)
    print('结束测试')