PHP超过10w条数据查询导出

328 阅读1分钟
从数据库中一次查出10w条数据并且导出来,可以优化点
1、一定要用到对应数据表索引
2、坚决分块查询,不管是chunk还是for循环还是分页
3、写入csv的时候,也要分块写入,如果需要处理数据,回调函数处理
4、每次写入一定量的csv,就要ob_flush一下缓存


$pageSize = 10000; //每次处理10000条数据
$page = ceil($countPhoneIds / $pageSize);

ob_start();
header('Content-Type: text/csv');
header('Content-Disposition: attachment;filename="outbound' . date('YmdHi') . '.csv"');
//写入header
$fp = fopen('php://output', 'a');
fwrite($fp,chr(0xEF).chr(0xBB).chr(0xBF));
$header = AutoOutboundTaskExportForm::EXPORT_HEADER_MAP[$form->status];

for ($loop = 1; $loop <= $page; $loop++) {
    $exportPhoneList = AutoOutboundTaskList::getPhoneList($form->taskId, $form->companyId, $form->phoneIds, $page, $pageSize)->transform(function (AutoOutboundTaskList $item) use ($phonePermission, $status, &$userDataKeys) {

    })->toArray();
    $exportData = $exportPhoneList['data'];
    if ($loop == 1) {
        array_splice($header, 1, 0, $userDataKeys);
        fputcsv($fp,$header);
    }
    foreach ($exportData as $datum) {
        fputcsv($fp, $datum);
    }
    ob_flush();
}
ob_end_flush();
fclose($fp);
exit;