tp3批量处理几万条数据

最近接到了退券任务，要求把几万条不同类型的券码值批量处理，主要的类型有三种，并保存在不同的表中，退券时要修改券码表中的状态，同时将操作过程记录下来，主要问题在于几万条数据先查询，然后分类插入数据库，一条一条处理时间太慢，所以我将数据读取到内存，然后处理，将结果整合重新入库。

8108684592429189  //券码值直接保存在券码表中，券码表也是最终要操作的表

http://t.cn/Rx2GM2q   //短链接是旧连接，数据表中存在大小写不同的数据，还要提取出来过滤一遍

http://v.51jifengou.com/t8209952787309387   //长链接存在于长链接表中，后边16位为券码值，需要截取字符串。

首先对三种数据通过正则分类，分别通过不同的表进行处理，然后将处理好的数据整合，批量插入表中。下面直接贴上代码。

contraller层：

public function voucherRefundList(){
        $type = I('post.type', '');
        if(empty($type) || empty($_FILES['source']['name'])){
            $this->error('缺少必要参数');
        }
        $kbpackageBusines = new KbPackageBusiness();
        $res = $kbpackageBusines->importVoidPackageTxt($type);
        
        if (count($res['data']) != count(array_unique($res['data']))) {
            $unique_arr = array_unique ( $res['data'] );
            // 获取重复数据的数组
            $repeat_arr = array_diff_assoc ( $res['data'], $unique_arr );
            foreach ($repeat_arr as $v=>$k){
                $res = $k;
            }
            $this->error('有重复数据！'.$res,U('voucherRefund'),'8');
        }
        //全部类型数据一起判断
        $VoucherBusiness = new VoucherBusiness;
        $res = $VoucherBusiness->upVoucherInfo($type,$res['data']);
        
        if($res!= 1&& $res!= 0){
            $this->error(count($res).'张错误券码值');
        }elseif ($res == 1){
            $this->success('操作成功！',U('voucherRefund'));
        }else{
            $this->error('操作失败');
        }

Business层：
public function upVoucherInfo($type,$res){
        foreach ($res as $k){
            if(preg_match("/^[0-9]*$/",$k) == 1){     //券码值
                $voucherData[] =  $k;
            }elseif(preg_match("/^(http:\/\/t.cn).*$/",$k)){     //短链接
                $sortUrlData[] = $k;
            }elseif(preg_match("/^(http:\/\/v.51jifengou.com).*$/",$k)) {     //长链接
                //$urlData[] = $k;
                $urlData[] = preg_match("/\d{16}/",$k,$arr);
                $urlNum['voucher_num'] = (string)$arr[0];
                $urlNum['short_url'] = $k;
                $urlVoucherNum[] = $urlNum;
            }else{
                $noData[] = $k;   //错误数据
            }
        }
        if($noData != null){    //返回错误券码
            return $noData;
        }
        //短链接的voucher_num
        $map['short_url'] = ['in', $sortUrlData];
        $sortUrlInfo = M("sys_voucher")->where($map)->getField('id,voucher_num,short_url');
        
        $arr2 = array_column($sortUrlInfo, 'short_url');
        $arr = array_diff($arr2,$sortUrlData);   //大小写原因多的数据
        
        foreach ($sortUrlInfo as $k){
            $urlVoucherData[] = $k;
        }
        
        foreach ($arr as $k){
            $key = array_search($k,$arr2);
            unset($urlVoucherData[$key]);     //短链接数据
        }

        $VoucherService =new VoucherService();
        $res = $VoucherService->voucherRefund($urlVoucherNum,$urlVoucherData,$voucherData,$type);
        return $res;

    }

Service层：
public function voucherRefund($urlVoucherNum,$urlVoucherData,$voucherData,$type){   //长链接、短链接、券码值、状态
        //将所有类型数组整合成一个数组$allVoucherData
        foreach($urlVoucherNum as $v){
            $list['short_url'] = $v['short_url'];
            $list['voucher_num'] = $v['voucher_num'];
            $allVoucherData[] = $list;
        }
        foreach($urlVoucherData as $v){
            $list['short_url'] = $v['short_url'];
            $list['voucher_num'] = $v['voucher_num'];
            $allVoucherData[] = $list;
        }
        foreach($voucherData as $v){
            $list['short_url'] = null;
            $list['voucher_num'] = $v;
            $allVoucherData[] = $list;
        }
        //合并所有的voucher_num
        $voucherNum =array_merge(array_column($urlVoucherNum, 'voucher_num'),array_column($urlVoucherData, 'voucher_num'),$voucherData);
        //判断数组和链接的重复值
        if (count($voucherNum) != count(array_unique($voucherNum))) {
            $unique_arr = array_unique ( $voucherNum );
            // 获取重复数据的数组
            $repeat_arr = array_diff_assoc ( $voucherNum, $unique_arr );
            foreach ($repeat_arr as $v=>$k){
                $res = $k;
            }
            return $res;
        }
        
        $map1['voucher_num']  = array('in',$voucherNum);
        $map1['state'] = 1;
        $map2['state'] = $type;
        $updata = M("user_voucher")->where($map1)->getField('voucher_num',true);

        $date = M("user_voucher")->where($map1)->save($map2); // 根据条件更新记录
        
        $admin_id = session('user_auth.uid');//获取操作员id
        $arr = array_diff($voucherNum,$updata);  //查询不同值

        $res3['up_number'] = count($voucherNum);
        $res3['updata_number'] = count($updata);
        $res3['admin_id'] = $admin_id;

        $rest =  M("user_voucher_refund_log")->add($res3);
        foreach ($allVoucherData as $k){
            if(in_array($k['voucher_num'], $arr)){
                $map['voucher_refund_logid'] = $rest;   //$rest对应上边的$rest操作id
                $map['short_url'] = $k['short_url'];
                $map['voucher_num'] = $k['voucher_num'];
                $map['state'] = 0;
            }else{
                $map['voucher_refund_logid'] = $rest;  //$rest对应上边的$rest操作id
                $map['short_url'] = $k['short_url'];
                $map['voucher_num'] = $k['voucher_num'];
                $map['state'] = 1;
            }
            $mapall[] = $map;
        }

        $return = M("user_voucher_refund_detail")->addAll($mapall);
        return 1;
    }

tp3批量处理几万条数据

猜你喜欢