人员类型历史数据处理接口开发

puhanshu

2022-01-19 6d5f67ea34ecafcb5c7266bccf32c7d98e597554

人员类型历史数据处理接口开发

 springcloud_k8s_panzhihuazhihuishequ/common/src/main/java/com/panzhihua/common/listen/ComMngPopulationTempFilledExcelListen.java

New file
@@ -0,0 +1,238 @@
package com.panzhihua.common.listen;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;

import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSONArray;
import com.google.common.collect.Lists;
import com.panzhihua.common.constants.Constants;
import com.panzhihua.common.enums.PopulPersonTypeEnum;
import com.panzhihua.common.exceptions.ServiceException;
import com.panzhihua.common.model.vos.R;
import com.panzhihua.common.model.vos.community.ComMngPopulationMistakeExcelVO;
import com.panzhihua.common.model.vos.community.ComMngPopulationServeExcelVO;
import com.panzhihua.common.service.community.CommunityService;
import com.panzhihua.common.utlis.ListUtils;
import com.panzhihua.common.utlis.PayUtil;
import com.panzhihua.common.utlis.StringUtils;

import lombok.extern.slf4j.Slf4j;

/**
 * @title: ComMngPopulationTempFilledExcelListen
 * @projectName: 成都呐喊信息技术有限公司-智慧社区项目
 * @description: 实有人口临时填充人员类型
 * @author: hans
 * @date: 2022/01/19 13:33
 */
@Slf4j
public class ComMngPopulationTempFilledExcelListen extends AnalysisEventListener<Map<Integer, String>> {
    /**
     * 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
     */
    private static final int BATCH_COUNT = 5000;
    private static int headSize = 0;
    List<Map<Integer, String>> list = new ArrayList<Map<Integer, String>>();
    private CommunityService communityService;
    private Long communityId;
    private Long userId;
    private Map<Integer, String> headData;
    private StringRedisTemplate stringRedisTemplate;

    public ComMngPopulationTempFilledExcelListen(CommunityService communityService, Long communityId, Long userId,
                                                 StringRedisTemplate stringRedisTemplate) {
        this.communityService = communityService;
        this.communityId = communityId;
        this.userId = userId;
        this.stringRedisTemplate = stringRedisTemplate;
    }

    @Override
    public void invoke(Map<Integer, String> data, AnalysisContext context) {
        list.add(data);
        if (list.size() >= BATCH_COUNT) {
            saveData();
            list.clear();
        }
    }

    /**
     * 这里会一行行的返回头
     *
     * @param headMap
     * @param context
     */
    @Override
    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
        headSize = headMap.size();
        headData = headMap;
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext context) {
        saveData();
        log.info("所有数据解析完成！");
    }

    /**
     * 不是固定的列只能手动处理
     */
    private void saveData() {
        log.info("开始填充历史数据人员类型");
        log.info("表格总数据：" + list.size());
        if (list.size() == 0) {
            throw new ServiceException("000", "数据为空！");
        }
        ValueOperations<String, String> valueOperations = stringRedisTemplate.opsForValue();
        String key = "POPULATION_FILLED_ERROR_LIST";

        int index = 2;
        try {
            ArrayList<ComMngPopulationServeExcelVO> voList = Lists.newArrayList();
            ArrayList<ComMngPopulationMistakeExcelVO> mistakes = Lists.newArrayList();

            for (Map<Integer, String> oneData : list) {
                ComMngPopulationServeExcelVO vo = new ComMngPopulationServeExcelVO();
                // 姓名和身份证都为空,为空户，无需操作,否则就解析年龄，性别，出生年月日
                if (StringUtils.isNotEmpty(oneData.get(0)) || StringUtils.isNotEmpty(oneData.get(1))) {
                    if (StringUtils.isEmpty(oneData.get(0))) {
                        ComMngPopulationMistakeExcelVO mistake = new ComMngPopulationMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("名字不可为空，请填写姓名");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setName(oneData.get(0));
                    if (StringUtils.isEmpty(oneData.get(1))) {
                        ComMngPopulationMistakeExcelVO mistake = new ComMngPopulationMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("身份证号不可为空，请填写身份证号");
                        mistakes.add(mistake);
                        continue;
                    }

                    // 判断身份证号码位数
                    if (oneData.get(1).length() != 18) {
                        index++;
                        ComMngPopulationMistakeExcelVO mistake = new ComMngPopulationMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("身份证号位数有误，请检查身份证号码是否正确");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setCardNo(oneData.get(1).toUpperCase());
                }
                if (StringUtils.isNotEmpty(oneData.get(2))) {
                    Integer isOk = PopulPersonTypeEnum.getCodeByName(oneData.get(2));
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMistakeExcelVO mistake = new ComMngPopulationMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的人员类型有误");
                        mistake.setPersonType(oneData.get(2));
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setPersonType(isOk);
                }
                // 将重复的数据进行MD5加密实现去重
                String distinct = vo.getName() + vo.getCardNo();
                try {
                    String distinctPass = PayUtil.MD5(distinct);
                    if (StringUtils.isNotEmpty(distinctPass)) {
                        vo.setDistinctPass(distinctPass);
                    }
                } catch (Exception e) {
                    log.error("组装MD5加密字段失败，数据表格行数：" + index);
                    continue;
                }
                voList.add(vo);
                index++;
            }
            // 根据list中的IdCard城市来去重
            List<ComMngPopulationServeExcelVO> newVoList =
                voList.stream().filter(ListUtils.distinctByKey(ComMngPopulationServeExcelVO::getDistinctPass))
                    .collect(Collectors.toList());
            log.info("开始进入业务层处理逻辑");
            R r = communityService.filledPopulationPersonType(newVoList, communityId, userId);
            log.info("业务层处理逻辑完成");
            if (!R.isOk(r)) {
                log.info("业务层处理成功");
                List<ComMngPopulationMistakeExcelVO> list =
                    JSONArray.parseArray(JSONArray.toJSONString(r.getData()), ComMngPopulationMistakeExcelVO.class);
                mistakes.addAll(list);
                log.info("将错误数据存入redis中");
                valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS);
                log.info("将错误数据存入redis中成功");
                throw new ServiceException("500", key);
            } else {
                log.info("业务层处理逻辑失败");
                if (!mistakes.isEmpty()) {
                    log.info("业务层处理逻辑失败，将错误数据缓存到redis中");
                    valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS);
                    log.info("业务层处理逻辑失败，将错误数据缓存到redis中成功");
                    throw new ServiceException("500", key);
                }
            }
        } catch (NumberFormatException e) {
            log.info("处理数据时失败");
            e.printStackTrace();
            log.error("数据格式有误，第" + index + "行");
            throw new ServiceException("500", "导入失败111");
        }
    }

    private void setMistake(Map<Integer, String> map, ComMngPopulationMistakeExcelVO vo) {
        vo.setName(map.get(0));
        vo.setCardNo(map.get(1));
        vo.setNation(map.get(2));
        vo.setPoliticalOutlook(map.get(3));
        vo.setIsRent(map.get(4));
        vo.setRelation(map.get(5));
        vo.setRoad(map.get(6));
        vo.setDoorNo(map.get(7));
        vo.setFloor(map.get(8));
        vo.setUnitNo(map.get(9));
        vo.setHouseNo(map.get(10));
        vo.setBuildPurpose(map.get(11));
        vo.setBuildArea(map.get(12));
        vo.setHouseStatus(map.get(13));
        vo.setHousePurpose(map.get(14));
        vo.setControlStatus(map.get(15));
        vo.setPhone(map.get(16));
        vo.setNativePlace(map.get(17));
        vo.setCultureLevel(map.get(18));
        vo.setMarriage(map.get(19));
        vo.setHealthy(map.get(20));
        vo.setBloodType(map.get(21));
        vo.setReligion(map.get(22));
        vo.setProfession(map.get(23));
        vo.setWorkCompany(map.get(24));
        vo.setOutOrLocal(map.get(25));
        vo.setCensusRegister(map.get(26));
        vo.setResidence(map.get(27));
        vo.setPersonType(map.get(28));
        vo.setCountry(map.get(29));
        vo.setStringOfDeparture(map.get(30));
        vo.setPersonStatus(map.get(31));
        vo.setMonthlyIncome(map.get(32));
        vo.setFamilyStatus(map.get(33));
        vo.setGoalInChina(map.get(34));
        vo.setStringOfArrival(map.get(35));
        vo.setRemark(map.get(36));
        vo.setIdCardPositive(map.get(37));
        vo.setIdCardBack(map.get(38));
        vo.setHouseHold(map.get(39));
        vo.setDeath(map.get(40));
    }
}

 springcloud_k8s_panzhihuazhihuishequ/common/src/main/java/com/panzhihua/common/service/community/CommunityService.java

@@ -8262,4 +8262,15 @@

    @GetMapping("/reserve/record/delete")
    R deleteRecord(@RequestParam("recordId")Long recordId);

    /**
     * 历史数据人员类型填充
     * @param newVoList
     * @param communityId
     * @param userId
     * @return
     */
    @PostMapping("/common/data/history/filled")
    R filledPopulationPersonType(@RequestBody List<ComMngPopulationServeExcelVO> newVoList,
                                 @RequestParam("communityId") Long communityId, @RequestParam("userId") Long userId);
}

 springcloud_k8s_panzhihuazhihuishequ/community_backstage/src/main/java/com/panzhihua/community_backstage/api/PopulationApi.java

@@ -10,6 +10,7 @@
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;

import com.panzhihua.common.exceptions.ServiceException;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
@@ -107,6 +108,28 @@
        return R.ok();
    }

    @ApiOperation(value = "excel填充旧数据的人员类型")
    @PostMapping(value = "/history/filled", consumes = "multipart/*", headers = "content-type=multipart/form-date")
    public R filledPopulationPersonType(@RequestParam MultipartFile file, HttpServletRequest request) {
        // 获取文件名
        String fileName = file.getOriginalFilename();
        log.info("传入文件名字【{}】", fileName);
        InputStream inputStream = null;
        try {
            inputStream = file.getInputStream();
            ComMngPopulationTempFilledExcelListen excelListen = new ComMngPopulationTempFilledExcelListen(
                    communityService, this.getCommunityId(), this.getLoginUserInfo().getUserId(), stringRedisTemplate);
            EasyExcel.read(inputStream, null, excelListen).sheet().doRead();
        } catch (IOException e) {
            log.error("填充旧数据的人员类型【{}】", e.getMessage());
            e.printStackTrace();
        } catch (ServiceException e) {
            log.error("错误数据【{}】", e.getMessage());
            return downloadErrorPopulation("POPULATION_FILLED_ERROR_LIST");
        }
        return R.ok();
    }

    @ApiOperation(value = "excel导入吸毒人员")
    @PostMapping(value = "/serve/importDrug", consumes = "multipart/*", headers = "content-type=multipart/form-date")
    public R importDrug(@RequestParam MultipartFile file, HttpServletRequest request) {

 springcloud_k8s_panzhihuazhihuishequ/service_community/src/main/java/com/panzhihua/service_community/api/CommonDataApi.java

@@ -946,4 +946,17 @@
    public R detailHousesUser(@RequestParam("id") Long id) {
        return comMngPopulationHouseUserService.detail(id);
    }

    /**
     * 历史数据人员类型填充
     * @param newVoList
     * @param communityId
     * @param userId
     * @return
     */
    @PostMapping("/history/filled")
    public R filledPopulationPersonType(@RequestBody List<ComMngPopulationServeExcelVO> newVoList,
                                 @RequestParam("communityId") Long communityId, @RequestParam("userId") Long userId) {
        return comMngPopulationService.filledPopulationPersonType(newVoList, communityId, userId);
    }
}

 springcloud_k8s_panzhihuazhihuishequ/service_community/src/main/java/com/panzhihua/service_community/service/ComMngPopulationService.java

@@ -538,4 +538,13 @@
     * @return
     */
    R indexInfo(Long communityId);

    /**
     * 历史数据人员类型填充
     * @param newVoList
     * @param communityId
     * @param userId
     * @return
     */
    R filledPopulationPersonType(List<ComMngPopulationServeExcelVO> newVoList, Long communityId, Long userId);
}

 springcloud_k8s_panzhihuazhihuishequ/service_community/src/main/java/com/panzhihua/service_community/service/impl/ComMngPopulationServiceImpl.java

@@ -7983,6 +7983,66 @@
        return R.ok(indexInfo);
    }

    /**
     * 历史数据人员类型填充
     * @param list
     * @param communityId
     * @param userId
     * @return
     */
    @Override
    @Transactional(rollbackFor = Exception.class)
    public R filledPopulationPersonType(List<ComMngPopulationServeExcelVO> list, Long communityId, Long userId) {
        // 需要修改的人口集合
        List<ComMngPopulationDO> updateList = new ArrayList<>();
        log.info("开始处理导入数据");
        List<ComMngPopulationMistakeExcelVO> mistakes = new ArrayList<>();
        try {
            // 查询所有人口数据放入HashMap中
            List<ComMngPopulationDO> populationList = this.baseMapper.selectList(null);
            HashMap<String, Object> populationMap = new HashMap<>();
            populationList.forEach(population -> {
                String key = population.getCardNo();
                populationMap.put(key, population);
            });
            for (ComMngPopulationServeExcelVO vo : list) {
                // 判断实有人口是否已存在
                log.info("开始查询实有人口是否已存在");
                ComMngPopulationDO populationDO = null;
                String populationKey = vo.getCardNo();
                if (!isOnly(populationKey, populationMap)) {
                    // 存在实有人口信息，则更新人员类型
                    populationDO = (ComMngPopulationDO)populationMap.get(populationKey);
                    Integer personType = vo.getPersonType();
                    if (nonNull(personType)) {
                        populationDO.setPersonType(personType.toString());
                    }
                    populationDO.setUpdateBy(userId);
                    updateList.add(populationDO);
                } else {
                    // 不存在实有人口,返回
                    ComMngPopulationMistakeExcelVO mistake = new ComMngPopulationMistakeExcelVO();
                    BeanUtils.copyProperties(vo, mistake);
                    setMistake(mistake, vo);
                    mistake.setMistake("人口信息不存在，请下载实有人口模板导入");
                    mistakes.add(mistake);
                }
            }
        } catch (Exception e) {
            log.info("出现错误，错误原因：" + e.getMessage());
        }
        if (!updateList.isEmpty()) {
            log.info("执行数据库更新人口");
            this.baseMapper.updateAll(updateList);
            log.info("数据库更新人口完成");
        }
        if (!mistakes.isEmpty()) {
            log.info("返回错误数据");
            return R.fail(mistakes);
        }
        return R.ok(mistakes);
    }

    private void setMistake(ComMngPopulationMistakeExcelVO mvo, ComMngPopulationServeExcelVO vo) {
        mvo.setPoliticalOutlook(PopulPoliticalOutlookEnum.getCnDescByName(vo.getPoliticalOutlook()));
        mvo.setIsRent(PopulHouseUseEnum.getCnDescByName(vo.getIsRent()));