Merge remote-tracking branch 'origin/zzj' into zzj

huanghongfa

2021-09-02 177249c76aeea0b4bf8d8816d4994e3b445b45ce

 springcloud_k8s_panzhihuazhihuishequ/common/src/main/java/com/panzhihua/common/listen/ComMngPopulationMajorExcelListen.java

@@ -1,35 +1,5 @@
package com.panzhihua.common.listen;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
import com.alibaba.fastjson.JSONArray;
import com.google.common.collect.Lists;
import com.panzhihua.common.constants.BcDictionaryConstants;
import com.panzhihua.common.constants.Constants;
import com.panzhihua.common.enums.*;
import com.panzhihua.common.excel.CustomSheetWriteHandler;
import com.panzhihua.common.exceptions.ServiceException;
import com.panzhihua.common.model.vos.BcDictionaryVO;
import com.panzhihua.common.model.vos.R;
import com.panzhihua.common.model.vos.community.*;
import com.panzhihua.common.service.community.CommunityService;
import com.panzhihua.common.utlis.ListUtils;
import com.panzhihua.common.utlis.PayUtil;
import com.panzhihua.common.utlis.SFTPUtil;
import com.panzhihua.common.utlis.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -37,417 +7,426 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;

import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSONArray;
import com.google.common.collect.Lists;
import com.panzhihua.common.constants.BcDictionaryConstants;
import com.panzhihua.common.constants.Constants;
import com.panzhihua.common.enums.*;
import com.panzhihua.common.exceptions.ServiceException;
import com.panzhihua.common.model.vos.BcDictionaryVO;
import com.panzhihua.common.model.vos.R;
import com.panzhihua.common.model.vos.community.ComMngPopulationMajorExcelVO;
import com.panzhihua.common.model.vos.community.ComMngPopulationMajorMistakeExcelVO;
import com.panzhihua.common.service.community.CommunityService;
import com.panzhihua.common.utlis.ListUtils;
import com.panzhihua.common.utlis.PayUtil;
import com.panzhihua.common.utlis.StringUtils;

import lombok.extern.slf4j.Slf4j;

/**
 * @description: 重精人员导入监听
 * @author: txb
 */
@Slf4j
public class ComMngPopulationMajorExcelListen extends AnalysisEventListener<Map<Integer, String>> {
    private CommunityService communityService;

    private Long communityId;

    /**
     * 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
     */
    private static final int BATCH_COUNT = 5000;
    private static int headSize = 0;

    List<Map<Integer, String>> list = new ArrayList<Map<Integer, String>>();
    private CommunityService communityService;
    private Long communityId;
    private Long userId;
    private Map<Integer, String> headData;

    private String userName;
    // FTP 登录密码
    private String password;
    // FTP 服务器地址IP地址
    private String host;
    // FTP 端口
    private int port;
    private String excelUrl;

    private StringRedisTemplate stringRedisTemplate;

    public static String loadUrl;
    public ComMngPopulationMajorExcelListen(CommunityService communityService, Long communityId, Long userId,
        StringRedisTemplate stringRedisTemplate) {
        this.communityService = communityService;
        this.communityId = communityId;
        this.userId = userId;
        this.stringRedisTemplate = stringRedisTemplate;
    }

    public ComMngPopulationMajorExcelListen(CommunityService communityService, Long communityId, String userName, String password, String host, int port, String excelUrl, StringRedisTemplate stringRedisTemplate) {
            this.communityService = communityService;
            this.communityId = communityId;
            this.userName = userName;
            this.password = password;
            this.host = host;
            this.port = port;
            this.excelUrl = excelUrl;
            this.stringRedisTemplate = stringRedisTemplate;
        }

        /**
         * 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
         */
        private static final int BATCH_COUNT = 5000;
        List<Map<Integer, String>> list = new ArrayList<Map<Integer, String>>();

        @Override
        public void invoke(Map<Integer, String> data, AnalysisContext context) {
            list.add(data);
            if (list.size() >= BATCH_COUNT) {
                saveData();
                list.clear();
            }
        }

        /**
         * 这里会一行行的返回头
         *
         * @param headMap
         * @param context
         */
        @Override
        public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
            headSize = headMap.size();
            headData = headMap;
        }

        @Override
        public void doAfterAllAnalysed(AnalysisContext context) {
    @Override
    public void invoke(Map<Integer, String> data, AnalysisContext context) {
        list.add(data);
        if (list.size() >= BATCH_COUNT) {
            saveData();
            log.info("所有数据解析完成！");
            list.clear();
        }
    }

        /**
         * 不是固定的列只能手动处理
         */
        private void saveData() {
            R<List<BcDictionaryVO>> dictionaryR = communityService.listDictionaryByKey(BcDictionaryConstants.FAMILY);
            log.info("开始导入重精人员数据");
            log.info("表格总数据：" + list.size());
            if (list.size() == 0) {
                throw new ServiceException("000", "导入数据为空！");
            }
            ValueOperations<String, String> valueOperations = stringRedisTemplate.opsForValue();
            String key = Constants.MAJOR_POPULATION_ERROR_LIST;
    /**
     * 这里会一行行的返回头
     *
     * @param headMap
     * @param context
     */
    @Override
    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
        headSize = headMap.size();
        headData = headMap;
    }

            Map<String,String> dictMap = dictionaryR.getData().stream().collect(Collectors.toMap(BcDictionaryVO::getDictName,BcDictionaryVO::getDictValue));
//        Map<String,Integer> dictMap = new HashMap<>();
//        for(BcDictionaryVO vo : dictionaryR.getData()){
//            dictMap.put(vo.getDictName(),Integer.valueOf(vo.getDictValue()));
//        }
    @Override
    public void doAfterAllAnalysed(AnalysisContext context) {
        saveData();
        log.info("所有数据解析完成！");
    }

    /**
     * 不是固定的列只能手动处理
     */
    private void saveData() {
        R<List<BcDictionaryVO>> dictionaryR = communityService.listDictionaryByKey(BcDictionaryConstants.FAMILY);
        log.info("开始导入重精人员数据");
        log.info("表格总数据：" + list.size());
        if (list.size() == 0) {
            throw new ServiceException("000", "导入数据为空！");
        }
        ValueOperations<String, String> valueOperations = stringRedisTemplate.opsForValue();
        String key = Constants.MAJOR_POPULATION_ERROR_LIST;

            int index = 2;
            try {
                ArrayList<ComMngPopulationMajorExcelVO> voList = Lists.newArrayList();
//                List<ComMngPopulationImportErrorVO> populationImportErrorVOList = new ArrayList<>();
                ArrayList<ComMngPopulationMajorMistakeExcelVO> mistakes = Lists.newArrayList();
                for (Map<Integer, String> oneData : list) {
                    ComMngPopulationMajorExcelVO vo = new ComMngPopulationMajorExcelVO();
                    if (StringUtils.isNotEmpty(oneData.get(0))){
                        vo.setSerialNumber(oneData.get(0));
                    }
                    //姓名和身份证都为空,为空户，无需操作,否则就解析年龄，性别，出生年月日
                    if(StringUtils.isEmpty(oneData.get(1))){
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("名字不可为空，请填写姓名");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setName(oneData.get(1));
                    if(StringUtils.isEmpty(oneData.get(2))){
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("身份证号不可为空，请填写身份证");
                        mistakes.add(mistake);
                        continue;
                    }
        Map<String, String> dictMap = dictionaryR.getData().stream()
            .collect(Collectors.toMap(BcDictionaryVO::getDictName, BcDictionaryVO::getDictValue));
        // Map<String,Integer> dictMap = new HashMap<>();
        // for(BcDictionaryVO vo : dictionaryR.getData()){
        // dictMap.put(vo.getDictName(),Integer.valueOf(vo.getDictValue()));
        // }

                    //判断身份证号码位数
                    if(oneData.get(2).length() != 18){
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("身份证号位数有误，请检查身份证号码是否正确");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setCardNo(oneData.get(2).toUpperCase());
                    //根据身份证号码解析年龄以及性别
                    //获取用户生日
                    String birthday = vo.getCardNo().substring(6, 14);
                    if(StringUtils.isNotEmpty(birthday)){
                        String year = birthday.substring(0,4);
                        String month = birthday.substring(4,6);
                        String day = birthday.substring(6,8);
                        vo.setBirthday(year + "-" + month + "-" + day);
                    }
//                    //设置用户年龄
//                    vo.setAge(AgeUtils.getAgeFromBirthTime(birthday));
                    //获取用户性别
                    int sex = Integer.parseInt(vo.getCardNo().substring(16, 17));
                    if(sex%2 == 1){
                        vo.setSex(PopulSexEnum.nan.getCode());
                    }else{
                        vo.setSex(PopulSexEnum.nv.getCode());
                    }

                    if(StringUtils.isEmpty(oneData.get(3))){
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("街路巷不可为空");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setRoad(oneData.get(3));
                    if(StringUtils.isEmpty(oneData.get(4))){
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("小区号不可为空");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setDoorNo(oneData.get(4).trim());
//                    if(StringUtils.isEmpty(oneData.get(5))){
//                        index++;
//                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
//                        setMistake(oneData, mistake);
//                        mistake.setMistake("楼排号不可为空");
//                        mistakes.add(mistake);
//                        continue;
//                    }
                    if(StringUtils.isNotEmpty(oneData.get(5))) {
                        vo.setFloor(oneData.get(5).trim());
                    }
//                    if(StringUtils.isEmpty(oneData.get(6))){
//                        index++;
//                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
//                        setMistake(oneData, mistake);
//                        mistake.setMistake("单元号不可为空");
//                        mistakes.add(mistake);
//                        continue;
//                    }
                    if(StringUtils.isNotEmpty(oneData.get(6))) {
                        vo.setUnitNo(oneData.get(6).trim());
                    }
//                    if(StringUtils.isEmpty(oneData.get(7))){
//                        index++;
//                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
//                        setMistake(oneData, mistake);
//                        mistake.setMistake("户室不可为空");
//                        mistakes.add(mistake);
//                        continue;
//                    }
                    if(StringUtils.isNotEmpty(oneData.get(7))) {
                        vo.setHouseNo(oneData.get(7).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(8))){
                        Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(8));
                        if(isOk.equals(-1)){
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            index++;
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的是否租住有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setIsRent(isOk);
                    }else {
                        vo.setIsRent(PopulHouseUseEnum.getCodeByName("否"));
                    }
                    if(StringUtils.isNotEmpty(oneData.get(9))){
                        Integer isOk = PopulHouseStatusEnum.getCodeByName(oneData.get(9).trim());
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的房屋状态有误");
                            mistakes.add(mistake);
                            continue;
                        }
                    vo.setHouseStatus(isOk);
                    }
                    if(StringUtils.isNotEmpty(oneData.get(10))){
                        Integer isOk = PopulHousePurposeEnum.getCodeByName(oneData.get(10).trim());
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的房屋用途有误");
                            mistakes.add(mistake);
                            continue;
                        }
                    vo.setHousePurpose(PopulHousePurposeEnum.getCodeByName(oneData.get(10).trim()));
                    }
                    if(StringUtils.isNotEmpty(oneData.get(11))){
                        Integer isOk = PopulHouseControlStatusEnum.getCodeByName(oneData.get(11));
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的管控状态有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setControlStatus(isOk);
                    }
                    if(StringUtils.isNotEmpty(oneData.get(12))){
                        Integer isOk = PopulOutOrLocalEnum.getCodeByName(oneData.get(12));
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的本地/外地有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setOutOrLocal(isOk);
                    }else {
                        vo.setOutOrLocal(PopulOutOrLocalEnum.getCodeByName("本地"));
                    }
                    if(StringUtils.isNotEmpty(oneData.get(13))){
                        vo.setCensusRegister(oneData.get(13).trim());
                    }
                    if(StringUtils.isEmpty(oneData.get(14))){
                        Integer isOk = PopulCultureLevelEnum.getCodeByName(oneData.get(14));
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的文化程度有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setOutOrLocal(isOk);
                    }
                    if(StringUtils.isNotEmpty(oneData.get(15))){
                        vo.setEconomicCondition(oneData.get(15).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(16))){
                        vo.setPersonType(oneData.get(16).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(17))){
                        vo.setFamilySituation(oneData.get(17).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(18))){
                        vo.setCustodyPerson(oneData.get(18).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(19))){
                        Integer isOk = PopulRelationEnum.getCodeByName(oneData.get(19));
                        if(isOk.equals(-1)){
                            index++;
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的与监护人关系有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setCustodyRelation(isOk);
                    }
                    if(StringUtils.isNotEmpty(oneData.get(20))){
                        vo.setCustodyCardNo(oneData.get(20).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(21))){
                        vo.setCustodyPersonPhone(oneData.get(21).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(22))){
                        vo.setCustodyAddress(oneData.get(22).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(23))){
                        vo.setHelpPerson(oneData.get(23).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(24))){
                        vo.setHelpPersonPhone(oneData.get(24).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(25))){
                        vo.setHelpMethod(oneData.get(25).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(26))){
                        vo.setHelpSituation(oneData.get(26).trim());
                    }
                    if(StringUtils.isNotEmpty(oneData.get(27))){
                        Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(27));
                        if(isOk.equals(-1)){
                            ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                            index++;
                            setMistake(oneData, mistake);
                            mistake.setMistake("您填写的是否违法犯罪有误");
                            mistakes.add(mistake);
                            continue;
                        }
                        vo.setIsCrime(isOk);
                    }else {
                        vo.setIsCrime(PopulHouseUseEnum.getCodeByName("否"));
                    }
                    if(StringUtils.isNotEmpty(oneData.get(28))){
                        vo.setCrimeSutiation(oneData.get(28).trim());
                    }
                    vo.getUserTagStr().add("精神障碍患者");
                    for (int i = 29; i < headSize; i++) {
                        if (oneData.get(i) != null && oneData.get(i).equals("是")) {
                            vo.getUserTagStr().add(headData.get(i).substring(0,headData.get(i).indexOf("(")));
                        }
                    }
                    //将重复的数据进行MD5加密实现去重
                    String distinct = vo.getName() + vo.getCardNo() + vo.getRoad() + vo.getDoorNo() + vo.getFloor() + vo.getUnitNo() + vo.getHouseNo();
                    try {
                        String distinctPass = PayUtil.MD5(distinct);
                        if(StringUtils.isNotEmpty(distinctPass)){
                            vo.setDistinctPass(distinctPass);
                        }
                    }catch (Exception e){
                        log.error("组装MD5加密字段失败，数据表格行数：" + index);
                        continue;
                    }
                    voList.add(vo);
        int index = 2;
        try {
            ArrayList<ComMngPopulationMajorExcelVO> voList = Lists.newArrayList();
            // List<ComMngPopulationImportErrorVO> populationImportErrorVOList = new ArrayList<>();
            ArrayList<ComMngPopulationMajorMistakeExcelVO> mistakes = Lists.newArrayList();
            for (Map<Integer, String> oneData : list) {
                ComMngPopulationMajorExcelVO vo = new ComMngPopulationMajorExcelVO();
                if (StringUtils.isNotEmpty(oneData.get(0))) {
                    vo.setSerialNumber(oneData.get(0));
                }
                // 姓名和身份证都为空,为空户，无需操作,否则就解析年龄，性别，出生年月日
                if (StringUtils.isEmpty(oneData.get(1))) {
                    ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                    index++;
                    setMistake(oneData, mistake);
                    mistake.setMistake("名字不可为空，请填写姓名");
                    mistakes.add(mistake);
                    continue;
                }
                //客户需要暂时注释，等客户处理完成需要恢复
//            if(populationImportErrorVOList.isEmpty()){
//                R r = communityService.listSavePopulationServeExcelVO(voList, communityId);
//                if (!R.isOk(r)) {
//                    throw new ServiceException(r.getMsg());
//                }
//            }else{
//                throw new ServiceException("500", JSON.toJSONString(populationImportErrorVOList));
//            }
                //根据list中的IdCard城市来去重
                List<ComMngPopulationMajorExcelVO> newVoList = voList.stream().filter(ListUtils.distinctByKey(ComMngPopulationMajorExcelVO::getDistinctPass)).collect(Collectors.toList());
                log.info("开始进入业务层处理逻辑");
                R r = communityService.listSavePopulationMajorExcelVO(newVoList, communityId);
                log.info("业务层处理逻辑完成");
                if (!R.isOk(r)) {
                    log.info("业务层处理成功");
                    List<ComMngPopulationMajorMistakeExcelVO> list = JSONArray.parseArray(JSONArray.toJSONString(r.getData()), ComMngPopulationMajorMistakeExcelVO.class);
//                    String errMsg = r.getMsg();
//                    List<ComMngPopulationImportErrorVO> errorList = JSON.parseArray(errMsg,ComMngPopulationImportErrorVO.class);
//                    if(!errorList.isEmpty()){
//                        populationImportErrorVOList.addAll(errorList);
                vo.setName(oneData.get(1));
                if (StringUtils.isEmpty(oneData.get(2))) {
                    ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                    index++;
                    setMistake(oneData, mistake);
                    mistake.setMistake("身份证号不可为空，请填写身份证");
                    mistakes.add(mistake);
                    continue;
                }

                // 判断身份证号码位数
                if (oneData.get(2).length() != 18) {
                    ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                    index++;
                    setMistake(oneData, mistake);
                    mistake.setMistake("身份证号位数有误，请检查身份证号码是否正确");
                    mistakes.add(mistake);
                    continue;
                }
                vo.setCardNo(oneData.get(2).toUpperCase());
                // 根据身份证号码解析年龄以及性别
                // 获取用户生日
                String birthday = vo.getCardNo().substring(6, 14);
                if (StringUtils.isNotEmpty(birthday)) {
                    String year = birthday.substring(0, 4);
                    String month = birthday.substring(4, 6);
                    String day = birthday.substring(6, 8);
                    vo.setBirthday(year + "-" + month + "-" + day);
                }
                // //设置用户年龄
                // vo.setAge(AgeUtils.getAgeFromBirthTime(birthday));
                // 获取用户性别
                int sex = Integer.parseInt(vo.getCardNo().substring(16, 17));
                if (sex % 2 == 1) {
                    vo.setSex(PopulSexEnum.nan.getCode());
                } else {
                    vo.setSex(PopulSexEnum.nv.getCode());
                }

                if (StringUtils.isEmpty(oneData.get(3))) {
                    index++;
                    ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                    setMistake(oneData, mistake);
                    mistake.setMistake("街路巷不可为空");
                    mistakes.add(mistake);
                    continue;
                }
                vo.setRoad(oneData.get(3));
                if (StringUtils.isEmpty(oneData.get(4))) {
                    index++;
                    ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                    setMistake(oneData, mistake);
                    mistake.setMistake("小区号不可为空");
                    mistakes.add(mistake);
                    continue;
                }
                vo.setDoorNo(oneData.get(4).trim());
                // if(StringUtils.isEmpty(oneData.get(5))){
                // index++;
                // ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                // setMistake(oneData, mistake);
                // mistake.setMistake("楼排号不可为空");
                // mistakes.add(mistake);
                // continue;
                // }
                if (StringUtils.isNotEmpty(oneData.get(5))) {
                    vo.setFloor(oneData.get(5).trim());
                }
                // if(StringUtils.isEmpty(oneData.get(6))){
                // index++;
                // ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                // setMistake(oneData, mistake);
                // mistake.setMistake("单元号不可为空");
                // mistakes.add(mistake);
                // continue;
                // }
                if (StringUtils.isNotEmpty(oneData.get(6))) {
                    vo.setUnitNo(oneData.get(6).trim());
                }
                // if(StringUtils.isEmpty(oneData.get(7))){
                // index++;
                // ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                // setMistake(oneData, mistake);
                // mistake.setMistake("户室不可为空");
                // mistakes.add(mistake);
                // continue;
                // }
                if (StringUtils.isNotEmpty(oneData.get(7))) {
                    vo.setHouseNo(oneData.get(7).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(8))) {
                    Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(8));
                    if (isOk.equals(-1)) {
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的是否租住有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setIsRent(isOk);
                } else {
                    vo.setIsRent(PopulHouseUseEnum.getCodeByName("否"));
                }
                if (StringUtils.isNotEmpty(oneData.get(9))) {
                    Integer isOk = PopulHouseStatusEnum.getCodeByName(oneData.get(9).trim());
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的房屋状态有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setHouseStatus(isOk);
                }
                if (StringUtils.isNotEmpty(oneData.get(10))) {
                    Integer isOk = PopulHousePurposeEnum.getCodeByName(oneData.get(10).trim());
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的房屋用途有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setHousePurpose(PopulHousePurposeEnum.getCodeByName(oneData.get(10).trim()));
                }
                if (StringUtils.isNotEmpty(oneData.get(11))) {
                    Integer isOk = PopulHouseControlStatusEnum.getCodeByName(oneData.get(11));
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的管控状态有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setControlStatus(isOk);
                }
                if (StringUtils.isNotEmpty(oneData.get(12))) {
                    Integer isOk = PopulOutOrLocalEnum.getCodeByName(oneData.get(12));
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的本地/外地有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setOutOrLocal(isOk);
                } else {
                    vo.setOutOrLocal(PopulOutOrLocalEnum.getCodeByName("本地"));
                }
                if (StringUtils.isNotEmpty(oneData.get(13))) {
                    vo.setCensusRegister(oneData.get(13).trim());
                }
                if (StringUtils.isEmpty(oneData.get(14))) {
                    Integer isOk = PopulCultureLevelEnum.getCodeByName(oneData.get(14));
                    if (isOk.equals(-1)) {
                        index++;
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的文化程度有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setOutOrLocal(isOk);
                }
                if (StringUtils.isNotEmpty(oneData.get(15))) {
                    vo.setEconomicCondition(oneData.get(15).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(16))) {
                    vo.setPersonType(oneData.get(16).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(17))) {
                    vo.setFamilySituation(oneData.get(17).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(18))) {
                    vo.setCustodyPerson(oneData.get(18).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(19))) {
//                    Integer isOk = PopulRelationEnum.getCodeByName(oneData.get(19));
//                    if (isOk.equals(-1)) {
//                        index++;
//                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
//                        setMistake(oneData, mistake);
//                        mistake.setMistake("您填写的与监护人关系有误");
//                        mistakes.add(mistake);
//                        continue;
//                    }
                    mistakes.addAll(list);
                    log.info("将错误数据存入redis中");
                    valueOperations.set(key,JSONArray.toJSONString(mistakes),1, TimeUnit.HOURS);
                    log.info("将错误数据存入redis中成功");
                    throw new ServiceException("500", key);
                }else{
                    log.info("业务层处理逻辑失败");
                    if(!mistakes.isEmpty()){
                        log.info("业务层处理逻辑失败，将错误数据缓存到redis中");
                        valueOperations.set(key,JSONArray.toJSONString(mistakes),1, TimeUnit.HOURS);
                        log.info("业务层处理逻辑失败，将错误数据缓存到redis中成功");
                        throw new ServiceException("500", key);
                    vo.setCustodyRelation(oneData.get(19).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(20))) {
                    vo.setCustodyCardNo(oneData.get(20).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(21))) {
                    vo.setCustodyPersonPhone(oneData.get(21).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(22))) {
                    vo.setCustodyAddress(oneData.get(22).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(23))) {
                    vo.setHelpPerson(oneData.get(23).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(24))) {
                    vo.setHelpPersonPhone(oneData.get(24).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(25))) {
                    vo.setHelpMethod(oneData.get(25).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(26))) {
                    vo.setHelpSituation(oneData.get(26).trim());
                }
                if (StringUtils.isNotEmpty(oneData.get(27))) {
                    Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(27));
                    if (isOk.equals(-1)) {
                        ComMngPopulationMajorMistakeExcelVO mistake = new ComMngPopulationMajorMistakeExcelVO();
                        index++;
                        setMistake(oneData, mistake);
                        mistake.setMistake("您填写的是否违法犯罪有误");
                        mistakes.add(mistake);
                        continue;
                    }
                    vo.setIsCrime(isOk);
                } else {
                    vo.setIsCrime(PopulHouseUseEnum.getCodeByName("否"));
                }
                if (StringUtils.isNotEmpty(oneData.get(28))) {
                    vo.setCrimeSutiation(oneData.get(28).trim());
                }
                vo.getUserTagStr().add("精神障碍患者");
                for (int i = 29; i < headSize; i++) {
                    if (oneData.get(i) != null && oneData.get(i).equals("是")) {
                        vo.getUserTagStr().add(headData.get(i).substring(0, headData.get(i).indexOf("(")));
                    }
                }
            } catch (NumberFormatException e) {
                log.info("处理数据时失败");
                e.printStackTrace();
//                List<ComMngPopulationImportErrorVO> populationImportErrorVOList = new ArrayList<>();
//                ComMngPopulationImportErrorVO importErrorVO = new ComMngPopulationImportErrorVO();
//                importErrorVO.setErrorPosition("第" + index + "行");
//                importErrorVO.setErrorMsg("数据格式有误，请检查文档内数据");
//                populationImportErrorVOList.add(importErrorVO);
                log.error("数据格式有误，第" + index + "行");
                throw new ServiceException("500", "导入失败111");
                // 将重复的数据进行MD5加密实现去重
                String distinct = vo.getName() + vo.getCardNo() + vo.getRoad() + vo.getDoorNo() + vo.getFloor()
                    + vo.getUnitNo() + vo.getHouseNo();
                try {
                    String distinctPass = PayUtil.MD5(distinct);
                    if (StringUtils.isNotEmpty(distinctPass)) {
                        vo.setDistinctPass(distinctPass);
                    }
                } catch (Exception e) {
                    log.error("组装MD5加密字段失败，数据表格行数：" + index);
                    continue;
                }
                voList.add(vo);
                index++;
            }
            // 客户需要暂时注释，等客户处理完成需要恢复
            // if(populationImportErrorVOList.isEmpty()){
            // R r = communityService.listSavePopulationServeExcelVO(voList, communityId);
            // if (!R.isOk(r)) {
            // throw new ServiceException(r.getMsg());
            // }
            // }else{
            // throw new ServiceException("500", JSON.toJSONString(populationImportErrorVOList));
            // }
            // 根据list中的IdCard城市来去重
            List<ComMngPopulationMajorExcelVO> newVoList =
                voList.stream().filter(ListUtils.distinctByKey(ComMngPopulationMajorExcelVO::getDistinctPass))
                    .collect(Collectors.toList());
            log.info("开始进入业务层处理逻辑");
            R r = communityService.listSavePopulationMajorExcelVO(newVoList, communityId, userId);
            log.info("业务层处理逻辑完成");
            if (!R.isOk(r)) {
                log.info("业务层处理成功");
                List<ComMngPopulationMajorMistakeExcelVO> list = JSONArray
                    .parseArray(JSONArray.toJSONString(r.getData()), ComMngPopulationMajorMistakeExcelVO.class);
                // String errMsg = r.getMsg();
                // List<ComMngPopulationImportErrorVO> errorList =
                // JSON.parseArray(errMsg,ComMngPopulationImportErrorVO.class);
                // if(!errorList.isEmpty()){
                // populationImportErrorVOList.addAll(errorList);
                // }
                mistakes.addAll(list);
                log.info("将错误数据存入redis中");
                valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS);
                log.info("将错误数据存入redis中成功");
                throw new ServiceException("500", key);
            } else {
                log.info("业务层处理逻辑失败");
                if (!mistakes.isEmpty()) {
                    log.info("业务层处理逻辑失败，将错误数据缓存到redis中");
                    valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS);
                    log.info("业务层处理逻辑失败，将错误数据缓存到redis中成功");
                    throw new ServiceException("500", key);
                }
            }
        } catch (NumberFormatException e) {
            log.info("处理数据时失败");
            e.printStackTrace();
            // List<ComMngPopulationImportErrorVO> populationImportErrorVOList = new ArrayList<>();
            // ComMngPopulationImportErrorVO importErrorVO = new ComMngPopulationImportErrorVO();
            // importErrorVO.setErrorPosition("第" + index + "行");
            // importErrorVO.setErrorMsg("数据格式有误，请检查文档内数据");
            // populationImportErrorVOList.add(importErrorVO);
            log.error("数据格式有误，第" + index + "行");
            throw new ServiceException("500", "导入失败111");
        }
    }

    private void setMistake(Map<Integer, String> map, ComMngPopulationMajorMistakeExcelVO vo){
    private void setMistake(Map<Integer, String> map, ComMngPopulationMajorMistakeExcelVO vo) {
        vo.setSerialNumber(map.get(0));
        vo.setName(map.get(1));
        vo.setCardNo(map.get(2));
@@ -479,103 +458,103 @@
        vo.setCrimeSutiation(map.get(28));
    }

        private String convertMarriage(String ma) {
    private String convertMarriage(String ma) {

            if(Objects.equals("已",ma) || Objects.equals("一",ma)){
                ma = "已婚";
            }else if(Objects.equals("未",ma) || Objects.equals("未婚婚",ma)){
                ma = "未婚";
            }else if(Objects.equals("初",ma)){
                ma = "初婚";
            }else if(Objects.equals("再",ma)){
                ma = "再婚";
            }else if(Objects.equals("复",ma)){
                ma = "复婚";
            }else if(Objects.equals("丧",ma) || Objects.equals("丧偶偶",ma)){
                ma = "丧偶";
            }else if(Objects.equals("离",ma) || Objects.equals("离异",ma) || Objects.equals("离异婚",ma)){
                ma = "离婚";
            }else if(Objects.equals("分",ma)){
                ma = "分居";
            }

            return ma;
        if (Objects.equals("已", ma) || Objects.equals("一", ma)) {
            ma = "已婚";
        } else if (Objects.equals("未", ma) || Objects.equals("未婚婚", ma)) {
            ma = "未婚";
        } else if (Objects.equals("初", ma)) {
            ma = "初婚";
        } else if (Objects.equals("再", ma)) {
            ma = "再婚";
        } else if (Objects.equals("复", ma)) {
            ma = "复婚";
        } else if (Objects.equals("丧", ma) || Objects.equals("丧偶偶", ma)) {
            ma = "丧偶";
        } else if (Objects.equals("离", ma) || Objects.equals("离异", ma) || Objects.equals("离异婚", ma)) {
            ma = "离婚";
        } else if (Objects.equals("分", ma)) {
            ma = "分居";
        }

        private String convertRelation(String ra) {
            if(ra.contains("户主")){
                ra = "户主";
            }else if(Objects.equals(ra,"女儿")){
                ra = "女";
            }else if(Objects.equals(ra,"儿子") || Objects.equals(ra,"儿")){
                ra = "子";
            }else if(Objects.equals(ra,"大女")){
                ra = "长女";
            }else if(Objects.equals(ra,"二女") || Objects.equals(ra,"此女")){
                ra = "次女";
            }else if(Objects.equals(ra,"二子")){
                ra = "次子";
            }else if(Objects.equals(ra,"长男") || ra.contains("长子")){
                ra = "长子";
            }else if(Objects.equals(ra,"非亲属") || ra.contains("女友")  || ra.contains("女朋友")
                    || ra.contains("男友") || ra.contains("男朋友")){
                ra = "其他";
            }else if(Objects.equals(ra,"姐")){
                ra = "姐姐";
            }else if(Objects.equals(ra,"户主")){
                ra = "本人";
            }else if(Objects.equals(ra,"妻子") || Objects.equals(ra,"媳妇")){
                ra = "妻";
            }else if(Objects.equals(ra,"哥") || Objects.equals(ra,"哥哥")){
                ra = "兄";
            }else if(Objects.equals(ra,"丈夫")){
                ra = "夫";
            }else if(Objects.equals(ra,"丈母娘")){
                ra = "岳母";
            }else if(Objects.equals(ra,"继子") || Objects.equals(ra,"养子")){
                ra = "养子或继子";
            }else if(Objects.equals(ra,"继女") || Objects.equals(ra,"养女")){
                ra = "养女或继女";
            }else if(ra.contains("儿媳")||ra.contains("长媳")){
                ra = "儿媳";
            }else if(Objects.equals(ra,"三姨") || Objects.equals(ra,"姨姨")){
                ra = "姨母";
            }else if(Objects.equals(ra,"二外孙女")){
                ra = "外孙女";
            }else if(Objects.equals(ra,"外孙")){
                ra = "外孙子";
            }else if(Objects.equals(ra,"侄儿")){
                ra = "侄子";
            }else if(ra.contains("表")){
                ra = "表兄弟、表姐妹";
            }else if(Objects.equals(ra,"继父") || Objects.equals(ra,"养父")){
                ra = "继父或养父";
            }else if(Objects.equals(ra,"父")){
                ra = "父亲";
            }else if(Objects.equals(ra,"弟弟")){
                ra = "弟";
            }else if(Objects.equals(ra,"孙")){
                ra = "孙子";
            }else if(Objects.equals(ra,"孙媳妇") || Objects.equals(ra,"外孙媳妇")){
                ra = "孙媳妇或外孙媳妇";
            }else if(Objects.equals(ra,"妹")){
                ra = "妹妹";
            }else if(Objects.equals(ra,"祖父母")){
                ra = "祖父母或外祖父母";
            }else if(Objects.equals(ra,"姐妹") || Objects.equals(ra,"兄弟")){
                ra = "兄弟姐妹";
            }else if(Objects.equals(ra,"其他姐妹") || Objects.equals(ra,"其他兄弟")){
                ra = "其他兄弟姐妹";
            }else if(Objects.equals(ra,"奶奶")){
                ra = "祖母";
            }else if(Objects.equals(ra,"爷爷")){
                ra = "祖父";
            }else if(Objects.equals(ra,"爷爷")){
                ra = "祖父";
            }else if(ra.contains("堂")){
                ra = "堂兄弟、堂姐妹";
            }
        return ma;
    }

            return ra;
    private String convertRelation(String ra) {
        if (ra.contains("户主")) {
            ra = "户主";
        } else if (Objects.equals(ra, "女儿")) {
            ra = "女";
        } else if (Objects.equals(ra, "儿子") || Objects.equals(ra, "儿")) {
            ra = "子";
        } else if (Objects.equals(ra, "大女")) {
            ra = "长女";
        } else if (Objects.equals(ra, "二女") || Objects.equals(ra, "此女")) {
            ra = "次女";
        } else if (Objects.equals(ra, "二子")) {
            ra = "次子";
        } else if (Objects.equals(ra, "长男") || ra.contains("长子")) {
            ra = "长子";
        } else if (Objects.equals(ra, "非亲属") || ra.contains("女友") || ra.contains("女朋友") || ra.contains("男友")
            || ra.contains("男朋友")) {
            ra = "其他";
        } else if (Objects.equals(ra, "姐")) {
            ra = "姐姐";
        } else if (Objects.equals(ra, "户主")) {
            ra = "本人";
        } else if (Objects.equals(ra, "妻子") || Objects.equals(ra, "媳妇")) {
            ra = "妻";
        } else if (Objects.equals(ra, "哥") || Objects.equals(ra, "哥哥")) {
            ra = "兄";
        } else if (Objects.equals(ra, "丈夫")) {
            ra = "夫";
        } else if (Objects.equals(ra, "丈母娘")) {
            ra = "岳母";
        } else if (Objects.equals(ra, "继子") || Objects.equals(ra, "养子")) {
            ra = "养子或继子";
        } else if (Objects.equals(ra, "继女") || Objects.equals(ra, "养女")) {
            ra = "养女或继女";
        } else if (ra.contains("儿媳") || ra.contains("长媳")) {
            ra = "儿媳";
        } else if (Objects.equals(ra, "三姨") || Objects.equals(ra, "姨姨")) {
            ra = "姨母";
        } else if (Objects.equals(ra, "二外孙女")) {
            ra = "外孙女";
        } else if (Objects.equals(ra, "外孙")) {
            ra = "外孙子";
        } else if (Objects.equals(ra, "侄儿")) {
            ra = "侄子";
        } else if (ra.contains("表")) {
            ra = "表兄弟、表姐妹";
        } else if (Objects.equals(ra, "继父") || Objects.equals(ra, "养父")) {
            ra = "继父或养父";
        } else if (Objects.equals(ra, "父")) {
            ra = "父亲";
        } else if (Objects.equals(ra, "弟弟")) {
            ra = "弟";
        } else if (Objects.equals(ra, "孙")) {
            ra = "孙子";
        } else if (Objects.equals(ra, "孙媳妇") || Objects.equals(ra, "外孙媳妇")) {
            ra = "孙媳妇或外孙媳妇";
        } else if (Objects.equals(ra, "妹")) {
            ra = "妹妹";
        } else if (Objects.equals(ra, "祖父母")) {
            ra = "祖父母或外祖父母";
        } else if (Objects.equals(ra, "姐妹") || Objects.equals(ra, "兄弟")) {
            ra = "兄弟姐妹";
        } else if (Objects.equals(ra, "其他姐妹") || Objects.equals(ra, "其他兄弟")) {
            ra = "其他兄弟姐妹";
        } else if (Objects.equals(ra, "奶奶")) {
            ra = "祖母";
        } else if (Objects.equals(ra, "爷爷")) {
            ra = "祖父";
        } else if (Objects.equals(ra, "爷爷")) {
            ra = "祖父";
        } else if (ra.contains("堂")) {
            ra = "堂兄弟、堂姐妹";
        }

        return ra;
    }
}