package com.panzhihua.common.listen; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.ValueOperations; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.fastjson.JSONArray; import com.google.common.collect.Lists; import com.panzhihua.common.constants.BcDictionaryConstants; import com.panzhihua.common.constants.Constants; import com.panzhihua.common.enums.*; import com.panzhihua.common.exceptions.ServiceException; import com.panzhihua.common.model.vos.BcDictionaryVO; import com.panzhihua.common.model.vos.R; import com.panzhihua.common.model.vos.community.ComMngPopulationCultExcelVO; import com.panzhihua.common.model.vos.community.ComMngPopulationCultMistakeExcelVO; import com.panzhihua.common.service.community.CommunityService; import com.panzhihua.common.utlis.ListUtils; import com.panzhihua.common.utlis.PayUtil; import com.panzhihua.common.utlis.StringUtils; import cn.hutool.core.date.DateUtil; import lombok.extern.slf4j.Slf4j; /** * @description: 邪教人员导入监听 * @author: txb */ @Slf4j public class ComMngPopulationCultExcelListen extends AnalysisEventListener> { /** * 每隔5条存储数据库,实际使用中可以3000条,然后清理list ,方便内存回收 */ private static final int BATCH_COUNT = 5000; private static int headSize = 0; List> list = new ArrayList>(); private CommunityService communityService; private Long communityId; private Long userId; private Map headData; private StringRedisTemplate stringRedisTemplate; public ComMngPopulationCultExcelListen(CommunityService communityService, Long communityId, Long userId, StringRedisTemplate stringRedisTemplate) { this.communityService = communityService; this.communityId = communityId; this.userId = userId; this.stringRedisTemplate = stringRedisTemplate; } @Override public void invoke(Map data, AnalysisContext context) { list.add(data); if (list.size() >= BATCH_COUNT) { saveData(); list.clear(); } } /** * 这里会一行行的返回头 * * @param headMap * @param context */ @Override public void invokeHeadMap(Map headMap, AnalysisContext context) { headSize = headMap.size(); headData = headMap; } @Override public void doAfterAllAnalysed(AnalysisContext context) { saveData(); log.info("所有数据解析完成!"); } /** * 不是固定的列只能手动处理 */ private void saveData() { R> dictionaryR = communityService.listDictionaryByKey(BcDictionaryConstants.FAMILY); log.info("开始导入邪教人员数据"); log.info("表格总数据:" + list.size()); if (list.size() == 0) { throw new ServiceException("000", "导入数据为空!"); } ValueOperations valueOperations = stringRedisTemplate.opsForValue(); String key = Constants.CULT_POPULATION_ERROR_LIST; Map dictMap = dictionaryR.getData().stream() .collect(Collectors.toMap(BcDictionaryVO::getDictName, BcDictionaryVO::getDictValue)); // Map dictMap = new HashMap<>(); // for(BcDictionaryVO vo : dictionaryR.getData()){ // dictMap.put(vo.getDictName(),Integer.valueOf(vo.getDictValue())); // } int index = 2; try { String[] parsePatterns = new String[] {"yyyy-MM-dd", "yyyy-M-dd", "yyyy-MM-d", "yyyy-M-d", "yyyy/MM/dd", "yyyy/M/dd", "yyyy/MM/d", "yyyy/M/d"}; ArrayList voList = Lists.newArrayList(); // List populationImportErrorVOList = new ArrayList<>(); ArrayList mistakes = Lists.newArrayList(); for (Map oneData : list) { ComMngPopulationCultExcelVO vo = new ComMngPopulationCultExcelVO(); if (StringUtils.isNotEmpty(oneData.get(0))) { vo.setSerialNumber(oneData.get(0)); } // 姓名和身份证都为空,为空户,无需操作,否则就解析年龄,性别,出生年月日 if (StringUtils.isEmpty(oneData.get(1))) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("名字不可为空,请填写姓名"); mistakes.add(mistake); continue; } vo.setName(oneData.get(1)); if (StringUtils.isEmpty(oneData.get(2))) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("身份证号不可为空,请填写身份证"); mistakes.add(mistake); continue; } // 判断身份证号码位数 if (oneData.get(2).length() != 18) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("身份证号位数有误,请检查身份证号码是否正确"); mistakes.add(mistake); continue; } vo.setCardNo(oneData.get(2).toUpperCase()); // 根据身份证号码解析年龄以及性别 // 获取用户生日 String birthday = vo.getCardNo().substring(6, 14); if (StringUtils.isNotEmpty(birthday)) { String year = birthday.substring(0, 4); String month = birthday.substring(4, 6); String day = birthday.substring(6, 8); vo.setBirthday(year + "-" + month + "-" + day); } // //设置用户年龄 // vo.setAge(AgeUtils.getAgeFromBirthTime(birthday)); // 获取用户性别 int sex = Integer.parseInt(vo.getCardNo().substring(16, 17)); if (sex % 2 == 1) { vo.setSex(PopulSexEnum.nan.getCode()); } else { vo.setSex(PopulSexEnum.nv.getCode()); } if (StringUtils.isNotEmpty(oneData.get(3))) { vo.setNation(oneData.get(3)); } if (StringUtils.isNotEmpty(oneData.get(4))) { vo.setProfession(oneData.get(4)); } if (StringUtils.isNotEmpty(oneData.get(5))) { Integer isOk = PopulMarriageEnum.getCodeByName(oneData.get(5)); if (isOk.equals(-1)) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("您填写的婚姻状况有误"); mistakes.add(mistake); continue; } vo.setMarriage(isOk); } if (StringUtils.isNotEmpty(oneData.get(6))) { Integer isOk = PopulCultureLevelEnum.getCodeByName(oneData.get(6)); if (isOk.equals(-1)) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("您填写的文化程度有误"); mistakes.add(mistake); continue; } vo.setCultureLevel(isOk); } if (StringUtils.isNotEmpty(oneData.get(7))) { Integer isOk = PopulPoliticalOutlookEnum.getCodeByName(oneData.get(7)); if (isOk.equals(-1)) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("您填写的政治面貌有误"); mistakes.add(mistake); continue; } vo.setPoliticalOutlook(isOk); } if (StringUtils.isEmpty(oneData.get(8))) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("街路巷不可为空"); mistakes.add(mistake); continue; } vo.setRoad(oneData.get(8)); if (StringUtils.isEmpty(oneData.get(9))) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("小区号不可为空"); mistakes.add(mistake); continue; } vo.setDoorNo(oneData.get(9).trim()); // if(StringUtils.isEmpty(oneData.get(10))){ // index++; // ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); // setMistake(oneData, mistake); // mistake.setMistake("楼排号不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(10))) { vo.setFloor(oneData.get(10).trim()); } // if(StringUtils.isEmpty(oneData.get(11))){ // index++; // ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); // setMistake(oneData, mistake); // mistake.setMistake("单元号不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(11))) { vo.setUnitNo(oneData.get(11).trim()); } // if(StringUtils.isEmpty(oneData.get(12))){ // index++; // ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); // setMistake(oneData, mistake); // mistake.setMistake("户室不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(12))) { vo.setHouseNo(oneData.get(12).trim()); } if (StringUtils.isNotEmpty(oneData.get(13))) { Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(13)); if (isOk.equals(-1)) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("您填写的是否租住有误"); mistakes.add(mistake); continue; } vo.setIsRent(isOk); } else { vo.setIsRent(PopulHouseUseEnum.getCodeByName("否")); } if (StringUtils.isNotEmpty(oneData.get(14))) { Integer isOk = PopulHouseStatusEnum.getCodeByName(oneData.get(14).trim()); if (isOk.equals(-1)) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("您填写的房屋状态有误"); mistakes.add(mistake); continue; } vo.setHouseStatus(isOk); } if (StringUtils.isNotEmpty(oneData.get(15))) { Integer isOk = PopulHousePurposeEnum.getCodeByName(oneData.get(15).trim()); if (isOk.equals(-1)) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("您填写的房屋用途有误"); mistakes.add(mistake); continue; } vo.setHousePurpose(PopulHousePurposeEnum.getCodeByName(oneData.get(15).trim())); } if (StringUtils.isNotEmpty(oneData.get(11))) { Integer isOk = PopulHouseControlStatusEnum.getCodeByName(oneData.get(16)); if (isOk.equals(-1)) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("您填写的管控状态有误"); mistakes.add(mistake); continue; } vo.setControlStatus(isOk); } if (StringUtils.isNotEmpty(oneData.get(17))) { Integer isOk = PopulOutOrLocalEnum.getCodeByName(oneData.get(17)); if (isOk.equals(-1)) { index++; ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); setMistake(oneData, mistake); mistake.setMistake("您填写的本地/外地有误"); mistakes.add(mistake); continue; } vo.setOutOrLocal(isOk); } else { vo.setOutOrLocal(PopulOutOrLocalEnum.getCodeByName("本地")); } if (StringUtils.isNotEmpty(oneData.get(18))) { vo.setCensusRegister(oneData.get(18).trim()); } if (StringUtils.isNotEmpty(oneData.get(19))) { vo.setCultName(oneData.get(19).trim()); } if (StringUtils.isNotEmpty(oneData.get(20))) { vo.setBasicSituation(oneData.get(20).trim()); } if (StringUtils.isNotEmpty(oneData.get(21))) { vo.setJoinCultDate(DateUtil.parse(oneData.get(21).trim(), parsePatterns)); } if (StringUtils.isNotEmpty(oneData.get(22))) { Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(22)); if (isOk.equals(-1)) { ComMngPopulationCultMistakeExcelVO mistake = new ComMngPopulationCultMistakeExcelVO(); index++; setMistake(oneData, mistake); mistake.setMistake("您填写的是否对外宣传有误"); mistakes.add(mistake); continue; } vo.setIsExternalPublicity(isOk); } else { vo.setIsExternalPublicity(PopulHouseUseEnum.getCodeByName("否")); } if (StringUtils.isNotEmpty(oneData.get(23))) { vo.setRemark(oneData.get(23).trim()); } vo.getUserTagStr().add("邪教人员"); for (int i = 24; i < headSize; i++) { if (oneData.get(i) != null && oneData.get(i).equals("是")) { vo.getUserTagStr().add(headData.get(i).substring(0, headData.get(i).indexOf("("))); } } // 将重复的数据进行MD5加密实现去重 String distinct = vo.getName() + vo.getCardNo() + vo.getRoad() + vo.getDoorNo() + vo.getFloor() + vo.getUnitNo() + vo.getHouseNo(); try { String distinctPass = PayUtil.MD5(distinct); if (StringUtils.isNotEmpty(distinctPass)) { vo.setDistinctPass(distinctPass); } } catch (Exception e) { log.error("组装MD5加密字段失败,数据表格行数:" + index); continue; } voList.add(vo); index++; } // 客户需要暂时注释,等客户处理完成需要恢复 // if(populationImportErrorVOList.isEmpty()){ // R r = communityService.listSavePopulationServeExcelVO(voList, communityId); // if (!R.isOk(r)) { // throw new ServiceException(r.getMsg()); // } // }else{ // throw new ServiceException("500", JSON.toJSONString(populationImportErrorVOList)); // } // 根据list中的IdCard城市来去重 List newVoList = voList.stream().filter(ListUtils.distinctByKey(ComMngPopulationCultExcelVO::getDistinctPass)) .collect(Collectors.toList()); log.info("开始进入业务层处理逻辑"); R r = communityService.listSavePopulationCultExcelVO(newVoList, communityId, userId); log.info("业务层处理逻辑完成"); if (!R.isOk(r)) { log.info("业务层处理成功"); List list = JSONArray.parseArray(JSONArray.toJSONString(r.getData()), ComMngPopulationCultMistakeExcelVO.class); // String errMsg = r.getMsg(); // List errorList = // JSON.parseArray(errMsg,ComMngPopulationImportErrorVO.class); // if(!errorList.isEmpty()){ // populationImportErrorVOList.addAll(errorList); // } mistakes.addAll(list); log.info("将错误数据存入redis中"); valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS); log.info("将错误数据存入redis中成功"); throw new ServiceException("500", key); } else { log.info("业务层处理逻辑失败"); if (!mistakes.isEmpty()) { log.info("业务层处理逻辑失败,将错误数据缓存到redis中"); valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS); log.info("业务层处理逻辑失败,将错误数据缓存到redis中成功"); throw new ServiceException("500", key); } } } catch (NumberFormatException e) { log.info("处理数据时失败"); e.printStackTrace(); // List populationImportErrorVOList = new ArrayList<>(); // ComMngPopulationImportErrorVO importErrorVO = new ComMngPopulationImportErrorVO(); // importErrorVO.setErrorPosition("第" + index + "行"); // importErrorVO.setErrorMsg("数据格式有误,请检查文档内数据"); // populationImportErrorVOList.add(importErrorVO); log.error("数据格式有误,第" + index + "行"); throw new ServiceException("500", "NumberFormatException"); } } private void setMistake(Map map, ComMngPopulationCultMistakeExcelVO vo) { vo.setSerialNumber(map.get(0)); vo.setName(map.get(1)); vo.setCardNo(map.get(2)); vo.setNation(map.get(3)); vo.setProfession(map.get(4)); vo.setMarriage(map.get(5)); vo.setCultureLevel(map.get(6)); vo.setPoliticalOutlook(map.get(7)); vo.setRoad(map.get(8)); vo.setDoorNo(map.get(9)); vo.setFloor(map.get(10)); vo.setUnitNo(map.get(11)); vo.setHouseNo(map.get(12)); vo.setIsRent(map.get(13)); vo.setHouseStatus(map.get(14)); vo.setHousePurpose(map.get(15)); vo.setControlStatus(map.get(16)); vo.setOutOrLocal(map.get(17)); vo.setCensusRegister(map.get(18)); vo.setCultName(map.get(19)); vo.setBasicSituation(map.get(20)); vo.setJoinCultDate(map.get(21)); vo.setIsExternalPublicity(map.get(22)); vo.setRemark(map.get(23)); } private String convertMarriage(String ma) { if (Objects.equals("已", ma) || Objects.equals("一", ma)) { ma = "已婚"; } else if (Objects.equals("未", ma) || Objects.equals("未婚婚", ma)) { ma = "未婚"; } else if (Objects.equals("初", ma)) { ma = "初婚"; } else if (Objects.equals("再", ma)) { ma = "再婚"; } else if (Objects.equals("复", ma)) { ma = "复婚"; } else if (Objects.equals("丧", ma) || Objects.equals("丧偶偶", ma)) { ma = "丧偶"; } else if (Objects.equals("离", ma) || Objects.equals("离异", ma) || Objects.equals("离异婚", ma)) { ma = "离婚"; } else if (Objects.equals("分", ma)) { ma = "分居"; } return ma; } private String convertRelation(String ra) { if (ra.contains("户主")) { ra = "户主"; } else if (Objects.equals(ra, "女儿")) { ra = "女"; } else if (Objects.equals(ra, "儿子") || Objects.equals(ra, "儿")) { ra = "子"; } else if (Objects.equals(ra, "大女")) { ra = "长女"; } else if (Objects.equals(ra, "二女") || Objects.equals(ra, "此女")) { ra = "次女"; } else if (Objects.equals(ra, "二子")) { ra = "次子"; } else if (Objects.equals(ra, "长男") || ra.contains("长子")) { ra = "长子"; } else if (Objects.equals(ra, "非亲属") || ra.contains("女友") || ra.contains("女朋友") || ra.contains("男友") || ra.contains("男朋友")) { ra = "其他"; } else if (Objects.equals(ra, "姐")) { ra = "姐姐"; } else if (Objects.equals(ra, "户主")) { ra = "本人"; } else if (Objects.equals(ra, "妻子") || Objects.equals(ra, "媳妇")) { ra = "妻"; } else if (Objects.equals(ra, "哥") || Objects.equals(ra, "哥哥")) { ra = "兄"; } else if (Objects.equals(ra, "丈夫")) { ra = "夫"; } else if (Objects.equals(ra, "丈母娘")) { ra = "岳母"; } else if (Objects.equals(ra, "继子") || Objects.equals(ra, "养子")) { ra = "养子或继子"; } else if (Objects.equals(ra, "继女") || Objects.equals(ra, "养女")) { ra = "养女或继女"; } else if (ra.contains("儿媳") || ra.contains("长媳")) { ra = "儿媳"; } else if (Objects.equals(ra, "三姨") || Objects.equals(ra, "姨姨")) { ra = "姨母"; } else if (Objects.equals(ra, "二外孙女")) { ra = "外孙女"; } else if (Objects.equals(ra, "外孙")) { ra = "外孙子"; } else if (Objects.equals(ra, "侄儿")) { ra = "侄子"; } else if (ra.contains("表")) { ra = "表兄弟、表姐妹"; } else if (Objects.equals(ra, "继父") || Objects.equals(ra, "养父")) { ra = "继父或养父"; } else if (Objects.equals(ra, "父")) { ra = "父亲"; } else if (Objects.equals(ra, "弟弟")) { ra = "弟"; } else if (Objects.equals(ra, "孙")) { ra = "孙子"; } else if (Objects.equals(ra, "孙媳妇") || Objects.equals(ra, "外孙媳妇")) { ra = "孙媳妇或外孙媳妇"; } else if (Objects.equals(ra, "妹")) { ra = "妹妹"; } else if (Objects.equals(ra, "祖父母")) { ra = "祖父母或外祖父母"; } else if (Objects.equals(ra, "姐妹") || Objects.equals(ra, "兄弟")) { ra = "兄弟姐妹"; } else if (Objects.equals(ra, "其他姐妹") || Objects.equals(ra, "其他兄弟")) { ra = "其他兄弟姐妹"; } else if (Objects.equals(ra, "奶奶")) { ra = "祖母"; } else if (Objects.equals(ra, "爷爷")) { ra = "祖父"; } else if (Objects.equals(ra, "爷爷")) { ra = "祖父"; } else if (ra.contains("堂")) { ra = "堂兄弟、堂姐妹"; } return ra; } }