在二手车业务线，现阶段无法实现车辆、人、车商信息的在业务审核流程中的数据查重应用，因此业务方为了达成这一目标，基于数据采集和数据查询，应运而生了关系图谱服务。
本文由于字数太多分为上下篇完成。
3.2、数据查重
上图是整个代码设计的UML类图，从上述图可以看出：
- 1、第一层的"API"层，对外提供HTTP接口。
- 2、第二层的"查询执行器"，包装API层的请求，并交由查询处理器处理。
- 3、第三层的"查询处理器"，抽象类仅派生一种业务场景多条件查询处理器
MultiSearchHandler。 - 3、第四层的"命中查询处理器"，按照业务场景派生出三个子类
PersonInfoHitQuerier、VehicleInfoHitQuerier、DealerInfoHitQuerier，同时提供了两种数据命中模式ExactSearchMode、SimilarSearchMode。
3.2.1、MultiSearchRequest
/**
* @description: 多条件查重查询条件DTO
* @Date : 2020/12/15 3:17 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MultiSearchRequest {
@ApiModelProperty(value="请求ID(可以使用UUID生成)",dataType="java.lang.String")
@NotNull(message = "请求ID[requestId]非空")
private String requestId;
@ApiModelProperty(value="查询条件",dataType="List<Condition>")
@NotNull(message = "查询条件[conditions]非空")
private List<Condition> conditions;
/**
* @description: 查询条件
* @Date : 2020/12/17 3:42 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public static class Condition{
@ApiModelProperty(value="查询字段类型",dataType="SourceTypeEnum")
private SourceTypeEnum sourceType;
@ApiModelProperty(value="查询字段名称",dataType="java.lang.String")
@NotNull(message = "查询字段名称[searchFieldName]不能为空！")
@NotEmpty(message = "查询字段名称[searchFieldName]不能为空！")
private String searchFieldName;
@ApiModelProperty(value="模糊查询分数区间",dataType="java.lang.Double")
@Size(max = 2, message = "查询字段名称[scoreRange]列表长度应该0-2")
private List<Double> scoreRange;
@ApiModelProperty(value="查询字段输入值",dataType="java.lang.String")
@NotNull(message = "查询字段输入值[searchFieldValue]不能为空！")
@NotEmpty(message = "查询字段输入值[searchFieldValue]不能为空！")
private String searchFieldValue;
@ApiModelProperty(value="查询字段描述",dataType="java.lang.String")
@NotNull(message = "查询字段描述[searchFieldDesc]不能为空！")
@NotEmpty(message = "查询字段描述[searchFieldDesc]不能为空！")
private String searchFieldDesc;
}
}
/**
* @description: 来源类型(1-主贷人;2-销售;3-销售主管;4-配偶;5-担保人;6-二手车卖方;7-紧急联系人1;8-紧急联系人2;9-车商法人;10-车商负责人;11-车商收款人;12-车辆;)
* @Date : 2020/11/27 4:08 PM
* @Author : 石冬冬-Seig Heil
*/
public enum SourceTypeEnum implements EnumValue {
CREDITOR_INFO(1,"主贷人"),
SELLER_INFO(2,"销售"),
SALES_MANAGER_INFO(3,"销售主管"),
MATE(4,"配偶"),
GUARANTOR_INFO(5,"担保人"),
SELLER_INFO_OF_OLD_CAR(6,"二手车卖方"),
EMERGENCY_CONTACT_ONE(7,"紧急联系人1"),
EMERGENCY_CONTACT_TWO(8,"紧急联系人2"),
CAR_DEALER_LEGAL_PERSON(9,"车商法人"),
CAR_DEALER_PRINCIPAL(10,"车商负责人"),
CAR_DEALER_PAYEE(11,"车商收款人"),
VEHICLE_INFO(12,"车辆"),
;
private int index;
private String value;
SourceTypeEnum(int index, String value ){
this.value = value;
this.index = index;
}
@Override
public int getIndex() {
return index;
}
@Override
public String getName() {
return value;
}
/**
* 根据索引获取对象
* @param index
* @return
*/
public static SourceTypeEnum getByIndex(int index){
return Stream.of(SourceTypeEnum.values()).filter(each -> each.getIndex() == index).findFirst().get();
}
/**
* 根据索引获取名称
* @param index
* @return
*/
public static String getNameByIndex(int index){
SourceTypeEnum find = Stream.of(SourceTypeEnum.values()).filter(each -> each.getIndex() == index).findFirst().get();
return null == find ? "" : find.getName();
}
}
3.2.2、SearchResultRe
/**
* @description: 查重结果对象
* @Date : 2020/12/15 3:29 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SearchResultRe<E extends SearchResultRe.Record>{
@ApiModelProperty(value="提示信息",dataType="java.lang.String")
private String message;
@ApiModelProperty(value="查询字段名称",dataType="java.lang.String")
private String searchFieldName;
@ApiModelProperty(value="查询字段输入值",dataType="java.lang.String")
private String searchFieldValue;
@ApiModelProperty(value="查询字段描述",dataType="java.lang.String")
private String searchFieldDesc;
@ApiModelProperty(value="命中条数",dataType="java.lang.Integer")
private int hitCount;
@ApiModelProperty(value="命中记录",dataType="List<Record>")
private List<E> hitRecords;
/**
* 累加命中条数
* @param delta
*/
public synchronized void increaseHitCount(int delta){
this.hitCount += delta;
}
/**
* 累加命中记录
* @param hits
*/
public synchronized void addHits(List<E> hits){
if(!isEmpty(hits)){
if(isEmpty(this.hitRecords)){
this.hitRecords = new ArrayList<>(hits.size());
}
this.hitRecords.addAll(hits);
}
}
/**
* 判断集合元素是否为空
* @param collection
* @return
*/
boolean isEmpty(Collection<?> collection){
return null == collection || collection.isEmpty();
}
/**
* @description: 查重结果对象
* @Date : 2020/12/15 3:29 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@SuperBuilder
@NoArgsConstructor
@AllArgsConstructor
public static class Record{
@ApiModelProperty(value="主键ID",dataType="java.lang.Integer")
private Integer id;
@ApiModelProperty(value="外部数据id",dataType="java.lang.String")
private String externalId;
@ApiModelProperty(value="业务单号",dataType="java.lang.String")
private String appCode;
@ApiModelProperty(value="数据单号",dataType="java.lang.String")
private String dataCode;
/**
* {@link SceneEnum#getIndex()}
*/
@ApiModelProperty(value="场景",dataType="java.lang.Integer")
private Integer scene;
private String sceneDesc;
/**
* {@link SourceTypeEnum#getIndex()}
*/
@ApiModelProperty(value="来源类型",dataType="java.lang.Integer")
private Integer sourceType;
private String sourceTypeDesc;
}
/**
* @description: 自然人信息
* @Date : 2020/12/15 3:29 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@SuperBuilder
@NoArgsConstructor
public static class PersonInfoRecord extends Record{
@ApiModelProperty(value="姓名",dataType="java.lang.String")
private String name;
@ApiModelProperty(value="身份证号",dataType="java.lang.String")
private String idNo;
@ApiModelProperty(value="手机号",dataType="java.lang.String")
private String mobile;
@ApiModelProperty(value="银行卡号",dataType="java.lang.String")
private String creditCardNo;
@ApiModelProperty(value="省",dataType="java.lang.String")
private String provinceAddress;
@ApiModelProperty(value="市",dataType="java.lang.String")
private String cityAddress;
@ApiModelProperty(value="区",dataType="java.lang.String")
private String districtAddress;
@ApiModelProperty(value="详细地址",dataType="java.lang.String")
private String detailAddress;
}
/**
* @description: 车辆信息
* @Date : 2020/12/15 3:29 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@SuperBuilder
@NoArgsConstructor
public static class VehicleInfoRecord extends Record{
@ApiModelProperty(value="VIN码",dataType="java.lang.String")
private String vin;
@ApiModelProperty(value="公里数",dataType="java.lang.Integer")
private Integer mileage;
@ApiModelProperty(value="评估备注",dataType="java.lang.String")
private String evaluateRemark;
}
}
3.2.3、SearchController
/**
* @description: 查重数据
* @Date : 2020/12/15 4:03 PM
* @Author : 石冬冬-Seig Heil
*/
@RestController
@Api(description = "查重数据", tags = "查重数据")
@RequestMapping("/search")
public class SearchController {
@Autowired
SearchQueryExecutor searchQueryExecutor;
/**
* 多条件查询
* @return
*/
@PostMapping("/multiQuery")
@ApiOperation(value = "多条件查询", notes = "多条件查询")
@NoAuthRequired
@OvalValidator(value = "多条件查询[multiQuery]")
public Result<List<SearchResultRe>> multiQuery(@RequestBody MultiSearchRequest request) {
return searchQueryExecutor.execute(SearchQueryExecutor.Type.MULTI,request);
}
}
3.2.4、SearchQueryExecutor
/**
* @description: 查询执行器
* @Date : 2020/12/16 11:27 AM
* @Author : 石冬冬-Seig Heil
*/
@Component
public class SearchQueryExecutor {
final Map<Type,AbstractSearchHandler> HANDLER_MAP = Maps.newHashMap();
@Resource
MultiSearchHandler multiSearchHandler;
@PostConstruct
void init(){
HANDLER_MAP.put(Type.MULTI, multiSearchHandler);
}
public enum Type{
/**
* 单项查询
*/
SIMPLE,
/**
* 多项
*/
MULTI
}
/**
* 执行
* @param type
* @param request
* @param <T>
*/
public <T> Result<List<SearchResultRe>> execute(Type type, T request){
if(Type.MULTI == type){
SearchQueryContext<MultiSearchRequest> context = SearchQueryContext.<MultiSearchRequest>builder().param((MultiSearchRequest)request).build();
HANDLER_MAP.get(type).execute(context);
return Result.suc(context.getResults());
}
return Result.suc();
}
}
3.2.5、SearchQueryContext
/**
* @description: 数据查重上下文对象
* @Date : 2020/12/16 11:06 AM
* @Author : 石冬冬-Seig Heil
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class SearchQueryContext<T> {
/**
* 请求ID
*/
private String requestId;
/**
* 查询条件
*/
private T param;
/**
* 查询结果
*/
private List<SearchResultRe> results;
/**
* 查询是否成功
*/
private boolean success;
/**
* 业务信息
*/
private String message;
public SearchQueryContext withSuccess(boolean success,String message){
this.setSuccess(success);
this.setMessage(message);
return this;
}
}
3.2.6、AbstractSearchHandler
/**
* @description: 抽象查询处理器
* @Date : 2020/12/16 11:08 AM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
public abstract class AbstractSearchHandler<T> {
/**
* 上下文容器
*/
final ThreadLocal<SearchQueryContext<T>> CONTEXT = new ThreadLocal<>();
@Resource
DiamondConfigProxy diamondConfigProxy;
@Resource
HitQuerierManager hitQuerierManager;
@Resource
ExecutorService searchQueryPoolExecutor;
/**
* 外部调用执行方法
* @param context
*/
public void execute(SearchQueryContext<T> context){
try {
CONTEXT.set(context);
doQuery(context);
} catch (Exception e) {
log.error("[execute],requestId={},ctx={}",context.getRequestId(),JSONObject.toJSONString(context),e);
}finally {
CONTEXT.remove();
}
}
/**
* 执行查询
* @param context
*/
abstract void doQuery(SearchQueryContext<T> context);
/**
* 构建一个异常的查询结果
* @param condition
* @param message
* @return
*/
SearchResultRe buildExceptionSearchResult(MultiSearchRequest.Condition condition,String message){
return SearchResultRe.builder()
.searchFieldName(condition.getSearchFieldName())
.searchFieldValue(condition.getSearchFieldValue())
.searchFieldDesc(condition.getSearchFieldDesc())
.message(message)
.hitRecords(Collections.emptyList())
.build();
}
}
3.2.7、MultiSearchHandler
多条件查询处理器，这里采用多线程，以条件维度，以子线程处理，主线程阻塞等待所有子线程处理，并把命中数据结果统一封装返回。
/**
* @description: 抽象查询处理器
* @Date : 2020/12/16 11:08 AM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class MultiSearchHandler extends AbstractSearchHandler<MultiSearchRequest> {
@Override
void doQuery(SearchQueryContext<MultiSearchRequest> context) {
context.setRequestId(context.getParam().getRequestId());
String requestId = context.getRequestId();
//条件列表
List<MultiSearchRequest.Condition> conditions = context.getParam().getConditions();
//结果列表
List<SearchResultRe> results = Lists.newArrayListWithExpectedSize(conditions.size());
//Future任务列表
List<Future<HitQuerierContext>> futureList = new ArrayList<>(conditions.size());
for(MultiSearchRequest.Condition condition : conditions){
//参数验证
if(!checkCondition(condition)){
results.add(super.buildExceptionSearchResult(condition, MessageFormat.format("查询异常|参数非法,field={0}",condition.getSearchFieldName())));
continue;
}
Future<HitQuerierContext> future = searchQueryPoolExecutor.submit(() -> {
HitQuerierContext hitQuerierContext = HitQuerierContext.builder()
.requestId(requestId)
.condition(condition)
.build();
hitQuerierManager.execute(hitQuerierContext);
return hitQuerierContext;
});
futureList.add(future);
}
int count = 0;
for (Future<HitQuerierContext> f : futureList) {
try {
HitQuerierContext hitQuerierContext = f.get();
results.add(hitQuerierContext.getHitResult());
} catch (InterruptedException | ExecutionException e) {
log.error("[doQuery][InterruptedException|ExecutionException],requestId={},context={}",requestId, JSONObject.toJSON(context),e);
results.add(super.buildExceptionSearchResult(conditions.get(count), MessageFormat.format("查询异常|运行时异常,message={0}",e.getMessage())));
continue;
}
count++;
}
context.setResults(results);
context.withSuccess(Boolean.TRUE,"查询成功");
}
/**
* 参数验证
* @param condition
*/
boolean checkCondition(MultiSearchRequest.Condition condition){
Set<String> configFields = diamondConfigProxy.searchFieldConfig().keySet();
log.info("[checkCondition],configFields={}", configFields.toArray());
String searchFieldName = condition.getSearchFieldName();
return configFields.contains(searchFieldName);
}
}
3.2.8、HitQuerierManager
命中查询管理器，由于一个查询条件会去多张表进行数据查询，然后再数据合并，因此这里吧查询器统一注册给
hitList，每个请求则统一调用execute方法循环迭代所有查询器处理。
/**
* @description: 命中查询管理器
* @Date : 2020/12/17 4:21 PM
* @Author : 石冬冬-Seig Heil
*/
@Component
public class HitQuerierManager {
static final List<AbstractHitQuerier> hitList = new ArrayList<>();
@Resource
PersonInfoHitQuerier personInfoHitQuerier;
@Resource
DealerInfoHitQuerier dealerInfoHitQuerier;
@Resource
VehicleInfoHitQuerier vehicleInfoHitQuerier;
@PostConstruct
void init(){
hitList.add(personInfoHitQuerier);
hitList.add(dealerInfoHitQuerier);
hitList.add(vehicleInfoHitQuerier);
}
/**
* 执行查询
* @param context
*/
public void execute(HitQuerierContext context){
MultiSearchRequest.Condition condition = context.getCondition();
SearchResultRe hitResult = SearchResultRe.builder()
.searchFieldValue(condition.getSearchFieldValue())
.searchFieldName(condition.getSearchFieldName())
.searchFieldDesc(condition.getSearchFieldDesc())
.message("查询成功")
.build();
for(AbstractHitQuerier querier : hitList){
SearchResultRe hit = querier.execute(context);
hitResult.increaseHitCount(hit.getHitCount());
hitResult.addHits(hit.getHitRecords());
}
context.setHitResult(hitResult);
}
}
3.2.9、HitQuerierContext
/**
* @description: 数据查询器上下文
* @Date : 2020/12/17 3:38 PM
* @Author : 石冬冬-Seig Heil
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class HitQuerierContext {
/**
* 请求ID
*/
private String requestId;
/**
* 查询条件
*/
private MultiSearchRequest.Condition condition;
/**
* 查询结果
*/
private SearchResultRe hitResult;
}
3.2.10、AbstractHitQuerier
/**
* @description: 抽象命中查询器
* @Date : 2020/12/17 3:38 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
public abstract class AbstractHitQuerier<E extends SearchResultRe.Record> {
/**
* 处理查询table
*/
protected final SearchFieldConfig.Table table;
@Resource
DiamondConfigProxy diamondConfigProxy;
public AbstractHitQuerier(SearchFieldConfig.Table table) {
this.table = table;
}
/**
* 查询
* @param context
* @return
*/
abstract SearchResultRe<E> doQuery(HitQuerierContext context);
/**
* 外部执行方法
* @param context
* @return
*/
public SearchResultRe execute(HitQuerierContext context){
SearchResultRe searchResultRe;
try {
if(!executeCurrent(context)){
searchResultRe = buildDefaultSearchResult(context.getCondition());
return searchResultRe;
}
searchResultRe = doQuery(context);
} catch (Exception e) {
log.error("[execute],requestId={},ctx={}",context.getRequestId(),JSONObject.toJSONString(context),e);
searchResultRe = buildDefaultSearchResult(context.getCondition());
searchResultRe.setMessage("查询异常|message=" + e.getMessage());
}
return searchResultRe;
}
/**
* 是否执行当前查询器
* @param context
* @return
*/
protected boolean executeCurrent(HitQuerierContext context){
return getSearchFieldConfig(context.getCondition().getSearchFieldName()).getTables().contains(table);
}
/**
* 构建默认查询结果
* @param condition
* @return
*/
SearchResultRe buildDefaultSearchResult(MultiSearchRequest.Condition condition){
String fieldName = condition.getSearchFieldName();
String fieldValue = condition.getSearchFieldValue();
SearchResultRe hitResult = SearchResultRe.builder()
.searchFieldName(fieldName)
.searchFieldValue(fieldValue)
.hitCount(0)
.hitRecords(Collections.emptyList())
.build();
return hitResult;
}
/**
* 获取配置字段
* @param fieldName
* @return
*/
SearchFieldConfig getSearchFieldConfig(String fieldName){
Map<String,SearchFieldConfig> mapping = diamondConfigProxy.searchFieldConfig();
SearchFieldConfig searchFieldConfig = mapping.get(fieldName);
log.info("[getSearchFieldConfig],fieldName={},config={}",fieldName, JSONObject.toJSONString(searchFieldConfig));
return searchFieldConfig;
}
/**
* 处理命中数据
* @param context
* @param queryForm
* @param recordsCaller
* @return
*/
List<E> hitRecords(HitQuerierContext context, Object queryForm, BiFunction<String,Object,List<E>> recordsCaller){
MultiSearchRequest.Condition condition = context.getCondition();
SearchFieldConfig searchFieldConfig = getSearchFieldConfig(condition.getSearchFieldName());
SearchFieldConfig.SearchMode searchMode = searchFieldConfig.getSearchMode();
return AbstractSearchMode.getAbstractSearchMode(searchMode).execute(searchFieldConfig, context, queryForm, recordsCaller);
}
}
/**
* @description: 车商信息命中查询
* @Date : 2020/12/17 3:51 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class DealerInfoHitQuerier extends AbstractHitQuerier<SearchResultRe.PersonInfoRecord> {
public DealerInfoHitQuerier() {
super(SearchFieldConfig.Table.DEALER_INFO);
}
@Resource
DealerInfoService dealerInfoService;
@Override
SearchResultRe<SearchResultRe.PersonInfoRecord> doQuery(HitQuerierContext context) {
MultiSearchRequest.Condition condition = context.getCondition();
SearchResultRe hitResult = super.buildDefaultSearchResult(condition);
DealerInfoForm queryForm = DealerInfoForm.builder().dataStatus(0).build();
List<SearchResultRe.PersonInfoRecord> hitRecords = super.hitRecords(context,queryForm,(String fieldName,Object searchForm) -> {
List<DealerInfo> recordList = dealerInfoService.queryList((DealerInfoForm)searchForm);
SearchFieldConfig config = getSearchFieldConfig(condition.getSearchFieldName());
if(SearchFieldConfig.SearchMode.SIMILAR == config.getSearchMode() && condition.getSearchFieldName().contains("Address")){
String provinceAddressFiledName = config.getMapping().get(config.getMapping().size()-1);
String cityAddressFiledName = config.getMapping().get(config.getMapping().size()-1);
String districtAddressFiledName = config.getMapping().get(config.getMapping().size()-1);
String detailAddressFiledName = config.getMapping().get(config.getMapping().size()-1);
return BeanConverter.convertFromDealer(recordList,
each -> BeanTool.getObjectValue(each, provinceAddressFiledName),
each -> BeanTool.getObjectValue(each, cityAddressFiledName),
each -> BeanTool.getObjectValue(each, districtAddressFiledName),
each -> BeanTool.getObjectValue(each, detailAddressFiledName));
}
return BeanConverter.convertFromDealer(recordList,
null, null, null,null);
});
hitResult.setHitRecords(hitRecords);
hitResult.setHitCount(hitRecords.size());
return hitResult;
}
}
/**
* @description: 自然人信息命中查询
* @Date : 2020/12/17 3:51 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class PersonInfoHitQuerier extends AbstractHitQuerier<SearchResultRe.PersonInfoRecord> {
/**
* 手机号字段输入
*/
static final String MOBILE_FIELD = "mobile";
/**
* 地址类字段命名后缀
*/
static final String ADDRESS_FIELD_SUFFIX = "Address";
public PersonInfoHitQuerier() {
super(SearchFieldConfig.Table.PERSON_INFO);
}
@Resource
PersonInfoService personInfoService;
@Override
SearchResultRe<SearchResultRe.PersonInfoRecord> doQuery(HitQuerierContext context) {
MultiSearchRequest.Condition condition = context.getCondition();
SearchResultRe hitResult = super.buildDefaultSearchResult(condition);
PersonInfoForm queryForm = PersonInfoForm.builder().dataStatus(0).build();
List<SearchResultRe.PersonInfoRecord> hitRecords = super.hitRecords(context,queryForm,(String fieldName,Object searchForm) -> {
List<PersonInfo> recordList = personInfoService.queryList((PersonInfoForm)searchForm);
if(MOBILE_FIELD.equals(condition.getSearchFieldName())){
return BeanConverter.convertFromPerson(recordList, each -> BeanTool.getObjectValue(each, fieldName),
null, null, null, null);
}
SearchFieldConfig config = getSearchFieldConfig(condition.getSearchFieldName());
if(SearchFieldConfig.SearchMode.SIMILAR == config.getSearchMode() && condition.getSearchFieldName().contains(ADDRESS_FIELD_SUFFIX)){
String provinceAddressFiledName = config.getMapping().get(0);
String cityAddressFiledName = config.getMapping().get(1);
String districtAddressFiledName = config.getMapping().get(2);
String detailAddressFiledName = config.getMapping().get(3);
return BeanConverter.convertFromPerson(recordList, null,
each -> BeanTool.getObjectValue(each, provinceAddressFiledName),
each -> BeanTool.getObjectValue(each, cityAddressFiledName),
each -> BeanTool.getObjectValue(each, districtAddressFiledName),
each -> BeanTool.getObjectValue(each, detailAddressFiledName));
}
return BeanConverter.convertFromPerson(recordList);
});
hitResult.setHitRecords(hitRecords);
hitResult.setHitCount(hitRecords.size());
return hitResult;
}
}
/**
* @description: 车辆信息命中查询
* @Date : 2020/12/17 3:51 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class VehicleInfoHitQuerier extends AbstractHitQuerier<SearchResultRe.VehicleInfoRecord> {
public VehicleInfoHitQuerier() {
super(SearchFieldConfig.Table.VEHICLE_INFO);
}
@Resource
VehicleInfoService vehicleInfoService;
@Override
SearchResultRe<SearchResultRe.VehicleInfoRecord> doQuery(HitQuerierContext context) {
MultiSearchRequest.Condition condition = context.getCondition();
SearchResultRe hitResult = super.buildDefaultSearchResult(condition);
VehicleInfoForm queryForm = VehicleInfoForm.builder().dataStatus(0).build();
List<SearchResultRe.VehicleInfoRecord> hitRecords = super.hitRecords(context,queryForm,(String fieldName,Object searchForm) -> {
List<VehicleInfo> recordList = vehicleInfoService.queryList((VehicleInfoForm)searchForm);
return BeanConverter.convertFromVehicle(recordList);
});
hitResult.setHitRecords(hitRecords);
hitResult.setHitCount(hitRecords.size());
return hitResult;
}
}
3.2.11、AbstractSearchMode
/**
* @description: 命中查询方式
* @see com.creditease.horus.core.search.querier.mode.impl.ExactSearchMode
* @see com.creditease.horus.core.search.querier.mode.impl.SimilarSearchMode
* @Date : 2020/12/24 3:27 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
public abstract class AbstractSearchMode {
/**
* 策略集合
*/
public static Map<SearchFieldConfig.SearchMode, AbstractSearchMode> abstractSearchModeMap = new HashMap<>();
/**
* 选择策略
* @param modeEnum
* @return
*/
public static AbstractSearchMode getAbstractSearchMode(SearchFieldConfig.SearchMode modeEnum){
return abstractSearchModeMap.get(modeEnum);
}
/**
* 策略抽象方法
* @param searchFieldConfig
* @param context
* @param queryForm
* @param recordsCaller
* @param <E>
* @return
*/
public abstract <E extends SearchResultRe.Record> List<E> execute(SearchFieldConfig searchFieldConfig,
HitQuerierContext context,
Object queryForm,
BiFunction<String, Object, List<E>> recordsCaller);
/**
* 拷贝配置的扩展参数到实体类
* @param searchFieldConfig
* @param queryForm
*/
public void copyExclude (SearchFieldConfig searchFieldConfig, Object queryForm){
Map<String,Object> exclude = searchFieldConfig.getExclude();
if(MapUtils.isNotEmpty(exclude)){
BeanTool.copyFromOneMap(exclude,queryForm);
}
}
}
/**
* @description: 命中查询方式-精确处理方式
* @Date : 2020/12/24 3:28 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class ExactSearchMode extends AbstractSearchMode {
{
abstractSearchModeMap.put(SearchFieldConfig.SearchMode.EXACT, this);
}
/**
* 精准查询
* @param searchFieldConfig
* @param context
* @param queryForm
* @param recordsCaller
* @return
*/
@Override
public <E extends SearchResultRe.Record> List<E> execute(SearchFieldConfig searchFieldConfig,
HitQuerierContext context,
Object queryForm,
BiFunction<String, Object, List<E>> recordsCaller) {
//拷贝配置的扩展参数到实体类
copyExclude (searchFieldConfig, queryForm);
//入参条件
String requestId = context.getRequestId();
String fieldName = context.getCondition().getSearchFieldName();
String fieldValue = context.getCondition().getSearchFieldValue();
//结果集
List<E> hitRecords;
//有效查询参数
Map<String,Object> sourceValues = Maps.newHashMap();
//入参一个参数映射成查询两个参数（入参mobile对应数据库primaryMobile,SecondMobile）
List<String> mapping = searchFieldConfig.getMapping();
if(null != mapping && !mapping.isEmpty()){
hitRecords = Lists.newArrayList();
for(String fieldNameAlias : mapping){
sourceValues.put(fieldNameAlias,fieldValue);
//map键值对拷贝到实体类
BeanTool.copyFromOneMap(sourceValues,queryForm);
log.info("[execute][hitRecords],requestId={},queryForm={}",requestId, JSONObject.toJSONString(queryForm));
List<E> hitRecordsTemp = recordsCaller.apply(fieldNameAlias,queryForm);
log.info("[execute][hitRecords],requestId={},hitRecordsFromDB={}",requestId,hitRecordsTemp.size());
if(CollectionsTools.isNotEmpty(hitRecordsTemp)){
hitRecords.addAll(hitRecordsTemp);
}
//抹掉本次参数（本次参数设置为null，并拷贝到查询实体类）
sourceValues.put(fieldNameAlias,null);
}
}else{
sourceValues.put(fieldName,fieldValue);
BeanTool.copyFromOneMap(sourceValues,queryForm);
log.info("[execute][hitRecords],requestId={},queryForm={}",requestId,JSONObject.toJSONString(queryForm));
hitRecords = recordsCaller.apply(fieldName,queryForm);
log.info("[execute][hitRecords],requestId={},hitRecordsFromDB={}",requestId,hitRecords.size());
}
return hitRecords;
}
}
/**
* @description: 命中查询方式-相似度处理方式
* @Date : 2020/12/24 3:28 PM
* @Author : 石冬冬-Seig Heil
*/
@Slf4j
@Component
public class SimilarSearchMode extends AbstractSearchMode {
/**
* 相似度切分四级详细字段
*/
final String SIMILAR_DETAIL_FIELD = "detail";
/**
* 输出模型，地址字段名称
*/
final String OUT_MODEL_PROVINCE_ADDRESS = "provinceAddress";
final String OUT_MODEL_CITY_ADDRESS = "cityAddress";
final String OUT_MODEL_DISTRICT_ADDRESS = "districtAddress";
final String OUT_MODEL_DETAIL_ADDRESS = "detailAddress";
/**
* 相似度查询类型
*/
final String SIMILAR_TYPE = "m:organization.organization.name";
{
abstractSearchModeMap.put(SearchFieldConfig.SearchMode.SIMILAR, this);
}
/**
* 相似度查询
* @param context
* @param queryForm
* @param recordsCaller
* @return
*/
@Override
public <E extends SearchResultRe.Record> List<E> execute(SearchFieldConfig searchFieldConfig,
HitQuerierContext context,
Object queryForm,
BiFunction<String, Object, List<E>> recordsCaller) {
//拷贝配置的扩展参数到实体类
copyExclude (searchFieldConfig, queryForm);
//入参条件
String requestId = context.getRequestId();
String fieldName = context.getCondition().getSearchFieldName();
String fieldValue = context.getCondition().getSearchFieldValue();
//入参条件拆分（地址拆分为省、市、区、详细）
List<String> mapping = searchFieldConfig.getMapping();
String[] fieldValues = fieldValue.split("\\|");
//需要分词比较的详细地址（具体字段名称）
String similarField = mapping.get(mapping.size()-1);
//用户传过来的详细地址
String addressDetail = fieldValues[fieldValues.length-1];
HsmmAddressNormalizer anm = new HsmmAddressNormalizer();
String addressDetailFormat = fieldValues[0] + fieldValues[1] + fieldValues[2]
+ ( (HashMap<String,String>)anm.splitAddress(addressDetail) ).get(SIMILAR_DETAIL_FIELD);
//有效查询参数
Map<String,Object> sourceValues;
//结果集
List<E> hitRecords = Collections.emptyList();
if(CollectionUtils.isEmpty(mapping)){
log.warn("[execute]diamond mapping is null,requestId={},fieldName={}",requestId,fieldName);
return hitRecords;
}
sourceValues = Maps.newHashMap();
String[] mappingValues = mapping.toArray(new String[mapping.size()]);
//最后一项不作为查询条件
for (int i = 0; i < mappingValues.length - 1; i++) {
sourceValues.put(mappingValues[i],fieldValues[i]);
}
BeanTool.copyFromOneMap(sourceValues,queryForm);
log.info("[execute][hitRecords],requestId={},queryForm={}",requestId, JSONObject.toJSONString(queryForm));
List<E> hitRecordsTemp = recordsCaller.apply(fieldName,queryForm);
log.debug("[execute][hitRecords],requestId={},hitRecordsFromDB={}",requestId,hitRecordsTemp.size());
if(CollectionsTools.isEmpty(hitRecordsTemp)){
return hitRecords;
}
//匹配度分数区间,长度限制0-2【分数区间为空-没有分数要求；分数区间长度为1-最低分要求；分数区间长度为2-分数区间要求】
List<Double> scoreRangeList = context.getCondition().getScoreRange();
Double[] scoreRange = CollectionUtils.isEmpty(scoreRangeList) ? new Double[0] : scoreRangeList.toArray(new Double[scoreRangeList.size()]);
if(scoreRange.length == 0){
//分数区间为空-没有分数要求
return hitRecordsTemp;
}else{
String provinceAddressFromDb;
String cityAddressFromDb;
String districtAddressFromDb;
String detailAddressFromDb;
String detailAddressFromDbFormat;
Double score;
hitRecords = Lists.newArrayListWithExpectedSize(hitRecordsTemp.size());
for (E item : hitRecordsTemp) {
//相似分数符合条件的添加到结果集
provinceAddressFromDb = BeanTool.getObjectValue(item, OUT_MODEL_PROVINCE_ADDRESS);
cityAddressFromDb = BeanTool.getObjectValue(item, OUT_MODEL_CITY_ADDRESS);
districtAddressFromDb = BeanTool.getObjectValue(item, OUT_MODEL_DISTRICT_ADDRESS);
detailAddressFromDb = BeanTool.getObjectValue(item, OUT_MODEL_DETAIL_ADDRESS);
detailAddressFromDbFormat = provinceAddressFromDb + cityAddressFromDb + districtAddressFromDb
+ ( (HashMap<String,String>)anm.splitAddress(detailAddressFromDb) ).get(SIMILAR_DETAIL_FIELD);
score = NLPUtil.getUtil().similarity(SIMILAR_TYPE, detailAddressFromDbFormat, addressDetailFormat);
log.info("[execute][hitRecords][similarScore],requestId={},addressDetailFormat={},addressDetailFromDb={},score={}",
requestId, addressDetailFormat, detailAddressFromDbFormat, score);
//分数区间长度为1-最低分要求
if(scoreRange.length == 1 && score >= scoreRange[0]){
hitRecords.add(item);
}
//分数区间长度为2-分数区间要求
if(scoreRange.length == 2 && score >= scoreRange[0] && score <= scoreRange[1]){
hitRecords.add(item);
}
}
}
return hitRecords;
}
}
4、扩展部分
4.1、查重服务请求参数
请求参数示例
{
"requestId":"1",
"conditions": [{
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "主贷人身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "350****8114118"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "主贷人手机号",
"searchFieldName": "mobile",
"searchFieldValue": "186****2901"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "销售手机号",
"searchFieldName": "mobile",
"searchFieldValue": "182****4023"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "二手车卖方身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "3522****2138"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "紧急联系人1手机号",
"searchFieldName": "mobile",
"searchFieldValue": "139****603"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "紧急联系人2手机号",
"searchFieldName": "mobile",
"searchFieldValue": "18****637"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "担保人身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "350****38"
}, {
"sourceType": "CREDITOR_INFO",
"searchFieldDesc": "担保人手机号",
"searchFieldName": "mobile",
"searchFieldValue": "15****020"
}, {
"sourceType": "VEHICLE_INFO",
"searchFieldDesc": "车辆VIN",
"searchFieldName": "vin",
"searchFieldValue": "LFV****3721"
}]
}
响应结果示例
部分数据脱敏展示了，比如手机号、银行卡号、身份证号。
{
"code": 0,
"data": [
{
"hitCount": 1,
"hitRecords": [
{
"appCode": "F2009111915000180101",
"creditCardNo": "-",
"dataCode": "P20121700551635",
"externalId": "100021309",
"id": 243677,
"idNo": "350122**4118",
"mobile": "186**01",
"name": "郑**",
"scene": 1003,
"sourceType": 1
}
],
"message": "查询成功",
"searchFieldDesc": "主贷人身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "3501**14118"
},
{
"hitCount": 2,
"hitRecords": [
{
"appCode": "F2009151915000180101",
"creditCardNo": "-",
"dataCode": "P20121700538028",
"externalId": "100022351",
"id": 219723,
"idNo": "-",
"mobile": "186**901",
"name": "郑**",
"scene": 1002,
"sourceType": 8
},
{
"appCode": "F2009111915000180101",
"creditCardNo": "-",
"dataCode": "P20121700551635",
"externalId": "100021309",
"id": 243677,
"idNo": "3501**118",
"mobile": "186**01",
"name": "郑**",
"scene": 1003,
"sourceType": 1
}
],
"message": "查询成功",
"searchFieldDesc": "主贷人手机号",
"searchFieldName": "mobile",
"searchFieldValue": "186**2901"
},
{
"hitCount": 1,
"hitRecords": [
{
"appCode": "F2009151915000180106",
"creditCardNo": "-",
"dataCode": "P20121700537447",
"externalId": "100022605",
"id": 218697,
"idNo": "3505**5550",
"mobile": "182**4023",
"name": "欧**",
"scene": 1003,
"sourceType": 1
}
],
"message": "查询成功",
"searchFieldDesc": "销售手机号",
"searchFieldName": "mobile",
"searchFieldValue": "182**023"
},
{
"hitCount": 6,
"hitRecords": [
{
"appCode": "F2011091915000180104",
"creditCardNo": "-",
"dataCode": "P20121700417419",
"externalId": "100038555",
"id": 7575,
"idNo": "35223**38",
"mobile": "-",
"name": "阮**",
"scene": 1002,
"sourceType": 6
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "二手车卖方身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "3522**32138"
},
{
"hitCount": 3,
"hitRecords": [
{
"appCode": "F2010191915000180107",
"creditCardNo": "-",
"dataCode": "P20121700481894",
"externalId": "100032121",
"id": 121031,
"idNo": "-",
"mobile": "139**603",
"name": "陈**",
"scene": 1002,
"sourceType": 7
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "紧急联系人1手机号",
"searchFieldName": "mobile",
"searchFieldValue": "139**603"
},
{
"hitCount": 3,
"hitRecords": [
{
"appCode": "F2007281915000180103",
"creditCardNo": "-",
"dataCode": "P20121700447417",
"externalId": "100010141",
"id": 60345,
"idNo": "-",
"mobile": "1810**37",
"name": "林**",
"scene": 1002,
"sourceType": 8
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "紧急联系人2手机号",
"searchFieldName": "mobile",
"searchFieldValue": "181**637"
},
{
"hitCount": 3,
"hitRecords": [
{
"appCode": "F2009281915000180104",
"creditCardNo": "-",
"dataCode": "P20121700442684",
"externalId": "100026661",
"id": 52033,
"idNo": "35012**938",
"mobile": "152**20",
"name": "许**",
"scene": 1003,
"sourceType": 1
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "担保人身份证号",
"searchFieldName": "idNo",
"searchFieldValue": "35012****195938"
},
{
"hitCount": 4,
"hitRecords": [
{
"appCode": "F2009281915000180104",
"creditCardNo": "-",
"dataCode": "P20121700442684",
"externalId": "100026661",
"id": 52033,
"idNo": "3501**8",
"mobile": "152050**",
"name": "许**",
"scene": 1003,
"sourceType": 1
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "担保人手机号",
"searchFieldName": "mobile",
"searchFieldValue": "152**"
},
{
"hitCount": 6,
"hitRecords": [
{
"appCode": "F2011091915000180104",
"dataCode": "V20121700417457",
"evaluateRemark": "正常。1.备胎槽照片重新拍摄，要求完整清晰。\n2.补充左右后叶子板流水槽照片\n3.补充左右前纵梁照片\n4.补充主副驾驶座椅滑轨照片\n5.有补领记录，补充车架拓印号照片（铁皮上的）",
"externalId": "100038555",
"id": 1045,
"mileage": 136029,
"scene": 1002,
"sourceType": 12,
"vin": "LFV4A24F7A30837**"
},
//省略其他
],
"message": "查询成功",
"searchFieldDesc": "车辆VIN",
"searchFieldName": "vin",
"searchFieldValue": "LFV4A24F7A3083**"
}
],
"msg": "操作成功",
"success": true
}
4.2、数据查重字段配置
为了提高数据查重接口的扩展性，基于配置化的元数据配置。
value包含如下参数
desc：参数描述，无业务逻辑；仅仅作为字段说明使用。searchMode：查询方式，目前仅支持两种EXACT（精准查询）、SIMILAR（相似度查询）。tables：json字符串数组，适用于该域查询的表，目前表共三个（PERSON_INFO、DEALER_INFO、VEHICLE_INFO）。mapping：查询字段映射，字符串数据，查询形式以或作为条件，结果集会进行合并。exclude：查询过滤条件，查询结果集以该配置参数作为过滤条件。字段key作为查询条件field，value作为条件。
{
"idNo": {
"desc": "身份证号",
"searchMode": "EXACT",
"mapping": [],
"tables": [
"PERSON_INFO",
"DEALER_INFO"
]
},
"name": {
"desc": "姓名",
"searchMode": "EXACT",
"mapping": [],
"tables": [
"PERSON_INFO",
"DEALER_INFO"
]
},
"mobile": {
"desc": "手机号（primaryMobile,SecondMobile）",
"searchMode": "EXACT",
"mapping": [
"primaryMobile",
"secondMobile"
],
"tables": [
"PERSON_INFO",
"DEALER_INFO"
],
"exclude": {
"sourceTypeScopeExclude": [
2,
3
]
}
},
"creditCardNo": {
"desc": "银行卡号",
"searchMode": "EXACT",
"mapping": [],
"tables": [
"PERSON_INFO",
"DEALER_INFO"
]
},
"companyAddress": {
"desc": "单位地址",
"searchMode": "SIMILAR",
"mapping": [
"companyAddressProvince",
"companyAddressCity",
"companyAddressDistrict",
"companyAddressDetail"
],
"tables": [
"PERSON_INFO",
"DEALER_INFO"
]
},
"censusAddress": {
"desc": "户籍地址",
"searchMode": "SIMILAR",
"mapping": [
"censusAddressProvince",
"censusAddressCity",
"censusAddressDistrict",
"censusAddressDetail"
],
"tables": [
"PERSON_INFO"
]
},
"residenceAddress": {
"desc": "居住地址",
"searchMode": "SIMILAR",
"mapping": [
"residenceAddressProvince",
"residenceAddressCity",
"residenceAddressDistrict",
"residenceAddressDetail"
],
"tables": [
"PERSON_INFO"
]
},
"vin": {
"desc": "车辆VIN",
"searchMode": "EXACT",
"mapping": [],
"tables": [
"VEHICLE_INFO"
]
}
}
5、总结
总体设计上运用了相关设计模式，并分成了多个模块，每个模块负责各自的业务逻辑职责。其中在数据查重接口设计上，考虑查询数据量比较多，基于输入的多个条件，运用并行处理，并把多个处理器的查询结果再进行合并，从而提高接口的性能。
